From c4a73f0d63bf7088b267dc2674cfc4de224791cc Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 25 Oct 2024 23:26:08 +0300 Subject: [PATCH 01/44] first recommendation draft --- app/api/__init__.py | 2 + app/api/home/__init__.py | 13 ++++ app/api/plugins/mixes.py | 63 +++++++++++++++ app/crons/__init__.py | 12 +++ app/crons/cron.py | 23 ++++++ app/crons/mixes.py | 15 ++++ app/models/mix.py | 37 +++++++++ app/models/track.py | 2 + app/plugins/mixes.py | 164 +++++++++++++++++++++++++++++++++++++++ app/store/homepage.py | 30 +++++++ app/utils/dates.py | 13 +++- app/utils/stats.py | 6 +- manage.py | 2 + poetry.lock | 16 +++- pyproject.toml | 1 + 15 files changed, 393 insertions(+), 6 deletions(-) create mode 100644 app/api/plugins/mixes.py create mode 100644 app/crons/__init__.py create mode 100644 app/crons/cron.py create mode 100644 app/crons/mixes.py create mode 100644 app/models/mix.py create mode 100644 app/plugins/mixes.py create mode 100644 app/store/homepage.py diff --git a/app/api/__init__.py b/app/api/__init__.py index f0d474f0..671e9b15 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -14,6 +14,7 @@ from app.config import UserConfig from app.db.userdata import UserTable from app.settings import Info as AppInfo from .plugins import lyrics as lyrics_plugin +from .plugins import mixes as mixes_plugin from app.api import ( album, artist, @@ -113,6 +114,7 @@ def create_api(): # Plugins app.register_api(plugins.api) app.register_api(lyrics_plugin.api) + app.register_api(mixes_plugin.api) # Logger app.register_api(scrobble.api) diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index c06a37bb..9c50a5ec 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -5,6 +5,7 @@ from flask_openapi3 import APIBlueprint from app.api.apischemas import GenericLimitSchema from app.lib.home.recentlyadded import get_recently_added_items from app.lib.home.recentlyplayed import get_recently_played +from app.store.homepage import HomepageStore bp_tag = Tag(name="Home", description="Homepage items") api = APIBlueprint("home", __name__, url_prefix="/home", abp_tags=[bp_tag]) @@ -24,3 +25,15 @@ def get_recent_plays(query: GenericLimitSchema): Get recently played """ return {"items": get_recently_played(query.limit)} + + +@api.get("/") +def homepage_items(): + return { + "artist_mixes": { + "title": "Artist mixes for you", + "description": "Curated based on artists you have played in the past", + "items": HomepageStore.get_artist_mixes(), + "extra": {}, + }, + } diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py new file mode 100644 index 00000000..3f292ca9 --- /dev/null +++ b/app/api/plugins/mixes.py @@ -0,0 +1,63 @@ +from flask_openapi3 import Tag +from flask_openapi3 import APIBlueprint +from pydantic import BaseModel, Field + +from app.plugins.mixes import MixesPlugin +from app.store.homepage import HomepageStore +from app.store.tracks import TrackStore + + +bp_tag = Tag(name="Mixes Plugin", description="Mixes plugin hehe") +api = APIBlueprint( + "mixesplugin", __name__, url_prefix="/plugins/mixes", abp_tags=[bp_tag] +) + + +@api.post("/track") +def get_track_mix(): + """ + Get a track mix + """ + mixes = MixesPlugin() + track = TrackStore.trackhashmap["9eeee292264ad01b"].get_best() + tracks = mixes.get_track_mix(track) + + return { + "total": len(tracks), + "tracks": tracks, + } + + +@api.post("/artist") +def get_artist_mix(): + mixes = MixesPlugin() + return mixes.get_artists() + # tracks = mixes.get_artist_mix("09306be8039b98ad") + + # return { + # "total": len(tracks), + # "tracks": tracks, + # } + + return "hi" + + +class MixQuery(BaseModel): + mixid: str = Field(description="The mix id") + + +@api.get("/") +def get_mix(query: MixQuery): + mixtype = "" + + match query.mixid[0]: + case "a": + mixtype = "artist_mixes" + case _: + raise ValueError(f"Invalid mix ID: {query.mixid}") + + mix = HomepageStore.get_mix(mixtype, query.mixid[1:]) + if mix: + return mix, 200 + + return {"msg": "Mix not found"}, 404 diff --git a/app/crons/__init__.py b/app/crons/__init__.py new file mode 100644 index 00000000..7357fbff --- /dev/null +++ b/app/crons/__init__.py @@ -0,0 +1,12 @@ +import time +import schedule + +from app.crons.mixes import Mixes +from app.utils.threading import background + + +@background +def start_cron_jobs(): + Mixes().run() + + # schedule.run_pending() diff --git a/app/crons/cron.py b/app/crons/cron.py new file mode 100644 index 00000000..939c94a7 --- /dev/null +++ b/app/crons/cron.py @@ -0,0 +1,23 @@ +import schedule + + +from abc import ABC, abstractmethod + + +class CronJob(ABC): + """ + A cron job that will be run on a regular interval. + """ + + def __init__(self, name: str, hours: int): + self.name = name + self.hours = hours + + schedule.every(self.hours).seconds.do(self.run) + + @abstractmethod + def run(self): + """ + The function that will be called by the cron job. + """ + ... diff --git a/app/crons/mixes.py b/app/crons/mixes.py new file mode 100644 index 00000000..44e62945 --- /dev/null +++ b/app/crons/mixes.py @@ -0,0 +1,15 @@ +from app.crons.cron import CronJob +from app.plugins.mixes import MixesPlugin +from app.store.homepage import HomepageStore + + +class Mixes(CronJob): + def __init__(self): + super().__init__("mixes", 5) + + def run(self): + print("⭐⭐⭐⭐ Mixes cron job running") + mixes = MixesPlugin() + artist_mixes = mixes.get_artists() + + HomepageStore.set_artist_mixes(artist_mixes) diff --git a/app/models/mix.py b/app/models/mix.py new file mode 100644 index 00000000..b5e807ef --- /dev/null +++ b/app/models/mix.py @@ -0,0 +1,37 @@ +from dataclasses import asdict, dataclass, field +import time + +from app.lib.playlistlib import get_first_4_images +from app.serializers.track import serialize_tracks +from app.store.tracks import TrackStore +from app.utils.dates import seconds_to_time_string + + +@dataclass +class Mix: + id: str + title: str + description: str + tracks: list[str] + + timestamp: int = field(default_factory=lambda: int(time.time())) + extra: dict = field(default_factory=dict) + saved: bool = False + + def to_full_dict(self): + tracks = TrackStore.get_tracks_by_trackhashes(self.tracks) + serialized_tracks = serialize_tracks(tracks) + + _dict = asdict(self) + _dict["tracks"] = serialized_tracks + _dict["images"] = get_first_4_images(tracks) + _dict["duration"] = seconds_to_time_string(sum(t.duration for t in tracks)) + _dict["trackcount"] = len(tracks) + + return _dict + + def to_dict(self): + item = self.to_full_dict() + del item["tracks"] + + return item diff --git a/app/models/track.py b/app/models/track.py index 8f764bf9..5598b9e4 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -45,6 +45,7 @@ class Track: og_title: str = "" artisthashes: list[str] = field(default_factory=list) genrehashes: list[str] = field(default_factory=list) + weakhash: str = "" _pos: int = 0 _ati: str = "" @@ -76,6 +77,7 @@ class Track: self.og_title = self.title self.og_album = self.album self.folder = self.folder + "/" + self.weakhash = create_hash(self.title, self.artists) self.image = self.albumhash + ".webp" self.extra = { diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py new file mode 100644 index 00000000..54036fa4 --- /dev/null +++ b/app/plugins/mixes.py @@ -0,0 +1,164 @@ +import json +import string +import requests +from urllib.parse import quote + +from app.models.artist import Artist +from app.models.mix import Mix +from app.models.track import Track +from app.plugins import Plugin, plugin_method +from app.store.artists import ArtistStore +from app.store.tracks import TrackStore +from app.utils.dates import get_date_range +from app.utils.remove_duplicates import remove_duplicates +from app.utils.stats import get_artists_in_period + + +class MixesPlugin(Plugin): + MAX_TRACKS_TO_FETCH = 5 + TRACK_MIX_LENGTH = 50 + MIN_TRACK_MIX_LENGTH = 15 + + MIN_DAY_LISTEN_DURATION = 3 * 60 # 3 minutes + MIN_WEEK_LISTEN_DURATION = 10 * 60 # 10 minutes + MIN_MONTH_LISTEN_DURATION = 20 * 60 # 20 minutes + + def __init__(self): + super().__init__("mixes", "Mixes") + self.server = "https://smcloud.mungaist.com" + self.set_active(True) + + @plugin_method + def get_track_mix(self, tracks: list[Track], with_help: bool = False): + # query = f"{track.title} - {','.join(a['name'] for a in track.artists)}" + queries = [ + { + "query": f"{track.title} - {','.join(a['name'] for a in track.artists)}", + "album": track.og_album, + "with_help": with_help, + } + for track in tracks + ] + + response = requests.post( + f"{self.server}/radio", + json=queries, + ) + results = response.json() + + # artisthashes = results["artists"] + trackhashes: list[str] = results["tracks"] + + trackmatches = TrackStore.get_flat_list() + trackmatches = [t for t in trackmatches if t.weakhash in trackhashes] + + # filter out duplicates of the same weakhash + # group by weakhash and pick the one with the highest bitrate + grouped: dict[str, list[Track]] = {} + for track in trackmatches: + grouped.setdefault(track.weakhash, []).append(track) + + trackmatches = [max(group, key=lambda x: x.bitrate) for group in grouped.values()] + + # sort by trackhash order + trackmatches = sorted(trackmatches, key=lambda x: trackhashes.index(x.weakhash)) + return trackmatches + + @plugin_method + def get_artist_mix(self, artisthash: str): + artist = ArtistStore.artistmap[artisthash] + tracks = TrackStore.get_tracks_by_trackhashes(artist.trackhashes) + + tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) + return self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) + + @plugin_method + def get_artists(self, limit: int = 10): + mixes: list[Mix] = [] + indexed = set() + + today_start, today_end = get_date_range(duration="day") + last_2_days_start, last_2_days_end = get_date_range(duration="day", units_ago=2) + last_7_days_start, last_7_days_end = get_date_range(duration="week") + last_1_month_start, last_1_month_end = get_date_range(duration="month") + + artists = { + "today": { + "max": 2, + "artists": get_artists_in_period(today_start, today_end), + "created": 0, + }, + "last_2_days": { + "max": 2, + "artists": get_artists_in_period(last_2_days_start, last_2_days_end), + "created": 0, + }, + "last_7_days": { + "max": 3, + "artists": get_artists_in_period(last_7_days_start, last_7_days_end), + "created": 0, + }, + "last_1_month": { + "max": 2, + "artists": get_artists_in_period(last_1_month_start, last_1_month_end), + "created": 0, + }, + } + + for i, period in enumerate(artists.values()): + # if previous period has less than its max + # add the difference to this period's limit + limit = period["max"] + + if i > 0: + previous_period = artists[list(artists.keys())[i - 1]] + if previous_period["created"] < previous_period["max"]: + limit += previous_period["max"] - previous_period["created"] + + for artist in period["artists"][:limit]: + mix = self.create_artist_mix(artist) + + if mix: + mixes.append(mix) + indexed.add(artist["artisthash"]) + period["created"] += 1 + + return mixes + + def get_mix_description(self, tracks: list[Track], artishash: str): + first_4_artists = [] + indexed = set() + + for track in tracks: + if len(first_4_artists) < 4: + if ( + track.artists[0]["artisthash"] != artishash + and track.artists[0]["artisthash"] not in indexed + ): + first_4_artists.append(track.artists[0]) + indexed.add(track.artists[0]["artisthash"]) + + if len(first_4_artists) == 4: + return f"Featuring {', '.join(a['name'] for a in first_4_artists)} and more" + + if len(first_4_artists) > 0: + return f"Featuring {', '.join(a['name'] for a in first_4_artists)}" + + return f"Featuring {tracks[0].artists[0]['name']}" + + def create_artist_mix(self, artist: dict[str, str]): + mix_tracks = self.get_artist_mix(artist["artisthash"]) + + if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: + return None + + return Mix( + id=artist["artisthash"], + title=artist["artist"], + description=self.get_mix_description(mix_tracks, artist["artisthash"]), + tracks=[t.trackhash for t in mix_tracks], + extra={ + "type": "artist", + "artisthash": artist["artisthash"], + }, + ) diff --git a/app/store/homepage.py b/app/store/homepage.py new file mode 100644 index 00000000..5691fb62 --- /dev/null +++ b/app/store/homepage.py @@ -0,0 +1,30 @@ +from app.models.mix import Mix +from app.store.tracks import TrackStore +from app.utils.auth import get_current_userid + + +class HomepageStore: + entries = { + "artist_mixes": {}, + } + + @classmethod + def set_artist_mixes(cls, mixes: list[Mix], userid: int = 1): + idmap = {mix.id: mix for mix in mixes} + cls.entries["artist_mixes"][userid] = idmap + + @classmethod + def get_artist_mixes(cls): + return [ + { + "type": "mix", + "item": mix.to_dict(), + } + for mix in cls.entries["artist_mixes"] + .get(get_current_userid(), {}) + .values() + ] + + @classmethod + def get_mix(cls, mixtype: str, mixid: str): + return cls.entries[mixtype].get(get_current_userid(), {}).get(mixid).to_full_dict() diff --git a/app/utils/dates.py b/app/utils/dates.py index 9d5228dd..484a0c14 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -66,16 +66,23 @@ def seconds_to_time_string(seconds): return f"{remaining_seconds} sec" -def get_date_range(duration: str): +def get_date_range(duration: str, units_ago: int = 0): """ Returns a tuple of dates representing the start and end of a given duration. """ date_range = None + seconds_ago = ( + pendulum.now() - pendulum.now().subtract().start_of(duration) + ).total_seconds() * units_ago + print("seconds_ago", duration, str(seconds_ago)) match duration: - case "week" | "month" | "year": + case "day" | "week" | "month" | "year": date_range = ( - pendulum.now().subtract().start_of(duration).timestamp(), + pendulum.now() + .subtract(seconds=seconds_ago) + .start_of(duration) + .timestamp(), pendulum.now().end_of(duration).timestamp(), ) case "alltime": diff --git a/app/utils/stats.py b/app/utils/stats.py index 8d353b3a..3c832506 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -13,7 +13,7 @@ from app.utils.dates import seconds_to_time_string def get_artists_in_period(start_time: int, end_time: int): scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time) - artists = defaultdict(lambda: {"playcount": 0, "playduration": 0}) + artists: Any = defaultdict(lambda: {"playcount": 0, "playduration": 0}) for scrobble in scrobbles: track = TrackStore.get_tracks_by_trackhashes([scrobble.trackhash]) @@ -24,11 +24,13 @@ def get_artists_in_period(start_time: int, end_time: int): for artist in track.artists: artisthash = artist["artisthash"] + artists[artisthash]["artist"] = artist["name"] artists[artisthash]["artisthash"] = artist["artisthash"] artists[artisthash]["playcount"] += 1 artists[artisthash]["playduration"] += scrobble.duration - return list(artists.values()) + artists = list(artists.values()) + return sorted(artists, key=lambda x: x["playduration"], reverse=True) def get_albums_in_period(start_time: int, end_time: int): diff --git a/manage.py b/manage.py index 20d70e69..c35fb1b6 100644 --- a/manage.py +++ b/manage.py @@ -21,6 +21,7 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs +from app.crons import start_cron_jobs from app.lib.index import IndexEverything from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info @@ -74,6 +75,7 @@ def run_swingmusic(): log_startup_info() bg_run_setup() register_plugins() + start_cron_jobs() # start_watchdog() diff --git a/poetry.lock b/poetry.lock index 0af17fe4..4d60be39 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2052,6 +2052,20 @@ files = [ {file = "roundrobin-0.0.4.tar.gz", hash = "sha256:7e9d19a5bd6123d99993fb935fa86d25c88bb2096e493885f61737ed0f5e9abd"}, ] +[[package]] +name = "schedule" +version = "1.2.2" +description = "Job scheduling for humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "schedule-1.2.2-py3-none-any.whl", hash = "sha256:5bef4a2a0183abf44046ae0d164cadcac21b1db011bdd8102e4a0c1e91e06a7d"}, + {file = "schedule-1.2.2.tar.gz", hash = "sha256:15fe9c75fe5fd9b9627f3f19cc0ef1420508f9f9a46f45cd0769ef75ede5f0b7"}, +] + +[package.extras] +timezone = ["pytz"] + [[package]] name = "setproctitle" version = "1.3.3" @@ -2761,4 +2775,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "43972b6ffadd14e5047f067a0258f2428ebe351df8bd032dc0bf05df379678a6" +content-hash = "85f8932739522e7b53b4fe5bbecc3c10a30bb690e25bf9404209c57ec71e88d3" diff --git a/pyproject.toml b/pyproject.toml index 87d5b809..d2f32de0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ memory-profiler = "^0.61.0" sortedcontainers = "^2.4.0" xxhash = "^3.4.1" ffmpeg-python = "^0.2.0" +schedule = "^1.2.2" [tool.poetry.dev-dependencies] pylint = "^2.15.5" From f2153d936dcd34da27d6d9c5de99952b62727a11 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 27 Oct 2024 06:35:37 +0100 Subject: [PATCH 02/44] implement mix tracklist balancing --- app/crons/__init__.py | 3 ++ app/crons/mixes.py | 6 +++ app/plugins/mixes.py | 39 ++++++++++++++++++ app/utils/mixes.py | 95 +++++++++++++++++++++++++++++++++++++++++++ manage.py | 12 +++--- 5 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 app/utils/mixes.py diff --git a/app/crons/__init__.py b/app/crons/__init__.py index 7357fbff..0618e31a 100644 --- a/app/crons/__init__.py +++ b/app/crons/__init__.py @@ -7,6 +7,9 @@ from app.utils.threading import background @background def start_cron_jobs(): + """ + This is the function that triggers the cron jobs. + """ Mixes().run() # schedule.run_pending() diff --git a/app/crons/mixes.py b/app/crons/mixes.py index 44e62945..0aea5869 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -4,10 +4,16 @@ from app.store.homepage import HomepageStore class Mixes(CronJob): + """ + This cron job creates mixes displayed on the homepage. + """ def __init__(self): super().__init__("mixes", 5) def run(self): + """ + Creates the artist mixes + """ print("⭐⭐⭐⭐ Mixes cron job running") mixes = MixesPlugin() artist_mixes = mixes.get_artists() diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 54036fa4..4024414b 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -3,6 +3,7 @@ import string import requests from urllib.parse import quote +from app.db.userdata import SimilarArtistTable from app.models.artist import Artist from app.models.mix import Mix from app.models.track import Track @@ -10,6 +11,7 @@ from app.plugins import Plugin, plugin_method from app.store.artists import ArtistStore from app.store.tracks import TrackStore from app.utils.dates import get_date_range +from app.utils.mixes import balance_mix from app.utils.remove_duplicates import remove_duplicates from app.utils.stats import get_artists_in_period @@ -62,6 +64,7 @@ class MixesPlugin(Plugin): # sort by trackhash order trackmatches = sorted(trackmatches, key=lambda x: trackhashes.index(x.weakhash)) + trackmatches = balance_mix(trackmatches) return trackmatches @plugin_method @@ -123,6 +126,7 @@ class MixesPlugin(Plugin): indexed.add(artist["artisthash"]) period["created"] += 1 + print(f"⭐⭐⭐⭐ Created {len(mixes)} mixes") return mixes def get_mix_description(self, tracks: list[Track], artishash: str): @@ -162,3 +166,38 @@ class MixesPlugin(Plugin): "artisthash": artist["artisthash"], }, ) + + + def fallback_create_artist_mix(self, artist: dict[str, str], similar_artists: list[str], trackhashes: set[str], limit: int): + """ + Creates an artist mix by selecting random tracks from similar artists. + + This is used when: + - The Swing Music recommendation server is down. + - The artist has less than self.MIN_TRACK_MIX_LENGTH tracks from the cloud mix. + - When we need to dilute the mix to balance the artist distribution. + + :param artist: The artist to create a mix for. + :param similar_artists: A list of similar artists to select tracks from. If not provided, we try reading from the local database. When we exhaust the passed list, we also try reading from the local database. + :param trackhashes: A set of trackhashes to omit from the new tracklist. + :param limit: The maximum number of tracks to select. + """ + artists = similar_artists + + if len(similar_artists) == 0: + local_similar_artists = SimilarArtistTable.get_by_hash(artist["artisthash"]) + + if local_similar_artists: + artists = [a.artisthash for a in local_similar_artists.similar_artists] + + if len(artists) == 0: + return [] + + # CHECKPOINT: I'M TIRED AF AND I NEED TO SLEEP + # The plan: + # Figure out which artists we should skip for the new tracklist + # these would be artists with a large number of tracks in the mix already + + # Since the artisthashes are ordered by similarity score, we iterate from the start + # and go forward collecting tracks that aren't in the mix yet. + # \ No newline at end of file diff --git a/app/utils/mixes.py b/app/utils/mixes.py new file mode 100644 index 00000000..9688aef4 --- /dev/null +++ b/app/utils/mixes.py @@ -0,0 +1,95 @@ +from app.models.track import Track +from typing import List, Dict, Tuple +from collections import Counter + + +def violates_gap_rule(balanced_mix: Dict[int, Track], position: int, track: Track, gap: int = 3) -> bool: + """ + Check if placing the track at the given position violates the gap rule. + """ + track_artists = set(artist['artisthash'] for artist in track.artists) + + for i in range(max(0, position - gap), position): + if i in balanced_mix: + existing_artists = set(artist['artisthash'] for artist in balanced_mix[i].artists) + if track_artists.intersection(existing_artists): + return True + return False + +def find_next_position(balanced_mix: Dict[int, Track], start: int, track: Track, total_tracks: int) -> int: + """ + Find the next available position for the track, starting from 'start' and wrapping around. + """ + for i in range(start, total_tracks): + if i not in balanced_mix and not violates_gap_rule(balanced_mix, i, track): + return i + for i in range(start): + if i not in balanced_mix and not violates_gap_rule(balanced_mix, i, track): + return i + return start # If no better position is found, return the original position + +def is_tracklist_balanced(tracks: List[Track], gap: int = 3) -> Tuple[bool, bool]: + """ + Checks if a tracklist is balanced or can be balanced. + + Args: + - tracks: List of Track objects + - gap: Minimum number of tracks between songs by the same artist (default 3) + + Returns: + - A tuple (can_be_balanced, is_currently_balanced) + """ + total_tracks = len(tracks) + + # Count tracks per artist (considering only the first artist) + artist_counts = Counter(track.artists[0]['artisthash'] for track in tracks) + + # Calculate the maximum number of tracks an artist can have in a balanced list + max_tracks_per_artist = (total_tracks + gap) // (gap + 1) + + # Check if it's mathematically possible to balance the tracklist + can_be_balanced = all(count <= max_tracks_per_artist for count in artist_counts.values()) + + if not can_be_balanced: + return False, False + + # Check if the current arrangement is balanced + is_currently_balanced = True + artist_last_positions = {} + + for i, track in enumerate(tracks): + artist = track.artists[0]['artisthash'] + if artist in artist_last_positions: + if i - artist_last_positions[artist] <= gap: + is_currently_balanced = False + break + artist_last_positions[artist] = i + + return can_be_balanced, is_currently_balanced + +def balance_mix(tracks: List[Track]) -> List[Track]: + """ + Balances the mix by ensuring that the tracks in a mix are distributed evenly. + Preserves the overall rating order of tracks while minimizing disruption. + """ + can_be_balanced, is_balanced = is_tracklist_balanced(tracks) + + if not can_be_balanced: + print("Warning: This tracklist cannot be perfectly balanced.") + # Proceed with best-effort balancing + + if is_balanced: + return tracks # Already balanced, no need to modify + + balanced_mix: Dict[int, Track] = {} + total_tracks = len(tracks) + + for i, track in enumerate(tracks): + if i in balanced_mix or not violates_gap_rule(balanced_mix, i, track): + balanced_mix[i] = track + else: + new_position = find_next_position(balanced_mix, i, track, total_tracks) + balanced_mix[new_position] = track + + # Convert the dictionary back to a list, preserving the new order + return [balanced_mix[i] for i in sorted(balanced_mix.keys())] diff --git a/manage.py b/manage.py index c35fb1b6..f222f326 100644 --- a/manage.py +++ b/manage.py @@ -60,7 +60,7 @@ mimetypes.add_type("application/manifest+json", ".webmanifest") # Background tasks -@background +# @background def bg_run_setup(): IndexEverything() @@ -73,21 +73,19 @@ def bg_run_setup(): @background def run_swingmusic(): log_startup_info() - bg_run_setup() register_plugins() - start_cron_jobs() # start_watchdog() setproctitle.setproctitle(f"swingmusic ::{FLASKVARS.get_flask_port()}") + # bg_run_setup() + start_cron_jobs() # Setup function calls Info.load() ProcessArgs() run_setup() -load_into_mem() -run_swingmusic() # Create the Flask app @@ -226,6 +224,10 @@ def print_memory_usage(response: Response): if __name__ == "__main__": + + load_into_mem() + run_swingmusic() + host = FLASKVARS.get_flask_host() port = FLASKVARS.get_flask_port() From f6373292aa0980709332db80e247fc40bb461456 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 28 Oct 2024 16:42:51 +0300 Subject: [PATCH 03/44] document and add image to mix --- app/api/plugins/mixes.py | 4 +-- app/crons/mixes.py | 2 +- app/models/mix.py | 1 + app/plugins/mixes.py | 57 +++++++++++++++++++++++++++++++++++----- app/utils/mixes.py | 56 +++++++++++++++++++++++++-------------- 5 files changed, 91 insertions(+), 29 deletions(-) diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index 3f292ca9..a4048a77 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -31,7 +31,7 @@ def get_track_mix(): @api.post("/artist") def get_artist_mix(): mixes = MixesPlugin() - return mixes.get_artists() + return mixes.create_artist_mixes() # tracks = mixes.get_artist_mix("09306be8039b98ad") # return { @@ -54,7 +54,7 @@ def get_mix(query: MixQuery): case "a": mixtype = "artist_mixes" case _: - raise ValueError(f"Invalid mix ID: {query.mixid}") + return {"msg": "Invalid mix ID"}, 400 mix = HomepageStore.get_mix(mixtype, query.mixid[1:]) if mix: diff --git a/app/crons/mixes.py b/app/crons/mixes.py index 0aea5869..aee30a6a 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -16,6 +16,6 @@ class Mixes(CronJob): """ print("⭐⭐⭐⭐ Mixes cron job running") mixes = MixesPlugin() - artist_mixes = mixes.get_artists() + artist_mixes = mixes.create_artist_mixes() HomepageStore.set_artist_mixes(artist_mixes) diff --git a/app/models/mix.py b/app/models/mix.py index b5e807ef..9af84e7a 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -13,6 +13,7 @@ class Mix: title: str description: str tracks: list[str] + image: dict timestamp: int = field(default_factory=lambda: int(time.time())) extra: dict = field(default_factory=dict) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 4024414b..8aaafad5 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -32,7 +32,19 @@ class MixesPlugin(Plugin): @plugin_method def get_track_mix(self, tracks: list[Track], with_help: bool = False): - # query = f"{track.title} - {','.join(a['name'] for a in track.artists)}" + """ + Given a list of tracks, creates a mix by fetching data from the + Swing Music Cloud recommendation server. + + The server returns a list of weak trackhashes. We use these to fetch + the matching track data from our library database. Found tracks are + then balanced and returned as the final mix tracklist. + + :param with_help: Whether to include the help flag in the query. + The flag tells the server to find more data using other tracks from the same album. + + + """ queries = [ { "query": f"{track.title} - {','.join(a['name'] for a in track.artists)}", @@ -60,15 +72,23 @@ class MixesPlugin(Plugin): for track in trackmatches: grouped.setdefault(track.weakhash, []).append(track) - trackmatches = [max(group, key=lambda x: x.bitrate) for group in grouped.values()] + trackmatches = [ + max(group, key=lambda x: x.bitrate) for group in grouped.values() + ] # sort by trackhash order trackmatches = sorted(trackmatches, key=lambda x: trackhashes.index(x.weakhash)) + + # try to balance the mix trackmatches = balance_mix(trackmatches) return trackmatches @plugin_method def get_artist_mix(self, artisthash: str): + """ + Given an artisthash, creates an artist mix using the + self.MAX_TRACKS_TO_FETCH most listened to tracks. + """ artist = ArtistStore.artistmap[artisthash] tracks = TrackStore.get_tracks_by_trackhashes(artist.trackhashes) @@ -76,7 +96,7 @@ class MixesPlugin(Plugin): return self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) @plugin_method - def get_artists(self, limit: int = 10): + def create_artist_mixes(self, limit: int = 10): mixes: list[Mix] = [] indexed = set() @@ -130,6 +150,10 @@ class MixesPlugin(Plugin): return mixes def get_mix_description(self, tracks: list[Track], artishash: str): + """ + Constructs a description for a mix by putting together the first n=4 + artists in the mix tracklist. + """ first_4_artists = [] indexed = set() @@ -151,13 +175,22 @@ class MixesPlugin(Plugin): return f"Featuring {tracks[0].artists[0]['name']}" def create_artist_mix(self, artist: dict[str, str]): + """ + Given an artist dict, creates an artist mix. + """ mix_tracks = self.get_artist_mix(artist["artisthash"]) if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None + _artist = ArtistStore.get_artist_by_hash(artist["artisthash"]) + + if not _artist: + return None + return Mix( - id=artist["artisthash"], + # the a prefix indicates that this is an artist mix + id=f"a{artist['artisthash']}", title=artist["artist"], description=self.get_mix_description(mix_tracks, artist["artisthash"]), tracks=[t.trackhash for t in mix_tracks], @@ -165,10 +198,19 @@ class MixesPlugin(Plugin): "type": "artist", "artisthash": artist["artisthash"], }, + image={ + "image": _artist.image, + "color": _artist.color, + }, ) - - def fallback_create_artist_mix(self, artist: dict[str, str], similar_artists: list[str], trackhashes: set[str], limit: int): + def fallback_create_artist_mix( + self, + artist: dict[str, str], + similar_artists: list[str], + trackhashes: set[str], + limit: int, + ): """ Creates an artist mix by selecting random tracks from similar artists. @@ -200,4 +242,5 @@ class MixesPlugin(Plugin): # Since the artisthashes are ordered by similarity score, we iterate from the start # and go forward collecting tracks that aren't in the mix yet. - # \ No newline at end of file + # + \ No newline at end of file diff --git a/app/utils/mixes.py b/app/utils/mixes.py index 9688aef4..352c038f 100644 --- a/app/utils/mixes.py +++ b/app/utils/mixes.py @@ -3,20 +3,31 @@ from typing import List, Dict, Tuple from collections import Counter -def violates_gap_rule(balanced_mix: Dict[int, Track], position: int, track: Track, gap: int = 3) -> bool: +def violates_gap_rule( + balanced_mix: Dict[int, Track], position: int, track: Track, gap: int = 3 +) -> bool: """ Check if placing the track at the given position violates the gap rule. + + The gap rule is violated if the track has an artist in common with any + track within the gap range (default = 3). """ - track_artists = set(artist['artisthash'] for artist in track.artists) - + track_artists = set(artist["artisthash"] for artist in track.artists) + for i in range(max(0, position - gap), position): if i in balanced_mix: - existing_artists = set(artist['artisthash'] for artist in balanced_mix[i].artists) + existing_artists = set( + artist["artisthash"] for artist in balanced_mix[i].artists + ) if track_artists.intersection(existing_artists): return True + return False -def find_next_position(balanced_mix: Dict[int, Track], start: int, track: Track, total_tracks: int) -> int: + +def find_next_position( + balanced_mix: Dict[int, Track], start: int, track: Track, total_tracks: int +) -> int: """ Find the next available position for the track, starting from 'start' and wrapping around. """ @@ -28,59 +39,66 @@ def find_next_position(balanced_mix: Dict[int, Track], start: int, track: Track, return i return start # If no better position is found, return the original position + def is_tracklist_balanced(tracks: List[Track], gap: int = 3) -> Tuple[bool, bool]: """ Checks if a tracklist is balanced or can be balanced. - + Args: - tracks: List of Track objects - gap: Minimum number of tracks between songs by the same artist (default 3) - + Returns: - A tuple (can_be_balanced, is_currently_balanced) """ total_tracks = len(tracks) - + # Count tracks per artist (considering only the first artist) - artist_counts = Counter(track.artists[0]['artisthash'] for track in tracks) - + artist_counts = Counter(track.artists[0]["artisthash"] for track in tracks) + # Calculate the maximum number of tracks an artist can have in a balanced list max_tracks_per_artist = (total_tracks + gap) // (gap + 1) - + # Check if it's mathematically possible to balance the tracklist - can_be_balanced = all(count <= max_tracks_per_artist for count in artist_counts.values()) + can_be_balanced = all( + count <= max_tracks_per_artist for count in artist_counts.values() + ) if not can_be_balanced: return False, False - + # Check if the current arrangement is balanced is_currently_balanced = True artist_last_positions = {} - + for i, track in enumerate(tracks): - artist = track.artists[0]['artisthash'] + artist = track.artists[0]["artisthash"] if artist in artist_last_positions: if i - artist_last_positions[artist] <= gap: is_currently_balanced = False break artist_last_positions[artist] = i - + return can_be_balanced, is_currently_balanced + def balance_mix(tracks: List[Track]) -> List[Track]: """ Balances the mix by ensuring that the tracks in a mix are distributed evenly. Preserves the overall rating order of tracks while minimizing disruption. + + Tracks that need to be moved are moved down the tracklist until they no longer + violate the gap rule. """ can_be_balanced, is_balanced = is_tracklist_balanced(tracks) - + if not can_be_balanced: print("Warning: This tracklist cannot be perfectly balanced.") # Proceed with best-effort balancing - + if is_balanced: return tracks # Already balanced, no need to modify - + balanced_mix: Dict[int, Track] = {} total_tracks = len(tracks) From 2ee501cc6467ca4d173c612e212ef9a8de3e5c6a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 29 Oct 2024 01:57:01 +0300 Subject: [PATCH 04/44] minor edits --- app/models/mix.py | 6 ++++-- app/plugins/mixes.py | 29 ++++++++++++++--------------- app/store/homepage.py | 2 ++ 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/app/models/mix.py b/app/models/mix.py index 9af84e7a..0eb4efd2 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -13,7 +13,6 @@ class Mix: title: str description: str tracks: list[str] - image: dict timestamp: int = field(default_factory=lambda: int(time.time())) extra: dict = field(default_factory=dict) @@ -25,7 +24,10 @@ class Mix: _dict = asdict(self) _dict["tracks"] = serialized_tracks - _dict["images"] = get_first_4_images(tracks) + + if not self.extra.get("image"): + _dict["images"] = get_first_4_images(tracks) + _dict["duration"] = seconds_to_time_string(sum(t.duration for t in tracks)) _dict["trackcount"] = len(tracks) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 8aaafad5..806e61bd 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -42,8 +42,8 @@ class MixesPlugin(Plugin): :param with_help: Whether to include the help flag in the query. The flag tells the server to find more data using other tracks from the same album. - - + + """ queries = [ { @@ -96,7 +96,7 @@ class MixesPlugin(Plugin): return self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) @plugin_method - def create_artist_mixes(self, limit: int = 10): + def create_artist_mixes(self): mixes: list[Mix] = [] indexed = set() @@ -107,7 +107,7 @@ class MixesPlugin(Plugin): artists = { "today": { - "max": 2, + "max": 3, "artists": get_artists_in_period(today_start, today_end), "created": 0, }, @@ -178,29 +178,29 @@ class MixesPlugin(Plugin): """ Given an artist dict, creates an artist mix. """ - mix_tracks = self.get_artist_mix(artist["artisthash"]) - - if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: - return None - _artist = ArtistStore.get_artist_by_hash(artist["artisthash"]) if not _artist: return None + mix_tracks = self.get_artist_mix(artist["artisthash"]) + + if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: + return None + return Mix( # the a prefix indicates that this is an artist mix id=f"a{artist['artisthash']}", - title=artist["artist"], + title=artist["artist"] + " Radio", description=self.get_mix_description(mix_tracks, artist["artisthash"]), tracks=[t.trackhash for t in mix_tracks], extra={ "type": "artist", "artisthash": artist["artisthash"], - }, - image={ - "image": _artist.image, - "color": _artist.color, + "image": { + "image": _artist.image, + "color": _artist.color, + }, }, ) @@ -243,4 +243,3 @@ class MixesPlugin(Plugin): # Since the artisthashes are ordered by similarity score, we iterate from the start # and go forward collecting tracks that aren't in the mix yet. # - \ No newline at end of file diff --git a/app/store/homepage.py b/app/store/homepage.py index 5691fb62..380d745a 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -10,6 +10,8 @@ class HomepageStore: @classmethod def set_artist_mixes(cls, mixes: list[Mix], userid: int = 1): + print(f"⭐⭐⭐⭐ Setting {len(mixes)} artist mixes") + print("mix artists", [mix.title for mix in mixes]) idmap = {mix.id: mix for mix in mixes} cls.entries["artist_mixes"][userid] = idmap From f512b65312e1337f09021f9aa60e0498d9e4632b Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 29 Oct 2024 02:00:28 +0300 Subject: [PATCH 05/44] prevent artist mix duplicates --- app/plugins/mixes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 806e61bd..5adac0b7 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -138,7 +138,13 @@ class MixesPlugin(Plugin): if previous_period["created"] < previous_period["max"]: limit += previous_period["max"] - previous_period["created"] - for artist in period["artists"][:limit]: + for artist in period["artists"]: + if period["created"] >= limit: + break + + if artist["artisthash"] in indexed: + continue + mix = self.create_artist_mix(artist) if mix: From fe09a8a8ae911f910c27ed3cf89ac94d1c266c98 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 29 Oct 2024 02:10:13 +0300 Subject: [PATCH 06/44] fix: ids on homepage store --- app/store/homepage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/store/homepage.py b/app/store/homepage.py index 380d745a..49fbf032 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -10,9 +10,7 @@ class HomepageStore: @classmethod def set_artist_mixes(cls, mixes: list[Mix], userid: int = 1): - print(f"⭐⭐⭐⭐ Setting {len(mixes)} artist mixes") - print("mix artists", [mix.title for mix in mixes]) - idmap = {mix.id: mix for mix in mixes} + idmap = {mix.id[1:]: mix for mix in mixes} cls.entries["artist_mixes"][userid] = idmap @classmethod From f6f66c571c8ba7416b4d57f2df86c6095bbd68fb Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 29 Oct 2024 22:40:30 +0300 Subject: [PATCH 07/44] use cloud mix images --- app/api/home/__init__.py | 2 +- app/api/imgserver.py | 18 ++++++++++++++++ app/plugins/mixes.py | 46 ++++++++++++++++++++++++++++++++++++---- app/settings.py | 20 +++++++++++++++++ app/setup/files.py | 10 +++++++++ 5 files changed, 91 insertions(+), 5 deletions(-) diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index 9c50a5ec..4c5a865e 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -32,7 +32,7 @@ def homepage_items(): return { "artist_mixes": { "title": "Artist mixes for you", - "description": "Curated based on artists you have played in the past", + "description": "Based on artists you have been listening to", "items": HomepageStore.get_artist_mixes(), "extra": {}, }, diff --git a/app/api/imgserver.py b/app/api/imgserver.py index c5e2db49..a3aa8d48 100644 --- a/app/api/imgserver.py +++ b/app/api/imgserver.py @@ -140,3 +140,21 @@ def send_playlist_image(path: PlaylistImagePath): """ folder = Paths.get_playlist_img_path() return send_file_or_fallback(folder, path.imgpath, "playlist.svg") + +# MIXES +@api.get("/mix/medium/") +def send_md_mix_image(path: ImagePath): + """ + Get medium mix image + """ + folder = Paths.get_md_mixes_img_path() + return send_file_or_fallback(folder, path.imgpath, "playlist.svg") + + +@api.get("/mix/small/") +def send_sm_mix_image(path: ImagePath): + """ + Get small mix image + """ + folder = Paths.get_sm_mixes_img_path() + return send_file_or_fallback(folder, path.imgpath, "playlist.svg") diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 5adac0b7..f1a47d66 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -2,12 +2,15 @@ import json import string import requests from urllib.parse import quote +from PIL import Image from app.db.userdata import SimilarArtistTable +from app.lib.colorlib import get_image_colors from app.models.artist import Artist from app.models.mix import Mix from app.models.track import Track from app.plugins import Plugin, plugin_method +from app.settings import Paths from app.store.artists import ArtistStore from app.store.tracks import TrackStore from app.utils.dates import get_date_range @@ -194,6 +197,14 @@ class MixesPlugin(Plugin): if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None + # try downloading artist image + mix_image = {"image": _artist.image, "color": _artist.color} + downloaded_img_color = self.download_artist_image(_artist) + + if downloaded_img_color: + mix_image["image"] = f"{_artist.artisthash}.jpg" + mix_image["color"] = downloaded_img_color[0] + return Mix( # the a prefix indicates that this is an artist mix id=f"a{artist['artisthash']}", @@ -203,13 +214,40 @@ class MixesPlugin(Plugin): extra={ "type": "artist", "artisthash": artist["artisthash"], - "image": { - "image": _artist.image, - "color": _artist.color, - }, + "image": mix_image, }, ) + def download_artist_image(self, artist: Artist): + res = requests.get(f"{self.server}/image?artist={artist.name}") + + if res.status_code == 200: + # save to file + with open( + f"{Paths.get_md_mixes_img_path()}/{artist.artisthash}.jpg", "wb" + ) as f: + f.write(res.content) + + # resize to 256x256 + img = Image.open(f"{Paths.get_md_mixes_img_path()}/{artist.artisthash}.jpg") + aspect_ratio = img.width / img.height + + newwidth = 256 + + if aspect_ratio > 1: + newheight = int(256 / aspect_ratio) + else: + newheight = 256 + + img = img.resize((newwidth, newheight), Image.LANCZOS) + img.save(f"{Paths.get_sm_mixes_img_path()}/{artist.artisthash}.jpg") + + return get_image_colors( + f"{Paths.get_sm_mixes_img_path()}/{artist.artisthash}.jpg" + ) + + return None + def fallback_create_artist_mix( self, artist: dict[str, str], diff --git a/app/settings.py b/app/settings.py index 3577ca4f..df06e3d9 100644 --- a/app/settings.py +++ b/app/settings.py @@ -103,6 +103,26 @@ class Paths: def get_config_file_path(cls): return join(cls.get_app_dir(), "settings.json") + @classmethod + def get_mixes_img_path(cls): + return join(cls.get_img_path(), "mixes") + + @classmethod + def get_artist_mixes_img_path(cls): + return join(cls.get_mixes_img_path(), "artists") + + @classmethod + def get_og_mixes_img_path(cls): + return join(cls.get_mixes_img_path(), "original") + + @classmethod + def get_md_mixes_img_path(cls): + return join(cls.get_mixes_img_path(), "medium") + + @classmethod + def get_sm_mixes_img_path(cls): + return join(cls.get_mixes_img_path(), "small") + # defaults class Defaults: diff --git a/app/setup/files.py b/app/setup/files.py index 4e875750..72de4383 100644 --- a/app/setup/files.py +++ b/app/setup/files.py @@ -62,6 +62,12 @@ def create_config_dir() -> None: playlist_img_path = os.path.join("images", "playlists") + + mixes_img_path = settings.Paths.get_mixes_img_path() + og_mixes_img_path = settings.Paths.get_og_mixes_img_path() + md_mixes_img_path = settings.Paths.get_md_mixes_img_path() + sm_mixes_img_path = settings.Paths.get_sm_mixes_img_path() + dirs = [ "", # creates the config folder sm_thumb_path, @@ -73,6 +79,10 @@ def create_config_dir() -> None: md_artist_img_path, small_artist_img_path, large_artist_img_path, + mixes_img_path, + og_mixes_img_path, + md_mixes_img_path, + sm_mixes_img_path, ] for _dir in dirs: From eb4c65de83e10df75804c6b9f89f2fe3fe2e39e3 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 1 Nov 2024 09:56:33 +0300 Subject: [PATCH 08/44] fix: image aspect ratio --- app/plugins/mixes.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index f1a47d66..378fb0c2 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -228,16 +228,12 @@ class MixesPlugin(Plugin): ) as f: f.write(res.content) - # resize to 256x256 + # resize to 256px width while maintaining aspect ratio img = Image.open(f"{Paths.get_md_mixes_img_path()}/{artist.artisthash}.jpg") aspect_ratio = img.width / img.height newwidth = 256 - - if aspect_ratio > 1: - newheight = int(256 / aspect_ratio) - else: - newheight = 256 + newheight = int(256 / aspect_ratio) img = img.resize((newwidth, newheight), Image.LANCZOS) img.save(f"{Paths.get_sm_mixes_img_path()}/{artist.artisthash}.jpg") From 1fdd5ba4d1d4097f3ee0650b2fffa8b3a25da31c Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 1 Nov 2024 12:23:41 +0300 Subject: [PATCH 09/44] supplement mixes using other remote similar albums and artist data --- app/api/plugins/mixes.py | 2 +- app/crons/mixes.py | 8 ++- app/models/album.py | 4 ++ app/plugins/mixes.py | 134 ++++++++++++++++++++++++++++++++------- app/serializers/album.py | 1 + app/utils/dates.py | 49 ++++++++++++-- app/utils/stats.py | 2 +- manage.py | 2 +- 8 files changed, 171 insertions(+), 31 deletions(-) diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index a4048a77..e513ac6b 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -20,7 +20,7 @@ def get_track_mix(): """ mixes = MixesPlugin() track = TrackStore.trackhashmap["9eeee292264ad01b"].get_best() - tracks = mixes.get_track_mix(track) + tracks = mixes.get_track_mix([track]) return { "total": len(tracks), diff --git a/app/crons/mixes.py b/app/crons/mixes.py index aee30a6a..9f9f212a 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -7,6 +7,7 @@ class Mixes(CronJob): """ This cron job creates mixes displayed on the homepage. """ + def __init__(self): super().__init__("mixes", 5) @@ -16,6 +17,11 @@ class Mixes(CronJob): """ print("⭐⭐⭐⭐ Mixes cron job running") mixes = MixesPlugin() + + if not mixes.enabled: + return + artist_mixes = mixes.create_artist_mixes() - HomepageStore.set_artist_mixes(artist_mixes) + if artist_mixes: + HomepageStore.set_artist_mixes(artist_mixes) diff --git a/app/models/album.py b/app/models/album.py index 0dc4322f..afe4f80b 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -36,6 +36,7 @@ class Album: image: str = "" versions: list[str] = dataclasses.field(default_factory=list) fav_userids: list[int] = dataclasses.field(default_factory=list) + weakhash: str = "" @property def is_favorite(self): @@ -54,6 +55,9 @@ class Album: def __post_init__(self): self.image = self.albumhash + ".webp" self.populate_versions() + self.weakhash = create_hash( + self.og_title, ",".join(a["name"] for a in self.albumartists) + ) def populate_versions(self): _, self.versions = get_base_title_and_versions(self.og_title, get_versions=True) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 378fb0c2..03e37121 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -1,5 +1,8 @@ +import datetime import json +import random import string +import time import requests from urllib.parse import quote from PIL import Image @@ -11,9 +14,10 @@ from app.models.mix import Mix from app.models.track import Track from app.plugins import Plugin, plugin_method from app.settings import Paths +from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore -from app.utils.dates import get_date_range +from app.utils.dates import get_date_range, get_duration_ago from app.utils.mixes import balance_mix from app.utils.remove_duplicates import remove_duplicates from app.utils.stats import get_artists_in_period @@ -31,7 +35,18 @@ class MixesPlugin(Plugin): def __init__(self): super().__init__("mixes", "Mixes") self.server = "https://smcloud.mungaist.com" - self.set_active(True) + + server_online = self.ping_server() + self.set_active(server_online) + + def ping_server(self): + try: + requests.get(self.server, timeout=10) + except requests.exceptions.ConnectionError: + print("Failed to connect to the recommendation server") + return False + + return True @plugin_method def get_track_mix(self, tracks: list[Track], with_help: bool = False): @@ -57,13 +72,20 @@ class MixesPlugin(Plugin): for track in tracks ] - response = requests.post( - f"{self.server}/radio", - json=queries, - ) - results = response.json() + try: + response = requests.post(f"{self.server}/radio", json=queries, timeout=10) + except requests.exceptions.ConnectionError: + print("Failed to connect to recommendation server") + return [] + + try: + results = response.json() + except json.JSONDecodeError: + print("Failed to decode JSON response from recommendation server") + return [] # artisthashes = results["artists"] + # albumhashes = results["albums"] trackhashes: list[str] = results["tracks"] trackmatches = TrackStore.get_flat_list() @@ -82,6 +104,16 @@ class MixesPlugin(Plugin): # sort by trackhash order trackmatches = sorted(trackmatches, key=lambda x: trackhashes.index(x.weakhash)) + if len(trackmatches) < self.TRACK_MIX_LENGTH: + filler_tracks = self.fallback_create_artist_mix( + artist=tracks[0].artists[0], + similar_artists=results["artists"], + similar_albums=results["albums"], + omit_trackhashes={t.weakhash for t in trackmatches}, + limit=self.TRACK_MIX_LENGTH - len(trackmatches), + ) + trackmatches.extend(filler_tracks) + # try to balance the mix trackmatches = balance_mix(trackmatches) return trackmatches @@ -104,9 +136,9 @@ class MixesPlugin(Plugin): indexed = set() today_start, today_end = get_date_range(duration="day") - last_2_days_start, last_2_days_end = get_date_range(duration="day", units_ago=2) - last_7_days_start, last_7_days_end = get_date_range(duration="week") - last_1_month_start, last_1_month_end = get_date_range(duration="month") + last_2_days_start = get_duration_ago("day", 2) + last_7_days_start = get_duration_ago("week") + last_1_month_start = get_duration_ago("month") artists = { "today": { @@ -116,17 +148,17 @@ class MixesPlugin(Plugin): }, "last_2_days": { "max": 2, - "artists": get_artists_in_period(last_2_days_start, last_2_days_end), + "artists": get_artists_in_period(last_2_days_start, time.time()), "created": 0, }, "last_7_days": { "max": 3, - "artists": get_artists_in_period(last_7_days_start, last_7_days_end), + "artists": get_artists_in_period(last_7_days_start, time.time()), "created": 0, }, "last_1_month": { "max": 2, - "artists": get_artists_in_period(last_1_month_start, last_1_month_end), + "artists": get_artists_in_period(last_1_month_start, time.time()), "created": 0, }, } @@ -219,7 +251,10 @@ class MixesPlugin(Plugin): ) def download_artist_image(self, artist: Artist): - res = requests.get(f"{self.server}/image?artist={artist.name}") + try: + res = requests.get(f"{self.server}/image?artist={artist.name}") + except requests.exceptions.ConnectionError: + return None if res.status_code == 200: # save to file @@ -247,8 +282,9 @@ class MixesPlugin(Plugin): def fallback_create_artist_mix( self, artist: dict[str, str], + similar_albums: list[str], similar_artists: list[str], - trackhashes: set[str], + omit_trackhashes: set[str], limit: int, ): """ @@ -264,16 +300,70 @@ class MixesPlugin(Plugin): :param trackhashes: A set of trackhashes to omit from the new tracklist. :param limit: The maximum number of tracks to select. """ - artists = similar_artists - if len(similar_artists) == 0: - local_similar_artists = SimilarArtistTable.get_by_hash(artist["artisthash"]) + mixtracks = [] + albummatches = ( + a + for a in AlbumStore.albummap.values() + if a.album.weakhash in similar_albums + ) - if local_similar_artists: - artists = [a.artisthash for a in local_similar_artists.similar_artists] + for match in albummatches: + if len(mixtracks) >= limit: + print(f"Filled up to {limit} tracks with album tracks") + return mixtracks - if len(artists) == 0: - return [] + albumtracks = [ + t + for t in TrackStore.get_tracks_by_trackhashes(match.trackhashes) + if t.weakhash not in omit_trackhashes + ] + + if len(albumtracks) == 0: + continue + + sample = random.sample(albumtracks, k=1) + mixtracks.extend(sample) + print( + f"Supplement: album track {sample[0].title} from ALBUM: {match.album.og_title}" + ) + + artistmatches = ( + a + for a in ArtistStore.artistmap.values() + if a.artist.artisthash in similar_artists + ) + + for match in artistmatches: + if len(mixtracks) >= limit: + print(f"Filled up to {limit} tracks with artist tracks") + return mixtracks + + artisttracks = [ + t + for t in TrackStore.get_tracks_by_trackhashes(match.trackhashes) + if t.weakhash not in omit_trackhashes + ] + + if len(artisttracks) == 0: + continue + + sample = random.sample(artisttracks, k=1) + mixtracks.extend(sample) + print( + f"Supplement: track {sample[0].title} from ARTIST: {match.artist.name}" + ) + + return mixtracks + + # if len(similar_artists) == 0: + # local_similar_artists = SimilarArtistTable.get_by_hash(artist["artisthash"]) + + # if local_similar_artists: + # artists = [a.artisthash for a in local_similar_artists.similar_artists] + + # if len(artists) == 0: + # return [] # CHECKPOINT: I'M TIRED AF AND I NEED TO SLEEP # The plan: diff --git a/app/serializers/album.py b/app/serializers/album.py index 35405b4b..4e4c5183 100644 --- a/app/serializers/album.py +++ b/app/serializers/album.py @@ -38,6 +38,7 @@ def serialize_for_card(album: Album): "extra", "id", "lastplayed", + "weakhash", } return album_serializer(album, props_to_remove) diff --git a/app/utils/dates.py b/app/utils/dates.py index 484a0c14..c7b5306f 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -71,10 +71,13 @@ def get_date_range(duration: str, units_ago: int = 0): Returns a tuple of dates representing the start and end of a given duration. """ date_range = None - seconds_ago = ( - pendulum.now() - pendulum.now().subtract().start_of(duration) - ).total_seconds() * units_ago - print("seconds_ago", duration, str(seconds_ago)) + seconds_ago = 0 + + if duration != "alltime": + seconds_ago = ( + pendulum.now() - pendulum.now().subtract().start_of(duration) + ).total_seconds() * units_ago + print("seconds_ago", duration, str(seconds_ago)) match duration: case "day" | "week" | "month" | "year": @@ -83,7 +86,9 @@ def get_date_range(duration: str, units_ago: int = 0): .subtract(seconds=seconds_ago) .start_of(duration) .timestamp(), - pendulum.now().end_of(duration).timestamp(), + pendulum.now() + # .end_of(duration) + .timestamp(), ) case "alltime": date_range = (0, pendulum.now().timestamp()) @@ -93,6 +98,40 @@ def get_date_range(duration: str, units_ago: int = 0): return (int(date_range[0]), int(date_range[1])) +def get_duration_ago(duration: str, units_ago: int = 1) -> int: + """ + Returns the start of the last duration. + """ + seconds_in_day = 24 * 60 * 60 + now = pendulum.now() + + match duration: + case "day": + return int( + now.subtract(seconds=seconds_in_day * units_ago).timestamp() + ) + case "week": + return int( + now + .subtract(seconds=seconds_in_day * 7 * units_ago) + .timestamp() + ) + case "month": + return int( + now + .subtract(seconds=seconds_in_day * 30 * units_ago) + .timestamp() + ) + case "year": + return int( + now + .subtract(seconds=seconds_in_day * 365 * units_ago) + .timestamp() + ) + case _: + raise ValueError(f"Invalid duration: {duration}") + + def get_duration_in_seconds(duration: str) -> int: """ Returns the number of seconds in a given duration. diff --git a/app/utils/stats.py b/app/utils/stats.py index 3c832506..86df3eb4 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -11,7 +11,7 @@ from app.store.tracks import TrackStore from app.utils.dates import seconds_to_time_string -def get_artists_in_period(start_time: int, end_time: int): +def get_artists_in_period(start_time: int | float, end_time: int | float): scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time) artists: Any = defaultdict(lambda: {"playcount": 0, "playduration": 0}) diff --git a/manage.py b/manage.py index f222f326..afdb7a85 100644 --- a/manage.py +++ b/manage.py @@ -102,7 +102,7 @@ whitelisted_routes = { "/auth/refresh", "/docs", } -blacklist_extensions = {".webp"}.union(getClientFilesExtensions()) +blacklist_extensions = {".webp", ".jpg"}.union(getClientFilesExtensions()) def skipAuthAction(): From 38d08f07bb0165a3eb6ce4cfe702864ee7087b36 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 1 Nov 2024 12:40:37 +0300 Subject: [PATCH 10/44] properly setup crons --- app/crons/__init__.py | 9 ++++++--- app/crons/cron.py | 2 +- app/crons/mixes.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/crons/__init__.py b/app/crons/__init__.py index 0618e31a..1c72e96a 100644 --- a/app/crons/__init__.py +++ b/app/crons/__init__.py @@ -10,6 +10,9 @@ def start_cron_jobs(): """ This is the function that triggers the cron jobs. """ - Mixes().run() - - # schedule.run_pending() + Mixes() + schedule.run_all() + + while True: + schedule.run_pending() + time.sleep(1) diff --git a/app/crons/cron.py b/app/crons/cron.py index 939c94a7..35954768 100644 --- a/app/crons/cron.py +++ b/app/crons/cron.py @@ -13,7 +13,7 @@ class CronJob(ABC): self.name = name self.hours = hours - schedule.every(self.hours).seconds.do(self.run) + schedule.every(self.hours).hours.do(self.run) @abstractmethod def run(self): diff --git a/app/crons/mixes.py b/app/crons/mixes.py index 9f9f212a..e897775f 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -9,7 +9,7 @@ class Mixes(CronJob): """ def __init__(self): - super().__init__("mixes", 5) + super().__init__("mixes", 1) def run(self): """ From 498d0688b0ef2675579b7682b3a011e1d636a4c4 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 10 Nov 2024 19:38:51 +0300 Subject: [PATCH 11/44] migrate homepage items to homepage routine + add Mix db model --- app/api/home/__init__.py | 17 +- app/crons/mixes.py | 15 +- app/db/libdata.py | 352 ------------------------------------ app/db/userdata.py | 48 ++++- app/lib/playlistlib.py | 1 - app/lib/recipes/__init__.py | 66 +++++++ app/models/mix.py | 22 ++- app/plugins/mixes.py | 83 +++++---- app/store/albums.py | 6 - app/store/homepage.py | 104 +++++++++-- app/utils/stats.py | 20 +- 11 files changed, 292 insertions(+), 442 deletions(-) create mode 100644 app/lib/recipes/__init__.py diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index 4c5a865e..1179c555 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -1,6 +1,6 @@ -from flask_jwt_extended import current_user from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint +from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema from app.lib.home.recentlyadded import get_recently_added_items @@ -27,13 +27,14 @@ def get_recent_plays(query: GenericLimitSchema): return {"items": get_recently_played(query.limit)} +class HomepageItem(BaseModel): + limit: int = Field( + default=9, description="The max number of items per group to return" + ) + + @api.get("/") -def homepage_items(): +def homepage_items(query: HomepageItem): return { - "artist_mixes": { - "title": "Artist mixes for you", - "description": "Based on artists you have been listening to", - "items": HomepageStore.get_artist_mixes(), - "extra": {}, - }, + "artist_mixes": HomepageStore.get_mixes("artist_mixes", limit=query.limit), } diff --git a/app/crons/mixes.py b/app/crons/mixes.py index e897775f..884cfa87 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -1,4 +1,5 @@ from app.crons.cron import CronJob +from app.lib.recipes import ArtistMixes from app.plugins.mixes import MixesPlugin from app.store.homepage import HomepageStore @@ -16,12 +17,14 @@ class Mixes(CronJob): Creates the artist mixes """ print("⭐⭐⭐⭐ Mixes cron job running") - mixes = MixesPlugin() + ArtistMixes().run() + # mixes = MixesPlugin() - if not mixes.enabled: - return + # if not mixes.enabled: + # return - artist_mixes = mixes.create_artist_mixes() - if artist_mixes: - HomepageStore.set_artist_mixes(artist_mixes) + # artist_mixes = mixes.create_artist_mixes() + + # if artist_mixes: + # HomepageStore.set_artist_mixes(artist_mixes) diff --git a/app/db/libdata.py b/app/db/libdata.py index 6e68d6d7..3d40c73f 100644 --- a/app/db/libdata.py +++ b/app/db/libdata.py @@ -8,92 +8,6 @@ from sqlalchemy.orm import Mapped, mapped_column from typing import Any, Optional -# def create_all(): -# """ -# Create all the tables defined in this file. - -# NOTE: We need this function because the MasterBase does not collect -# the tables defined here (as they are grand-children of the MasterBase) -# """ -# Base.metadata.create_all(DbEngine.engine) - - -# class Base(MasterBase, DeclarativeBase): -# pass -# @classmethod -# def get_all_hashes(cls, create_date: int | None = None): -# with DbEngine.manager() as conn: -# if create_date: -# if cls.__tablename__ == "track": -# stmt = select(TrackTable.trackhash).where( -# cls.last_mod < create_date -# ) -# elif cls.__tablename__ == "album": -# stmt = select(AlbumTable.albumhash).where( -# cls.created_date < create_date -# ) -# elif cls.__tablename__ == "artist": -# stmt = select(ArtistTable.artisthash).where( -# cls.created_date < create_date -# ) -# else: -# if cls.__tablename__ == "track": -# stmt = select(TrackTable.trackhash) -# elif cls.__tablename__ == "album": -# stmt = select(AlbumTable.albumhash) -# elif cls.__tablename__ == "artist": -# stmt = select(ArtistTable.artisthash) - -# result = conn.execute(stmt) -# return {row[0] for row in result.fetchall()} - -# @classmethod -# def set_is_favorite(cls, hash: str, is_favorite: bool): -# """ -# Set the 'is_favorite' flag for a specific hash. - -# Args: -# hash (str): The hash value. -# is_favorite (bool): The value of the 'is_favorite' flag. -# """ -# with DbEngine.manager(commit=True) as conn: -# if cls.__tablename__ == "track": -# stmt = ( -# update(cls) -# .where(TrackTable.trackhash == hash) -# .values(is_favorite=is_favorite) -# ) -# elif cls.__tablename__ == "album": -# stmt = ( -# update(cls) -# .where(AlbumTable.albumhash == hash) -# .values(is_favorite=is_favorite) -# ) -# elif cls.__tablename__ == "artist": -# stmt = ( -# update(cls) -# .where(ArtistTable.artisthash == hash) -# .values(is_favorite=is_favorite) -# ) - -# conn.execute(stmt) - -# @classmethod -# def increment_scrobblecount( -# cls, table: Any, field: Any, hash: str, duration: int, timestamp: int -# ): -# cls.execute( -# update(table) -# .where(field == hash) -# .values( -# playcount=table.playcount + 1, -# playduration=table.playduration + duration, -# lastplayed=timestamp, -# ), -# commit=True, -# ) - - class TrackTable(Base): __tablename__ = "track" @@ -101,7 +15,6 @@ class TrackTable(Base): album: Mapped[str] = mapped_column(String()) albumartists: Mapped[str] = mapped_column(String()) albumhash: Mapped[str] = mapped_column(String(), index=True) - # artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) artists: Mapped[str] = mapped_column(String()) bitrate: Mapped[int] = mapped_column(Integer()) copyright: Mapped[Optional[str]] = mapped_column(String()) @@ -110,11 +23,8 @@ class TrackTable(Base): duration: Mapped[int] = mapped_column(Integer()) filepath: Mapped[str] = mapped_column(String(), index=True, unique=True) folder: Mapped[str] = mapped_column(String(), index=True) - # genrehashes: Mapped[list[str]] = mapped_column(JSON(), index=True) genres: Mapped[Optional[str]] = mapped_column(String()) last_mod: Mapped[float] = mapped_column(Integer()) - # og_album: Mapped[str] = mapped_column(String()) - # og_title: Mapped[str] = mapped_column(String()) title: Mapped[str] = mapped_column(String()) track: Mapped[int] = mapped_column(Integer()) trackhash: Mapped[str] = mapped_column(String(), index=True) @@ -141,45 +51,6 @@ class TrackTable(Base): ) return tracks_to_dataclasses(result.fetchall()) - # @classmethod - # def get_tracks_by_albumhash(cls, albumhash: str): - # with DbEngine.manager() as conn: - # result = conn.execute( - # select(TrackTable).where(TrackTable.albumhash == albumhash) - # ) - # tracks = tracks_to_dataclasses(result.fetchall()) - # return remove_duplicates(tracks, is_album_tracks=True) - - # @classmethod - # def get_track_by_trackhash(cls, hash: str, filepath: str = ""): - # with DbEngine.manager() as conn: - # if filepath: - # result = conn.execute( - # select(TrackTable) - # .where( - # (TrackTable.trackhash == hash) - # & (TrackTable.filepath == filepath), - # ) - # .order_by(TrackTable.bitrate.desc()) - # ) - # else: - # result = conn.execute( - # select(TrackTable).where(TrackTable.trackhash == hash) - # ) - - # track = result.fetchone() - - # if track: - # return track_to_dataclass(track) - - # @classmethod - # def get_tracks_by_artisthash(cls, artisthash: str): - # with DbEngine.manager() as conn: - # result = conn.execute( - # select(TrackTable).where(TrackTable.artists.contains(artisthash)) - # ) - # return tracks_to_dataclasses(result.fetchall()) - @classmethod def get_tracks_in_path(cls, path: str): with DbEngine.manager() as conn: @@ -190,230 +61,7 @@ class TrackTable(Base): ) return tracks_to_dataclasses(result.fetchall()) - # @classmethod - # def get_tracks_by_trackhashes(cls, hashes: Iterable[str], limit: int | None = None): - # with DbEngine.manager() as conn: - # result = conn.execute( - # select(TrackTable) - # .where(TrackTable.trackhash.in_(hashes)) - # .group_by(TrackTable.trackhash) - # .limit(limit) - # ) - # tracks = tracks_to_dataclasses(result.fetchall()) - - # # order the tracks in the same order as the hashes - # if type(hashes) == list: - # return sorted(tracks, key=lambda x: hashes.index(x.trackhash)) - - # return tracks - - # @classmethod - # def get_recently_added(cls, start: int, limit: int): - # with DbEngine.manager() as conn: - # result = conn.execute( - # select(TrackTable) - # .order_by(TrackTable.last_mod.desc()) - # .offset(start) - # .limit(limit) - # ) - - # return tracks_to_dataclasses(result.fetchall()) - - @classmethod - # def get_recently_played(cls, limit: int): - # result = cls.execute( - # select(cls) - # .group_by(cls.trackhash) - # .order_by(cls.lastplayed.desc()) - # .limit(limit) - # ) - # return tracks_to_dataclasses(result.fetchall()) - @classmethod def remove_tracks_by_filepaths(cls, filepaths: set[str]): with DbEngine.manager(commit=True) as conn: conn.execute(delete(TrackTable).where(TrackTable.filepath.in_(filepaths))) - - # @classmethod - # def increment_playcount(cls, trackhash: str, duration: int, timestamp: int): - # cls.increment_scrobblecount( - # TrackTable, TrackTable.trackhash, trackhash, duration, timestamp - # ) - - # @classmethod - # def update_artist_separators(cls, separators: set[str]): - # tracks = cls.get_all() - - # with DbEngine.manager(commit=True) as conn: - # for track in tracks: - # track.split_artists(separators) - # conn.execute( - # update(cls) - # .where(cls.trackhash == track.trackhash) - # .values(artists=track.artists, artisthashes=track.artisthashes) - # ) - - -# class AlbumTable(Base): -# __tablename__ = "album" - -# id: Mapped[int] = mapped_column(primary_key=True) -# albumartists: Mapped[list[dict[str, str]]] = mapped_column(JSON(), index=True) -# artisthashes: Mapped[list[str]] = mapped_column(JSON(), index=True) -# albumhash: Mapped[str] = mapped_column(String(), unique=True, index=True) -# base_title: Mapped[str] = mapped_column(String()) -# color: Mapped[Optional[str]] = mapped_column(String()) -# created_date: Mapped[int] = mapped_column(Integer()) -# date: Mapped[int] = mapped_column(Integer()) -# duration: Mapped[int] = mapped_column(Integer()) -# genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) -# genres: Mapped[str] = mapped_column(JSON()) -# og_title: Mapped[str] = mapped_column(String()) -# title: Mapped[str] = mapped_column(String()) -# trackcount: Mapped[int] = mapped_column(Integer()) -# lastplayed: Mapped[int] = mapped_column(Integer(), default=0) -# playcount: Mapped[int] = mapped_column(Integer(), default=0) -# playduration: Mapped[int] = mapped_column(Integer(), default=0) -# extra: Mapped[Optional[dict[str, Any]]] = mapped_column( -# JSON(), default_factory=dict -# ) - -# @classmethod -# def get_all(cls): -# with DbEngine.manager() as conn: -# result = conn.execute(select(AlbumTable)) -# all = result.fetchall() -# return albums_to_dataclasses(all) - -# @classmethod -# def get_album_by_albumhash(cls, hash: str): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(AlbumTable).where(AlbumTable.albumhash == hash) -# ) -# album = result.fetchone() - -# if album: -# return album_to_dataclass(album) - -# @classmethod -# def get_albums_by_albumhashes(cls, hashes: Iterable[str], limit: int | None = None): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(AlbumTable).where(AlbumTable.albumhash.in_(hashes)).limit(limit) -# ) -# albums = albums_to_dataclasses(result.fetchall()) - -# # order the albums in the same order as the hashes -# if type(hashes) == list: -# return sorted(albums, key=lambda x: hashes.index(x.albumhash)) - -# return albums - -# @classmethod -# def get_albums_by_artisthashes(cls, artisthashes: list[str]): -# with DbEngine.manager() as conn: -# albums: dict[str, list[AlbumModel]] = {} - -# for artist in artisthashes: -# result = conn.execute( -# select(AlbumTable).where(AlbumTable.artisthashes.contains(artist)) -# ) -# albums[artist] = albums_to_dataclasses(result.fetchall()) - -# return albums - -# @classmethod -# def get_albums_by_base_title(cls, base_title: str): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(AlbumTable).where(AlbumTable.base_title == base_title) -# ) -# return albums_to_dataclasses(result.fetchall()) - -# @classmethod -# def get_albums_by_artisthash(cls, artisthash: str): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(AlbumTable).where(AlbumTable.artisthashes.contains(artisthash)) -# ) -# return albums_to_dataclasses(result.all()) - -# @classmethod -# def increment_playcount(cls, albumhash: str, duration: int, timestamp: int): -# return cls.increment_scrobblecount( -# AlbumTable, AlbumTable.albumhash, albumhash, duration, timestamp -# ) - - -# class ArtistTable(Base): -# __tablename__ = "artist" - -# id: Mapped[int] = mapped_column(primary_key=True) -# albumcount: Mapped[int] = mapped_column(Integer()) -# artisthash: Mapped[str] = mapped_column(String(), unique=True, index=True) -# created_date: Mapped[int] = mapped_column(Integer()) -# date: Mapped[int] = mapped_column(Integer()) -# duration: Mapped[int] = mapped_column(Integer()) -# genrehashes: Mapped[list[str]] = mapped_column(JSON(), nullable=True, index=True) -# genres: Mapped[str] = mapped_column(JSON()) -# name: Mapped[str] = mapped_column(String(), index=True) -# trackcount: Mapped[int] = mapped_column(Integer()) -# lastplayed: Mapped[int] = mapped_column(Integer(), default=0) -# playcount: Mapped[int] = mapped_column(Integer(), default=0) -# playduration: Mapped[int] = mapped_column(Integer(), default=0) -# extra: Mapped[Optional[dict[str, Any]]] = mapped_column( -# JSON(), default_factory=dict -# ) - -# @classmethod -# def get_all(cls): -# with DbEngine.manager() as conn: -# result = conn.execute(select(cls)) -# all = result.fetchall() -# return artists_to_dataclasses(all) - -# @classmethod -# def get_artist_by_hash(cls, artisthash: str): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(ArtistTable).where(ArtistTable.artisthash == artisthash) -# ) -# return artist_to_dataclass(result.fetchone()) - -# @classmethod -# def get_artisthashes_not_in(cls, artisthashes: list[str]): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(ArtistTable.artisthash, ArtistTable.name).where( -# ~ArtistTable.artisthash.in_(artisthashes) -# ) -# ) -# return [{"artisthash": row[0], "name": row[1]} for row in result.fetchall()] - -# @classmethod -# def get_artists_by_artisthashes( -# cls, hashes: Iterable[str], limit: int | None = None -# ): -# with DbEngine.manager() as conn: -# result = conn.execute( -# select(ArtistTable) -# .where(ArtistTable.artisthash.in_(hashes)) -# .limit(limit) -# ) -# return artists_to_dataclasses(result.fetchall()) - -# @classmethod -# def increment_playcount( -# cls, artisthashes: list[str], duration: int, timestamp: int -# ): -# cls.execute( -# update(cls) -# .where(ArtistTable.artisthash.in_(artisthashes)) -# .values( -# playcount=ArtistTable.playcount + 1, -# playduration=ArtistTable.playduration + duration, -# lastplayed=timestamp, -# ), -# commit=True, -# ) diff --git a/app/db/userdata.py b/app/db/userdata.py index a4a3f48c..2ec438d8 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -1,3 +1,4 @@ +from dataclasses import asdict import datetime from typing import Any, Literal from sqlalchemy import ( @@ -31,6 +32,7 @@ from app.db.utils import ( ) from app.db import Base +from app.models.mix import Mix from app.utils.auth import get_current_userid, hash_password @@ -223,9 +225,7 @@ class FavoritesTable(Base): # .select_from(join(table, cls, field == cls.hash)) .where(and_(cls.type == type, cls.userid == get_current_userid())) .order_by(cls.timestamp.desc()) - .offset( - start - ) + .offset(start) # INFO: If start is 0, fetch all so we can get the total count .limit(limit if start != 0 else None) ) @@ -305,10 +305,15 @@ class ScrobbleTable(Base): return tracklog_to_dataclasses(result.fetchall()) @classmethod - def get_all_in_period(cls, start_time: int, end_time: int): + def get_all_in_period(cls, start_time: int, end_time: int, userid: int | None): + # UserId will be None if function is called from the API + # In that case, we use the request userid + if userid is None: + userid = get_current_userid() + result = cls.execute( select(cls) - .where(cls.userid == get_current_userid()) + .where(cls.userid == userid) .where(and_(cls.timestamp >= start_time, cls.timestamp <= end_time)) .order_by(cls.timestamp.desc()) ) @@ -458,3 +463,36 @@ class LibDataTable(Base): select(cls.itemhash, cls.color).where(cls.itemtype == type) ) return [{"itemhash": r[0], "color": r[1]} for r in result.fetchall()] + + +class MixTable(Base): + __tablename__ = "mix" + + id: Mapped[int] = mapped_column(primary_key=True) + mixid: Mapped[str] = mapped_column(String(), index=True) + title: Mapped[str] = mapped_column(String()) + description: Mapped[str] = mapped_column(String()) + timestamp: Mapped[int] = mapped_column(Integer()) + sourcehash: Mapped[str] = mapped_column(String(), unique=True, index=True) + tracks: Mapped[list[str]] = mapped_column(JSON(), default_factory=list) + extra: Mapped[dict[str, Any]] = mapped_column( + JSON(), nullable=True, default_factory=dict + ) + + @classmethod + def get_all(cls): + result = cls.execute(select(cls)) + return Mix.mixes_to_dataclasses(result.fetchall()) + + @classmethod + def get_by_sourcehash(cls, sourcehash: str): + result = cls.execute(select(cls).where(cls.sourcehash == sourcehash)) + return Mix.mix_to_dataclass(result.fetchone()) + + @classmethod + def insert_one(cls, mix: Mix): + mixdict = asdict(mix) + mixdict["mixid"] = mix.id + del mixdict["id"] + + return cls.execute(insert(cls).values(mixdict), commit=True) diff --git a/app/lib/playlistlib.py b/app/lib/playlistlib.py index 4757b323..e0767adc 100644 --- a/app/lib/playlistlib.py +++ b/app/lib/playlistlib.py @@ -10,7 +10,6 @@ from typing import Any from PIL import Image, ImageSequence from app import settings -from app.db.libdata import TrackTable from app.models.track import Track from app.store.albums import AlbumStore from app.store.tracks import TrackStore diff --git a/app/lib/recipes/__init__.py b/app/lib/recipes/__init__.py new file mode 100644 index 00000000..9065eb2e --- /dev/null +++ b/app/lib/recipes/__init__.py @@ -0,0 +1,66 @@ +""" +Recipes are a way to create mixes. +""" + +from abc import ABC, abstractmethod +from typing import Any, Dict, List + +from app.db.userdata import UserTable +from app.models.mix import Mix +from app.plugins.mixes import MixesPlugin +from app.store.homepage import HomepageStore + + +class HomepageRoutine(ABC): + """ + A routine creates a row of homepage items. + """ + + title: str + description: str + + items: List[Mix] + extra: Dict[str, Any] + + @property + @abstractmethod + def is_valid(self) -> bool: ... + + def __init__(self) -> None: + if not self.is_valid: + return + + self.items = self.run() + + @abstractmethod + def run(self) -> List[Mix]: + """ + Creates the homepage items and saves them to the + homepage store if self.is_valid is true. + """ + ... + + +class ArtistMixes(HomepageRoutine): + items: List[Mix] = [] + extra: Dict[str, Any] = {} + store_key = "artist_mixes" + + @property + def is_valid(self): + return MixesPlugin().enabled + + def run(self): + users = UserTable.get_all() + + for user in users: + mix = MixesPlugin() + mixes = mix.create_artist_mixes(user.id) + + if not mixes: + continue + + HomepageStore.set_mixes(mixes, mixkey=self.store_key, userid=user.id) + + def __init__(self) -> None: + super().__init__() diff --git a/app/models/mix.py b/app/models/mix.py index 0eb4efd2..5255b5cd 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -1,5 +1,6 @@ -from dataclasses import asdict, dataclass, field import time +from dataclasses import asdict, dataclass, field +from typing import Any from app.lib.playlistlib import get_first_4_images from app.serializers.track import serialize_tracks @@ -13,12 +14,17 @@ class Mix: title: str description: str tracks: list[str] + sourcehash: str + """ + A hash of the tracks used to generate the mix. + """ timestamp: int = field(default_factory=lambda: int(time.time())) extra: dict = field(default_factory=dict) saved: bool = False def to_full_dict(self): + # Limit track mix to 30 tracks tracks = TrackStore.get_tracks_by_trackhashes(self.tracks) serialized_tracks = serialize_tracks(tracks) @@ -34,7 +40,19 @@ class Mix: return _dict def to_dict(self): - item = self.to_full_dict() + item = asdict(self) del item["tracks"] return item + + @classmethod + def mix_to_dataclass(cls, entry: Any): + entry_dict = entry._asdict() + entry_dict["id"] = entry_dict["mixid"] + del entry_dict["mixid"] + + return Mix(**entry_dict) + + @classmethod + def mixes_to_dataclasses(cls, entries: Any): + return [cls.mix_to_dataclass(entry) for entry in entries] diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 03e37121..d0b57e38 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -1,5 +1,6 @@ import datetime import json +from pprint import pprint import random import string import time @@ -18,6 +19,7 @@ from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore from app.utils.dates import get_date_range, get_duration_ago +from app.utils.hashing import create_hash from app.utils.mixes import balance_mix from app.utils.remove_duplicates import remove_duplicates from app.utils.stats import get_artists_in_period @@ -25,7 +27,7 @@ from app.utils.stats import get_artists_in_period class MixesPlugin(Plugin): MAX_TRACKS_TO_FETCH = 5 - TRACK_MIX_LENGTH = 50 + TRACK_MIX_LENGTH = 30 MIN_TRACK_MIX_LENGTH = 15 MIN_DAY_LISTEN_DURATION = 3 * 60 # 3 minutes @@ -60,8 +62,6 @@ class MixesPlugin(Plugin): :param with_help: Whether to include the help flag in the query. The flag tells the server to find more data using other tracks from the same album. - - """ queries = [ { @@ -84,8 +84,6 @@ class MixesPlugin(Plugin): print("Failed to decode JSON response from recommendation server") return [] - # artisthashes = results["artists"] - # albumhashes = results["albums"] trackhashes: list[str] = results["tracks"] trackmatches = TrackStore.get_flat_list() @@ -104,13 +102,18 @@ class MixesPlugin(Plugin): # sort by trackhash order trackmatches = sorted(trackmatches, key=lambda x: trackhashes.index(x.weakhash)) - if len(trackmatches) < self.TRACK_MIX_LENGTH: + # if the mix is short, try to fill it up with tracks + # from album and artist data from the cloud! + + # Create as many filler tracks as possible + # Then the mix length will be controlled in the Mix model + # if len(trackmatches) < self.TRACK_MIX_LENGTH: + if True: filler_tracks = self.fallback_create_artist_mix( - artist=tracks[0].artists[0], similar_artists=results["artists"], similar_albums=results["albums"], omit_trackhashes={t.weakhash for t in trackmatches}, - limit=self.TRACK_MIX_LENGTH - len(trackmatches), + # limit=self.TRACK_MIX_LENGTH - len(trackmatches), ) trackmatches.extend(filler_tracks) @@ -123,15 +126,26 @@ class MixesPlugin(Plugin): """ Given an artisthash, creates an artist mix using the self.MAX_TRACKS_TO_FETCH most listened to tracks. + + Returns a tuple of the mix and the sourcehash. """ artist = ArtistStore.artistmap[artisthash] tracks = TrackStore.get_tracks_by_trackhashes(artist.trackhashes) tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) - return self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) + sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] + sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) + + # TODO: Check if we already have this mix in the + # database and return that instead + + return (self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]), sourcehash) @plugin_method - def create_artist_mixes(self): + def create_artist_mixes(self, userid: int): + """ + Creates artist mixes for a given userid. + """ mixes: list[Mix] = [] indexed = set() @@ -143,22 +157,28 @@ class MixesPlugin(Plugin): artists = { "today": { "max": 3, - "artists": get_artists_in_period(today_start, today_end), + "artists": get_artists_in_period(today_start, today_end, userid), "created": 0, }, "last_2_days": { "max": 2, - "artists": get_artists_in_period(last_2_days_start, time.time()), + "artists": get_artists_in_period( + last_2_days_start, time.time(), userid + ), "created": 0, }, "last_7_days": { "max": 3, - "artists": get_artists_in_period(last_7_days_start, time.time()), + "artists": get_artists_in_period( + last_7_days_start, time.time(), userid + ), "created": 0, }, "last_1_month": { "max": 2, - "artists": get_artists_in_period(last_1_month_start, time.time()), + "artists": get_artists_in_period( + last_1_month_start, time.time(), userid + ), "created": 0, }, } @@ -224,7 +244,7 @@ class MixesPlugin(Plugin): if not _artist: return None - mix_tracks = self.get_artist_mix(artist["artisthash"]) + mix_tracks, sourcehash = self.get_artist_mix(artist["artisthash"]) if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None @@ -243,11 +263,13 @@ class MixesPlugin(Plugin): title=artist["artist"] + " Radio", description=self.get_mix_description(mix_tracks, artist["artisthash"]), tracks=[t.trackhash for t in mix_tracks], + sourcehash=sourcehash, extra={ "type": "artist", "artisthash": artist["artisthash"], "image": mix_image, }, + timestamp=int(time.time()), ) def download_artist_image(self, artist: Artist): @@ -281,23 +303,23 @@ class MixesPlugin(Plugin): def fallback_create_artist_mix( self, - artist: dict[str, str], + # artist: dict[str, str], similar_albums: list[str], similar_artists: list[str], omit_trackhashes: set[str], - limit: int, + limit: int = 99, ): """ - Creates an artist mix by selecting random tracks from similar artists. + Creates an artist mix by selecting random tracks from similar albums and artists. This is used when: - The Swing Music recommendation server is down. - The artist has less than self.MIN_TRACK_MIX_LENGTH tracks from the cloud mix. - When we need to dilute the mix to balance the artist distribution. - :param artist: The artist to create a mix for. - :param similar_artists: A list of similar artists to select tracks from. If not provided, we try reading from the local database. When we exhaust the passed list, we also try reading from the local database. - :param trackhashes: A set of trackhashes to omit from the new tracklist. + :param similar_albums: A list of similar album weakhashes to select tracks from. + :param similar_artists: A list of similar artist hashes to select tracks from. + :param omit_trackhashes: A set of trackhashes to omit from the new tracklist. :param limit: The maximum number of tracks to select. """ @@ -356,20 +378,5 @@ class MixesPlugin(Plugin): return mixtracks - # if len(similar_artists) == 0: - # local_similar_artists = SimilarArtistTable.get_by_hash(artist["artisthash"]) - - # if local_similar_artists: - # artists = [a.artisthash for a in local_similar_artists.similar_artists] - - # if len(artists) == 0: - # return [] - - # CHECKPOINT: I'M TIRED AF AND I NEED TO SLEEP - # The plan: - # Figure out which artists we should skip for the new tracklist - # these would be artists with a large number of tracks in the mix already - - # Since the artisthashes are ordered by similarity score, we iterate from the start - # and go forward collecting tracks that aren't in the mix yet. - # + def get_mix_from_lastfm_data(self, artisthash: str, limit: int): + pass diff --git a/app/store/albums.py b/app/store/albums.py index ee4572ac..ed1860f1 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -1,20 +1,14 @@ -from itertools import groupby -import json -from pprint import pprint import random from typing import Iterable from app.lib.tagger import create_albums from app.models import Album, Track from app.store.artists import ArtistStore -from app.utils import flatten from app.utils.auth import get_current_userid from app.utils.customlist import CustomList -from app.utils.remove_duplicates import remove_duplicates from ..utils.hashing import create_hash from .tracks import TrackStore -from app.utils.progressbar import tqdm ALBUM_LOAD_KEY = "" diff --git a/app/store/homepage.py b/app/store/homepage.py index 49fbf032..02aa9ed3 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -1,30 +1,102 @@ +from abc import ABC +from dataclasses import dataclass +from typing import Any from app.models.mix import Mix -from app.store.tracks import TrackStore from app.utils.auth import get_current_userid +@dataclass +class HomepageEntry(ABC): + """ + Base class for all homepage entries. + + items is a dict of userid to a dict of stuff. + """ + + title: str + description: str + items: dict[int, dict[str, Any]] + + def __init__(self, title: str, description: str): + self.title = title + self.description = description + + def get_items(self, userid: int): + """ + Return usable items for the homepage. + """ + ... + + +@dataclass +class MixHomepageEntry(HomepageEntry): + """ + A homepage entry for mixes. + self.items is a dict of userid to a dict of mixid to mix. + """ + + items: dict[int, dict[str, Mix]] + + def __init__(self, title: str, description: str): + super().__init__(title, description) + self.items = {} + + def get_items(self, userid: int, limit: int | None = None): + items = [] + + for mix in self.items.get(userid, {}).values(): + if limit and len(items) >= limit: + break + + items.append( + { + "type": "mix", + "item": mix.to_dict(), + } + ) + + return { + "title": self.title, + "description": self.description, + "items": items, + } + + class HomepageStore: + """ + Stores the homepage items. + """ + entries = { - "artist_mixes": {}, + "artist_mixes": MixHomepageEntry( + title="Artist mixes for you", + description="Based on artists you have been listening to", + ), } @classmethod - def set_artist_mixes(cls, mixes: list[Mix], userid: int = 1): + def set_mixes(cls, mixes: list[Mix], mixkey: str, userid: int | None = None): idmap = {mix.id[1:]: mix for mix in mixes} - cls.entries["artist_mixes"][userid] = idmap + cls.entries[mixkey].items[userid or get_current_userid()] = idmap @classmethod - def get_artist_mixes(cls): - return [ - { - "type": "mix", - "item": mix.to_dict(), - } - for mix in cls.entries["artist_mixes"] - .get(get_current_userid(), {}) - .values() - ] + def get_mixes(cls, mixkey: str, limit: int | None = 9): + return cls.entries[mixkey].get_items(get_current_userid(), limit) @classmethod - def get_mix(cls, mixtype: str, mixid: str): - return cls.entries[mixtype].get(get_current_userid(), {}).get(mixid).to_full_dict() + def get_mix(cls, mixkey: str, mixid: str): + mix = cls.entries[mixkey].items.get(get_current_userid(), {}).get(mixid) + return mix.to_full_dict() if mix else None + + @classmethod + def get_mix_by_sourcehash(cls, sourcehash: str): + return next( + ( + mix + for mix in cls.entries["artist_mixes"] + .items.get(get_current_userid(), {}) + .values() + if mix.sourcehash == sourcehash + ), + None, + ) diff --git a/app/utils/stats.py b/app/utils/stats.py index 86df3eb4..0ed38eb1 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -11,8 +11,10 @@ from app.store.tracks import TrackStore from app.utils.dates import seconds_to_time_string -def get_artists_in_period(start_time: int | float, end_time: int | float): - scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time) +def get_artists_in_period( + start_time: int | float, end_time: int | float, userid: int | None = None +): + scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time, userid) artists: Any = defaultdict(lambda: {"playcount": 0, "playduration": 0}) for scrobble in scrobbles: @@ -33,8 +35,8 @@ def get_artists_in_period(start_time: int | float, end_time: int | float): return sorted(artists, key=lambda x: x["playduration"], reverse=True) -def get_albums_in_period(start_time: int, end_time: int): - scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time) +def get_albums_in_period(start_time: int, end_time: int, userid: int | None = None): + scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time, userid) albums: dict[str, Album] = {} for scrobble in scrobbles: @@ -60,8 +62,8 @@ def get_albums_in_period(start_time: int, end_time: int): return list(albums.values()) -def get_tracks_in_period(start_time: int, end_time: int): - scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time) +def get_tracks_in_period(start_time: int, end_time: int, userid: int | None = None): + scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time, userid) tracks: dict[str, Track] = {} duration = 0 @@ -160,12 +162,14 @@ def calculate_scrobble_trend(current_scrobbles: int, previous_scrobbles: int) -> ) -def calculate_new_artists(current_artists: List[dict[str, Any]], timestamp: int): +def calculate_new_artists( + current_artists: List[dict[str, Any]], timestamp: int, userid: int | None = None +): """ Calculate the number of new artists based on the current and all previous scrobbles. """ current_artists_set = set(artist["artisthash"] for artist in current_artists) - all_records = ScrobbleTable.get_all_in_period(0, timestamp) + all_records = ScrobbleTable.get_all_in_period(0, timestamp, userid) trackhashes = set(record.trackhash for record in all_records) previous_artists_set = set() From 333fd6603f2898383a181aa8e7441303f4540316 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 17 Nov 2024 20:08:04 +0300 Subject: [PATCH 12/44] move recently added to routines --- app/api/home/__init__.py | 4 +- app/crons/__init__.py | 15 +- app/crons/cron.py | 6 +- app/crons/mixes.py | 21 +-- app/db/userdata.py | 4 +- app/lib/home/recentlyadded.py | 91 +++++---- app/lib/home/recentlyplayed.py | 326 ++++++++++++++++++++++++--------- app/lib/recipes/__init__.py | 43 +---- app/lib/recipes/artistmixes.py | 27 +++ app/lib/recipes/recents.py | 46 +++++ app/lib/recipes/topstreamed.py | 84 +++++++++ app/models/logger.py | 6 + app/store/homepage.py | 92 +++++++--- app/utils/dates.py | 2 +- 14 files changed, 554 insertions(+), 213 deletions(-) create mode 100644 app/lib/recipes/artistmixes.py create mode 100644 app/lib/recipes/recents.py create mode 100644 app/lib/recipes/topstreamed.py diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index 1179c555..e7fa192a 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -35,6 +35,4 @@ class HomepageItem(BaseModel): @api.get("/") def homepage_items(query: HomepageItem): - return { - "artist_mixes": HomepageStore.get_mixes("artist_mixes", limit=query.limit), - } + return HomepageStore.get_homepage_items(limit=query.limit) \ No newline at end of file diff --git a/app/crons/__init__.py b/app/crons/__init__.py index 1c72e96a..41e09ed3 100644 --- a/app/crons/__init__.py +++ b/app/crons/__init__.py @@ -2,6 +2,8 @@ import time import schedule from app.crons.mixes import Mixes +from app.lib.recipes.recents import RecentlyAdded, RecentlyPlayed +from app.lib.recipes.topstreamed import TopArtists from app.utils.threading import background @@ -10,9 +12,20 @@ def start_cron_jobs(): """ This is the function that triggers the cron jobs. """ - Mixes() + # NOTE: RecentlyPlayed is not a CRON job, it's triggered here to + # populate the values for the very first time. + RecentlyPlayed() + RecentlyAdded() + + # Initialized CRON jobs + # Mixes() + TopArtists() + TopArtists(duration="week") + + # Trigger all CRON jobs when the app is started. schedule.run_all() + # Run all CRON jobs on a loop. while True: schedule.run_pending() time.sleep(1) diff --git a/app/crons/cron.py b/app/crons/cron.py index 35954768..4f9e02ad 100644 --- a/app/crons/cron.py +++ b/app/crons/cron.py @@ -9,10 +9,10 @@ class CronJob(ABC): A cron job that will be run on a regular interval. """ - def __init__(self, name: str, hours: int): - self.name = name - self.hours = hours + name: str + hours: int = 1 + def __init__(self): schedule.every(self.hours).hours.do(self.run) @abstractmethod diff --git a/app/crons/mixes.py b/app/crons/mixes.py index 884cfa87..db8c2773 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -1,7 +1,5 @@ from app.crons.cron import CronJob -from app.lib.recipes import ArtistMixes -from app.plugins.mixes import MixesPlugin -from app.store.homepage import HomepageStore +from app.lib.recipes.artistmixes import ArtistMixes class Mixes(CronJob): @@ -9,22 +7,15 @@ class Mixes(CronJob): This cron job creates mixes displayed on the homepage. """ + name: str = "mixes" + hours: int = 1 + def __init__(self): - super().__init__("mixes", 1) + super().__init__() def run(self): """ Creates the artist mixes """ print("⭐⭐⭐⭐ Mixes cron job running") - ArtistMixes().run() - # mixes = MixesPlugin() - - # if not mixes.enabled: - # return - - - # artist_mixes = mixes.create_artist_mixes() - - # if artist_mixes: - # HomepageStore.set_artist_mixes(artist_mixes) + ArtistMixes() diff --git a/app/db/userdata.py b/app/db/userdata.py index 2ec438d8..8cc0fd54 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -293,10 +293,10 @@ class ScrobbleTable(Base): return cls.insert_one(item) @classmethod - def get_all(cls, start: int, limit: int | None = None): + def get_all(cls, start: int, limit: int | None = None, userid: int | None = None): result = cls.execute( select(cls) - .where(cls.userid == get_current_userid()) + .where(cls.userid == (userid if userid else get_current_userid())) .order_by(cls.timestamp.desc()) .offset(start) .limit(limit) diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index abb93ae6..58ef1184 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -59,13 +59,19 @@ def create_track(t: Track): """ Creates a recently added track entry. """ - track = serialize_track(t, to_remove={"created_date"}) - track["help_text"] = "NEW TRACK" - return { "type": "track", - "item": track, + "hash": t.trackhash, + "timestamp": t.last_mod, + "help_text": "NEW TRACK", } + # track = serialize_track(t, to_remove={"created_date"}) + # track["help_text"] = "NEW TRACK" + + # return { + # "type": "track", + # "item": track, + # } # INFO: Keys: folder, tracks, time (timestamp) @@ -94,26 +100,25 @@ def check_folder_type(group_: dict): if entry is None: return None - album = album_serializer( - entry.album, - to_remove={ - "genres", - "og_title", - "date", - "duration", - "count", - "albumartists_hashes", - "base_title", - }, - ) - album["help_text"] = ( - "NEW ALBUM" if albumhash in existing_album_hashes else "NEW TRACKS" - ) - album["time"] = timestamp_to_time_passed(time) - + # album = album_serializer( + # entry.album, + # to_remove={ + # "genres", + # "og_title", + # "date", + # "duration", + # "count", + # "albumartists_hashes", + # "base_title", + # }, + # ) return { "type": "album", - "item": album, + "hash": albumhash, + "timestamp": time, + "help_text": ( + "NEW ALBUM" if albumhash in existing_album_hashes else "NEW TRACKS" + ), } is_artist, artisthash, trackcount = check_is_artist_folder(tracks) @@ -122,19 +127,28 @@ def check_folder_type(group_: dict): if entry is None: return None - - artist = serialize_for_card(entry.artist) - artist["trackcount"] = trackcount - artist["help_text"] = ( - "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" - ) - artist["time"] = timestamp_to_time_passed(time) - + return { "type": "artist", - "item": artist, + "hash": artisthash, + "timestamp": time, + "help_text": ( + "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" + ), } + # artist = serialize_for_card(entry.artist) + # artist["trackcount"] = trackcount + # artist["help_text"] = ( + # "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" + # ) + # artist["time"] = timestamp_to_time_passed(time) + + # return { + # "type": "artist", + # "item": artist, + # } + is_track_folder = check_is_track_folder(tracks) return ( @@ -142,12 +156,15 @@ def check_folder_type(group_: dict): if is_track_folder else { "type": "folder", - "item": { - "path": key, - "count": len(tracks), - "help_text": "NEW MUSIC", - "time": timestamp_to_time_passed(time), - }, + "hash": key, + "timestamp": time, + "help_text": "NEW MUSIC", + # "item": { + # "path": key, + # "count": len(tracks), + # "help_text": "NEW MUSIC", + # "time": timestamp_to_time_passed(time), + # }, } ) diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index dcf00459..0476d26b 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -22,7 +22,7 @@ from app.store.tracks import TrackStore from app.store.artists import ArtistStore -def get_recently_played(limit=7): +def get_recently_played(limit=7, userid: int | None = None): # TODO: Paginate this items = [] added = set() @@ -43,47 +43,52 @@ def get_recently_played(limit=7): added.add(entry.source) if entry.type == "album": - album = AlbumStore.get_album_by_hash(entry.type_src) + album = AlbumStore.albummap.get(entry.type_src) if album is None: continue - album = album_serializer( - album, - to_remove={ - "genres", - "date", - "count", - "duration", - "albumartists_hashes", - "og_title", - }, - ) - album["help_text"] = "album" - album["time"] = timestamp_to_time_passed(entry.timestamp) + # album = album_serializer( + # album, + # to_remove={ + # "genres", + # "date", + # "count", + # "duration", + # "albumartists_hashes", + # "og_title", + # }, + # ) + item = { + "type": "album", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + # album["help_text"] = "album" + # album["time"] = timestamp_to_time_passed(entry.timestamp) - items.append( - { - "type": "album", - "item": album, - } - ) + # { + # "type": "album", + # "item": album, + # } + items.append(item) continue if entry.type == "artist": - artist = ArtistStore.get_artist_by_hash(entry.type_src) + artist = ArtistStore.artistmap.get(entry.type_src) if artist is None: continue - artist = serialize_for_card(artist) - artist["help_text"] = "artist" - artist["time"] = timestamp_to_time_passed(entry.timestamp) + # artist = serialize_for_card(artist) + # artist["help_text"] = "artist" + # artist["time"] = timestamp_to_time_passed(entry.timestamp) items.append( { "type": "artist", - "item": artist, + "hash": entry.type_src, + "timestamp": entry.timestamp, } ) @@ -103,47 +108,56 @@ def get_recently_played(limit=7): if is_home_dir: folder = os.path.expanduser("~") - # print(folder) - # folder = os.path.join("/", folder, "") - # print(folder) - # count = len([t for t in TrackStore.tracks if t.folder == folder]) - count = FolderStore.count_tracks_containing_paths([folder]) - items.append( - { - "type": "folder", - "item": { - "path": folder, - "count": count[0]["trackcount"], - "help_text": "folder", - "time": timestamp_to_time_passed(entry.timestamp), - }, - } - ) + # count = FolderStore.count_tracks_containing_paths([folder]) + item = { + "type": "folder", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + # { + # "type": "folder", + # "item": { + # "path": folder, + # "count": count[0]["trackcount"], + # "help_text": "folder", + # "time": timestamp_to_time_passed(entry.timestamp), + # }, + # } + continue if entry.type == "playlist": is_custom = entry.type_src in [i["name"] for i in custom_playlists] - # is_recently_added = entry.type_src == "recentlyadded" if is_custom: - playlist, _ = next( - i["handler"]() - for i in custom_playlists - if i["name"] == entry.type_src - ) - playlist.images = [i["image"] for i in playlist.images] + # playlist, _ = next( + # i["handler"]() + # for i in custom_playlists + # if i["name"] == entry.type_src + # ) + # playlist.images = [i["image"] for i in playlist.images] - playlist = serialize_playlist( - playlist, to_remove={"settings", "duration"} - ) + # playlist = serialize_playlist( + # playlist, to_remove={"settings", "duration"} + # ) - playlist["help_text"] = "playlist" - playlist["time"] = timestamp_to_time_passed(entry.timestamp) + # playlist["help_text"] = "playlist" + # playlist["time"] = timestamp_to_time_passed(entry.timestamp) + # items.append( + # { + # "type": "playlist", + # "item": playlist, + # } + # ) items.append( { "type": "playlist", - "item": playlist, + "hash": entry.type_src, + "timestamp": entry.timestamp, + "is_custom": True, } ) continue @@ -152,34 +166,47 @@ def get_recently_played(limit=7): if playlist is None: continue - tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) - playlist.clear_lists() + item = { + "type": "playlist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } - if not playlist.has_image: - images = get_first_4_images(tracks) - images = [i["image"] for i in images] - playlist.images = images + items.append(item) - items.append( - { - "type": "playlist", - "item": { - "help_text": "playlist", - "time": timestamp_to_time_passed(entry.timestamp), - **serialize_playlist(playlist), - }, - } - ) + # tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) + # playlist.clear_lists() + + # if not playlist.has_image: + # images = get_first_4_images(tracks) + # images = [i["image"] for i in images] + # playlist.images = images + + # items.append( + # { + # "type": "playlist", + # "item": { + # "help_text": "playlist", + # "time": timestamp_to_time_passed(entry.timestamp), + # **serialize_playlist(playlist), + # }, + # } + # ) + continue if entry.type == "favorite": items.append( + # { + # "type": "favorite_tracks", + # "item": { + # "help_text": "playlist", + # "count": FavoritesTable.count(), + # "time": timestamp_to_time_passed(entry.timestamp), + # }, + # } { - "type": "favorite_tracks", - "item": { - "help_text": "playlist", - "count": FavoritesTable.count(), - "time": timestamp_to_time_passed(entry.timestamp), - }, + "type": "favorite", + "timestamp": entry.timestamp, } ) continue @@ -189,22 +216,23 @@ def get_recently_played(limit=7): if t is None: continue - track = serialize_track(t.get_best()) - track["help_text"] = "track" - track["time"] = timestamp_to_time_passed(entry.timestamp) + item = { + "type": "track", + "hash": entry.trackhash, + "timestamp": entry.timestamp, + } - items.append( - { - "type": "track", - "item": track, - } - ) + # track = serialize_track(t.get_best()) + # track["help_text"] = "track" + # track["time"] = timestamp_to_time_passed(entry.timestamp) + + items.append(item) BATCH_SIZE = 200 current_index = 0 entries = ScrobbleTable.get_all(0, BATCH_SIZE) - max_iterations = 20 # Safeguard against unexpected infinite loops + max_iterations = 20 # Safeguard against unexpected infinite loops iterations = 0 while len(items) < limit and iterations < max_iterations: @@ -212,7 +240,9 @@ def get_recently_played(limit=7): current_index += BATCH_SIZE if len(items) < limit: - entries = ScrobbleTable.get_all(current_index + 1, BATCH_SIZE) + entries = ScrobbleTable.get_all( + start=current_index + 1, limit=BATCH_SIZE, userid=userid + ) if not entries: break @@ -226,6 +256,126 @@ def get_recently_played(limit=7): return items +def recover_recently_played_items(items: list[dict]): + custom_playlists = [ + {"name": "recentlyadded", "handler": get_recently_added_playlist}, + {"name": "recentlyplayed", "handler": get_recently_played_playlist}, + ] + recovered = [] + + for item in items: + recovered_item = None + + if item["type"] == "album": + album = AlbumStore.get_album_by_hash(item["hash"]) + if album is None: + continue + + album = album_serializer( + album, + to_remove={ + "genres", + "date", + "count", + "duration", + "albumartists_hashes", + "og_title", + }, + ) + + recovered_item = { + "type": "album", + "item": album, + } + elif item["type"] == "artist": + artist = ArtistStore.get_artist_by_hash(item["hash"]) + if artist is None: + continue + + recovered_item = { + "type": "artist", + "item": serialize_for_card(artist), + } + elif item["type"] == "folder": + count = FolderStore.count_tracks_containing_paths([item["hash"]]) + + recovered_item = { + "type": "folder", + "item": { + "path": item["hash"], + "count": count[0]["trackcount"], + }, + } + elif item["type"] == "playlist": + if item["is_custom"]: + playlist, _ = next( + i["handler"]() + for i in custom_playlists + if i["name"] == item["hash"] + ) + playlist.images = [i["image"] for i in playlist.images] + + playlist = serialize_playlist( + playlist, to_remove={"settings", "duration"} + ) + recovered_item = { + "type": "playlist", + "item": playlist, + } + else: + playlist = PlaylistTable.get_by_id(item["hash"]) + if playlist is None: + continue + + tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) + playlist.clear_lists() + + if not playlist.has_image: + images = get_first_4_images(tracks) + images = [i["image"] for i in images] + playlist.images = images + + recovered_item = { + "type": "playlist", + "item": serialize_playlist(playlist), + } + elif item["type"] == "favorite": + recovered_item = { + "type": "favorite", + "item": { + "count": FavoritesTable.count(), + }, + } + elif item["type"] == "track": + track = TrackStore.trackhashmap.get(item["hash"]) + if track is None: + continue + + recovered_item = { + "type": "track", + "item": serialize_track(track.get_best()), + } + + if recovered_item is not None: + helptext = item.get("help_text") or item.get("type") + secondary_text = item.get("secondary_text") + + if "secondary_text" in item: + secondary_text = item["secondary_text"] + elif "timestamp" in item: + secondary_text = timestamp_to_time_passed(item["timestamp"]) + + if helptext: + recovered_item["item"]["help_text"] = helptext + + if secondary_text: + recovered_item["item"]["time"] = secondary_text + + recovered.append(recovered_item) + + return recovered + + def get_recently_played_playlist(limit: int = 100): playlist = Playlist( id="recentlyplayed", diff --git a/app/lib/recipes/__init__.py b/app/lib/recipes/__init__.py index 9065eb2e..b096c680 100644 --- a/app/lib/recipes/__init__.py +++ b/app/lib/recipes/__init__.py @@ -3,25 +3,13 @@ Recipes are a way to create mixes. """ from abc import ABC, abstractmethod -from typing import Any, Dict, List - -from app.db.userdata import UserTable -from app.models.mix import Mix -from app.plugins.mixes import MixesPlugin -from app.store.homepage import HomepageStore - +from typing import Any, List class HomepageRoutine(ABC): """ A routine creates a row of homepage items. """ - title: str - description: str - - items: List[Mix] - extra: Dict[str, Any] - @property @abstractmethod def is_valid(self) -> bool: ... @@ -30,37 +18,12 @@ class HomepageRoutine(ABC): if not self.is_valid: return - self.items = self.run() + self.run() @abstractmethod - def run(self) -> List[Mix]: + def run(self) -> List[Any]: """ Creates the homepage items and saves them to the homepage store if self.is_valid is true. """ ... - - -class ArtistMixes(HomepageRoutine): - items: List[Mix] = [] - extra: Dict[str, Any] = {} - store_key = "artist_mixes" - - @property - def is_valid(self): - return MixesPlugin().enabled - - def run(self): - users = UserTable.get_all() - - for user in users: - mix = MixesPlugin() - mixes = mix.create_artist_mixes(user.id) - - if not mixes: - continue - - HomepageStore.set_mixes(mixes, mixkey=self.store_key, userid=user.id) - - def __init__(self) -> None: - super().__init__() diff --git a/app/lib/recipes/artistmixes.py b/app/lib/recipes/artistmixes.py new file mode 100644 index 00000000..ac0d2199 --- /dev/null +++ b/app/lib/recipes/artistmixes.py @@ -0,0 +1,27 @@ +from app.db.userdata import UserTable +from app.lib.recipes import HomepageRoutine +from app.plugins.mixes import MixesPlugin +from app.store.homepage import HomepageStore + + +class ArtistMixes(HomepageRoutine): + store_key = "artist_mixes" + + @property + def is_valid(self): + return MixesPlugin().enabled + + def run(self): + users = UserTable.get_all() + + for user in users: + mix = MixesPlugin() + mixes = mix.create_artist_mixes(user.id) + + if not mixes: + continue + + HomepageStore.set_mixes(mixes, entrykey=self.store_key, userid=user.id) + + def __init__(self) -> None: + super().__init__() \ No newline at end of file diff --git a/app/lib/recipes/recents.py b/app/lib/recipes/recents.py new file mode 100644 index 00000000..2ac314f7 --- /dev/null +++ b/app/lib/recipes/recents.py @@ -0,0 +1,46 @@ +import pprint +from app.db.userdata import UserTable +from app.lib.home.recentlyadded import get_recently_added_items +from app.lib.home.recentlyplayed import get_recently_played +from app.lib.recipes import HomepageRoutine +from app.store.homepage import HomepageStore + + +class RecentlyPlayed(HomepageRoutine): + store_key = "recently_played" + + def __init__(self, userid: int | None = None) -> None: + """ + The userid is provided when we are running this routine + outside a cron job. ie. when a user records a new scrobble. + """ + self.userids = [userid] if userid else [user.id for user in UserTable.get_all()] + super().__init__() + + @property + def is_valid(self): + return True + + def run(self): + for userid in self.userids: + items = get_recently_played(limit=15, userid=userid) + HomepageStore.entries[self.store_key].items[userid] = items + + +class RecentlyAdded(HomepageRoutine): + ITEM_LIMIT = 15 + store_key = "recently_added" + + @property + def is_valid(self): + return True + + def __init__(self): + super().__init__() + + def run(self): + items = get_recently_added_items(limit=self.ITEM_LIMIT) + + # NOTE: Recently added is a global entry + # So we don't need a userid + HomepageStore.entries[self.store_key].items[0] = items diff --git a/app/lib/recipes/topstreamed.py b/app/lib/recipes/topstreamed.py new file mode 100644 index 00000000..9907d541 --- /dev/null +++ b/app/lib/recipes/topstreamed.py @@ -0,0 +1,84 @@ +from gettext import ngettext +from os import name +import pendulum + +from app.crons.cron import CronJob +from app.db.userdata import UserTable +from app.lib.recipes import HomepageRoutine +from app.store.homepage import HomepageStore +from app.utils.dates import get_date_range, seconds_to_time_string +from app.utils.stats import get_artists_in_period + + +class TopArtists(CronJob, HomepageRoutine): + """ + A routine to populate the top streamed artists/albums in the last week or month + """ + + hours = 1 + ITEM_LIMIT = 15 + + @property + def is_valid(self): + """ + Only valid if it's the middle or last 2 days of this month. + + When the duration is "week", it's valid on Saturday and Sunday. + """ + if self.duration == "month": + now = pendulum.now() + middle_day = now.days_in_month // 2 + + return ( + now.day in range(middle_day, middle_day + 2) + or now.day > now.days_in_month - 2 + ) + if self.duration == "week": + return pendulum.now().isoweekday() in (6, 7) + + return False + + def __init__(self, duration: str = "month") -> None: + super().__init__() + self.duration = duration + + if not self.is_valid: + return + + def run(self): + if not self.is_valid: + self.destroy() + return + + self.userids = [user.id for user in UserTable.get_all()] + + for userid in self.userids: + date_range = get_date_range(self.duration) + artists = get_artists_in_period(date_range[0], date_range[1], userid)[ + : self.ITEM_LIMIT + ] + + artists = [ + { + "type": "artist", + "hash": artist["artisthash"], + "help_text": seconds_to_time_string(artist["playduration"]), + "secondary_text": str(artist["playcount"]) + + " " + + ngettext("play", "plays", artist["playcount"]), + } + for artist in artists + ] + + HomepageStore.entries[f"top_streamed_{self.duration}ly_artists"].items[ + userid + ] = artists + + def destroy(self): + """ + Clear the top streamed entry from the homepage store. + """ + keys = [f"top_streamed_{self.duration}ly_artists"] + + for key in keys: + HomepageStore.entries[key].items = {} diff --git a/app/models/logger.py b/app/models/logger.py index e4b6e368..e3668ea9 100644 --- a/app/models/logger.py +++ b/app/models/logger.py @@ -13,11 +13,17 @@ class TrackLog: duration: int timestamp: int source: str + """ + The full source string, eg. "al:123456" + """ userid: int extra: dict[str, Any] type = "track" type_src = None + """ + The source identifier string, eg. albumhash, artisthash, etc. + """ def __post_init__(self): prefix_map = { diff --git a/app/store/homepage.py b/app/store/homepage.py index 02aa9ed3..12fb8808 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -1,11 +1,11 @@ from abc import ABC from dataclasses import dataclass from typing import Any +from app.lib.home.recentlyplayed import recover_recently_played_items from app.models.mix import Mix from app.utils.auth import get_current_userid -@dataclass class HomepageEntry(ABC): """ Base class for all homepage entries. @@ -15,20 +15,19 @@ class HomepageEntry(ABC): title: str description: str - items: dict[int, dict[str, Any]] + items: dict[int, Any] def __init__(self, title: str, description: str): self.title = title self.description = description - def get_items(self, userid: int): + def get_items(self, userid: int, limit: int | None = None): """ Return usable items for the homepage. """ ... -@dataclass class MixHomepageEntry(HomepageEntry): """ A homepage entry for mixes. @@ -62,26 +61,77 @@ class MixHomepageEntry(HomepageEntry): } +class RecentlyPlayedHomepageEntry(HomepageEntry): + """ + A homepage entry for recently played. + """ + + items: dict[int, list[dict[str, Any]]] + + def __init__(self, title: str, description: str = ""): + super().__init__(title, description) + self.items = {} + + def get_items(self, userid: int, limit: int | None = None): + items = self.items.get(userid, [])[:limit] + + return { + "title": self.title, + "description": self.description, + "items": recover_recently_played_items(items), + } + + +class RecentlyAddedHomepageEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for recently added. + """ + + def get_items(self, userid: int, limit: int | None = None): + return super().get_items(0, limit) + + +class TopStreamedHomepageEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for top streamed. + """ + + # NOTE: This extends RecentlyPlayedHomepageEntry because + # the shape of the data is the same. + pass + + class HomepageStore: """ Stores the homepage items. """ - entries = { + entries: dict[str, HomepageEntry] = { + "recently_played": RecentlyPlayedHomepageEntry( + title="Recently played", + ), "artist_mixes": MixHomepageEntry( title="Artist mixes for you", description="Based on artists you have been listening to", ), + "top_streamed_weekly_artists": TopStreamedHomepageEntry( + title="Top artists this week", + description="Your most played artists since Monday", + ), + "top_streamed_monthly_artists": TopStreamedHomepageEntry( + title="Top artists this month", + description="Your most played artists since the start of the month", + ), + "recently_added": RecentlyAddedHomepageEntry( + title="Recently added", + description="New music added to your library", + ), } @classmethod - def set_mixes(cls, mixes: list[Mix], mixkey: str, userid: int | None = None): - idmap = {mix.id[1:]: mix for mix in mixes} - cls.entries[mixkey].items[userid or get_current_userid()] = idmap - - @classmethod - def get_mixes(cls, mixkey: str, limit: int | None = 9): - return cls.entries[mixkey].get_items(get_current_userid(), limit) + def set_mixes(cls, items: list[Any], entrykey: str, userid: int | None = None): + idmap = {item.id[1:]: item for item in items} + cls.entries[entrykey].items[userid or get_current_userid()] = idmap @classmethod def get_mix(cls, mixkey: str, mixid: str): @@ -89,14 +139,10 @@ class HomepageStore: return mix.to_full_dict() if mix else None @classmethod - def get_mix_by_sourcehash(cls, sourcehash: str): - return next( - ( - mix - for mix in cls.entries["artist_mixes"] - .items.get(get_current_userid(), {}) - .values() - if mix.sourcehash == sourcehash - ), - None, - ) + def get_homepage_items(cls, limit: int): + # return a dict of entry name to entry items + return [ + {entry: cls.entries[entry].get_items(get_current_userid(), limit)} + for entry in cls.entries.keys() + if len(cls.entries[entry].items) + ] diff --git a/app/utils/dates.py b/app/utils/dates.py index c7b5306f..67e77ef7 100644 --- a/app/utils/dates.py +++ b/app/utils/dates.py @@ -46,7 +46,7 @@ def date_string_to_time_passed(prev_date: str) -> str: return timestamp_to_time_passed(then) -def seconds_to_time_string(seconds): +def seconds_to_time_string(seconds: int): """ Converts seconds to a time string. e.g. 1 hour 2 minutes, 1 hour 2 seconds, 1 hour, 1 minute 2 seconds, etc. """ From ef4ecc249904acaf031fe79fca8179d9b0077ec3 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 17 Nov 2024 20:53:00 +0300 Subject: [PATCH 13/44] build recently added and played via hooks --- app/api/scrobble/__init__.py | 10 ++-- app/crons/__init__.py | 2 +- app/db/userdata.py | 8 +++ app/lib/home/recentlyplayed.py | 102 +++++---------------------------- app/lib/index.py | 4 ++ app/lib/recipes/recents.py | 49 +++++++++++++++- 6 files changed, 79 insertions(+), 96 deletions(-) diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index 4a9133a2..834fe06f 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -1,20 +1,15 @@ -from dataclasses import dataclass from gettext import ngettext -from itertools import groupby -from math import e -from pprint import pprint from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint import pendulum from pydantic import Field, BaseModel from app.api.apischemas import TrackHashSchema from typing import Literal -from datetime import datetime, timedelta -from collections import defaultdict import locale from app.db.userdata import FavoritesTable, ScrobbleTable from app.lib.extras import get_extra_info +from app.lib.recipes.recents import RecentlyPlayed from app.models.album import Album from app.models.stats import StatItem from app.models.track import Track @@ -80,6 +75,9 @@ def log_track(body: LogTrackBody): scrobble_data["extra"] = get_extra_info(body.trackhash, "track") ScrobbleTable.add(scrobble_data) + # NOTE: Update the recently played homepage for this userid + RecentlyPlayed(userid=scrobble_data["userid"]) + # Update play data on the in-memory stores track = trackentry.tracks[0] album = AlbumStore.albummap.get(track.albumhash) diff --git a/app/crons/__init__.py b/app/crons/__init__.py index 41e09ed3..4d648e18 100644 --- a/app/crons/__init__.py +++ b/app/crons/__init__.py @@ -18,7 +18,7 @@ def start_cron_jobs(): RecentlyAdded() # Initialized CRON jobs - # Mixes() + Mixes() TopArtists() TopArtists(duration="week") diff --git a/app/db/userdata.py b/app/db/userdata.py index 8cc0fd54..f529413d 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -26,6 +26,7 @@ from app.db.utils import ( plugin_to_dataclasses, similar_artist_to_dataclass, similar_artists_to_dataclass, + tracklog_to_dataclass, tracklog_to_dataclasses, user_to_dataclass, user_to_dataclasses, @@ -319,6 +320,13 @@ class ScrobbleTable(Base): ) return tracklog_to_dataclasses(result.fetchall()) + @classmethod + def get_last_entry(cls, userid: int): + result = cls.execute( + select(cls).where(cls.userid == userid).order_by(cls.timestamp.desc()) + ) + return tracklog_to_dataclass(result.fetchone()) + class PlaylistTable(Base): __tablename__ = "playlist" diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index 0476d26b..1797c01c 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -22,7 +22,14 @@ from app.store.tracks import TrackStore from app.store.artists import ArtistStore -def get_recently_played(limit=7, userid: int | None = None): +def get_recently_played( + limit: int, userid: int | None = None, _entries: list[TrackLog] = [] +): + """ + Get the recently played items for the homepage. + + Pass a list of track log entries to use a subset of the scrobble table. + """ # TODO: Paginate this items = [] added = set() @@ -48,29 +55,12 @@ def get_recently_played(limit=7, userid: int | None = None): if album is None: continue - # album = album_serializer( - # album, - # to_remove={ - # "genres", - # "date", - # "count", - # "duration", - # "albumartists_hashes", - # "og_title", - # }, - # ) item = { "type": "album", "hash": entry.type_src, "timestamp": entry.timestamp, } - # album["help_text"] = "album" - # album["time"] = timestamp_to_time_passed(entry.timestamp) - # { - # "type": "album", - # "item": album, - # } items.append(item) continue @@ -80,10 +70,6 @@ def get_recently_played(limit=7, userid: int | None = None): if artist is None: continue - # artist = serialize_for_card(artist) - # artist["help_text"] = "artist" - # artist["time"] = timestamp_to_time_passed(entry.timestamp) - items.append( { "type": "artist", @@ -108,7 +94,6 @@ def get_recently_played(limit=7, userid: int | None = None): if is_home_dir: folder = os.path.expanduser("~") - # count = FolderStore.count_tracks_containing_paths([folder]) item = { "type": "folder", "hash": entry.type_src, @@ -116,42 +101,12 @@ def get_recently_played(limit=7, userid: int | None = None): } items.append(item) - # { - # "type": "folder", - # "item": { - # "path": folder, - # "count": count[0]["trackcount"], - # "help_text": "folder", - # "time": timestamp_to_time_passed(entry.timestamp), - # }, - # } - continue if entry.type == "playlist": is_custom = entry.type_src in [i["name"] for i in custom_playlists] if is_custom: - # playlist, _ = next( - # i["handler"]() - # for i in custom_playlists - # if i["name"] == entry.type_src - # ) - # playlist.images = [i["image"] for i in playlist.images] - - # playlist = serialize_playlist( - # playlist, to_remove={"settings", "duration"} - # ) - - # playlist["help_text"] = "playlist" - # playlist["time"] = timestamp_to_time_passed(entry.timestamp) - - # items.append( - # { - # "type": "playlist", - # "item": playlist, - # } - # ) items.append( { "type": "playlist", @@ -173,37 +128,10 @@ def get_recently_played(limit=7, userid: int | None = None): } items.append(item) - - # tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) - # playlist.clear_lists() - - # if not playlist.has_image: - # images = get_first_4_images(tracks) - # images = [i["image"] for i in images] - # playlist.images = images - - # items.append( - # { - # "type": "playlist", - # "item": { - # "help_text": "playlist", - # "time": timestamp_to_time_passed(entry.timestamp), - # **serialize_playlist(playlist), - # }, - # } - # ) continue if entry.type == "favorite": items.append( - # { - # "type": "favorite_tracks", - # "item": { - # "help_text": "playlist", - # "count": FavoritesTable.count(), - # "time": timestamp_to_time_passed(entry.timestamp), - # }, - # } { "type": "favorite", "timestamp": entry.timestamp, @@ -221,18 +149,18 @@ def get_recently_played(limit=7, userid: int | None = None): "hash": entry.trackhash, "timestamp": entry.timestamp, } - - # track = serialize_track(t.get_best()) - # track["help_text"] = "track" - # track["time"] = timestamp_to_time_passed(entry.timestamp) - items.append(item) BATCH_SIZE = 200 current_index = 0 - entries = ScrobbleTable.get_all(0, BATCH_SIZE) - max_iterations = 20 # Safeguard against unexpected infinite loops + if len(_entries): + entries = _entries + limit = 1 + else: + entries = ScrobbleTable.get_all(0, BATCH_SIZE) + + max_iterations = 20 iterations = 0 while len(items) < limit and iterations < max_iterations: diff --git a/app/lib/index.py b/app/lib/index.py index dff48dc1..81d11fac 100644 --- a/app/lib/index.py +++ b/app/lib/index.py @@ -7,6 +7,7 @@ from app.lib.mapstuff import ( map_scrobble_data, ) from app.lib.populate import CordinateMedia +from app.lib.recipes.recents import RecentlyAdded from app.lib.tagger import IndexTracks from app.store.albums import AlbumStore from app.store.artists import ArtistStore @@ -25,6 +26,9 @@ class IndexEverything: ArtistStore.load_artists(key) FolderStore.load_filepaths() + # NOTE: Rebuild recently added items on the homepage store + RecentlyAdded() + # map colors map_album_colors() map_artist_colors() diff --git a/app/lib/recipes/recents.py b/app/lib/recipes/recents.py index 2ac314f7..12ff558d 100644 --- a/app/lib/recipes/recents.py +++ b/app/lib/recipes/recents.py @@ -1,5 +1,5 @@ import pprint -from app.db.userdata import UserTable +from app.db.userdata import ScrobbleTable, UserTable from app.lib.home.recentlyadded import get_recently_added_items from app.lib.home.recentlyplayed import get_recently_played from app.lib.recipes import HomepageRoutine @@ -7,6 +7,7 @@ from app.store.homepage import HomepageStore class RecentlyPlayed(HomepageRoutine): + ITEM_LIMIT = 15 store_key = "recently_played" def __init__(self, userid: int | None = None) -> None: @@ -15,6 +16,11 @@ class RecentlyPlayed(HomepageRoutine): outside a cron job. ie. when a user records a new scrobble. """ self.userids = [userid] if userid else [user.id for user in UserTable.get_all()] + + # NOTE: When the userid is provided + # we need to update the store for that userid only + # using the last scrobble entry. + self.update_only = userid is not None super().__init__() @property @@ -22,8 +28,47 @@ class RecentlyPlayed(HomepageRoutine): return True def run(self): + if self.update_only: + last_entry = ScrobbleTable.get_last_entry(self.userids[0]) + + if last_entry: + items = get_recently_played( + limit=self.ITEM_LIMIT, userid=self.userids[0], _entries=[last_entry] + ) + + item = items[0] + store_entry = HomepageStore.entries[self.store_key].items[ + self.userids[0] + ][0] + + if ( + item["type"] + item["hash"] + == store_entry["type"] + store_entry["hash"] + ): + # If the item is the same as the one in the store + # only update the timestamp + HomepageStore.entries[self.store_key].items[self.userids[0]][0][ + "timestamp" + ] = item["timestamp"] + else: + # Otherwise, insert the new item + # and remove the oldest item if there are more than 15 items + HomepageStore.entries[self.store_key].items[self.userids[0]].insert( + 0, item + ) + + if ( + len( + HomepageStore.entries[self.store_key].items[self.userids[0]] + ) + > self.ITEM_LIMIT + ): + HomepageStore.entries[self.store_key].items[ + self.userids[0] + ].pop() + for userid in self.userids: - items = get_recently_played(limit=15, userid=userid) + items = get_recently_played(limit=self.ITEM_LIMIT, userid=userid) HomepageStore.entries[self.store_key].items[userid] = items From dd2bb16a8c7b290c16b2a40cc53ddbf808b4bcf4 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 17 Nov 2024 21:38:51 +0300 Subject: [PATCH 14/44] save mixes to the db --- app/api/plugins/mixes.py | 2 +- app/crons/mixes.py | 2 +- app/db/userdata.py | 6 ++- app/models/mix.py | 9 +++- app/plugins/mixes.py | 94 +++++++++++++++++++++++++++------------- 5 files changed, 79 insertions(+), 34 deletions(-) diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index e513ac6b..f79e3762 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -31,7 +31,7 @@ def get_track_mix(): @api.post("/artist") def get_artist_mix(): mixes = MixesPlugin() - return mixes.create_artist_mixes() + # return mixes.create_artist_mixes() # tracks = mixes.get_artist_mix("09306be8039b98ad") # return { diff --git a/app/crons/mixes.py b/app/crons/mixes.py index db8c2773..08274896 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -8,7 +8,7 @@ class Mixes(CronJob): """ name: str = "mixes" - hours: int = 1 + hours: int = 6 def __init__(self): super().__init__() diff --git a/app/db/userdata.py b/app/db/userdata.py index f529413d..8bb35d31 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -483,6 +483,7 @@ class MixTable(Base): timestamp: Mapped[int] = mapped_column(Integer()) sourcehash: Mapped[str] = mapped_column(String(), unique=True, index=True) tracks: Mapped[list[str]] = mapped_column(JSON(), default_factory=list) + saved: Mapped[bool] = mapped_column(Boolean(), default=False) extra: Mapped[dict[str, Any]] = mapped_column( JSON(), nullable=True, default_factory=dict ) @@ -495,7 +496,10 @@ class MixTable(Base): @classmethod def get_by_sourcehash(cls, sourcehash: str): result = cls.execute(select(cls).where(cls.sourcehash == sourcehash)) - return Mix.mix_to_dataclass(result.fetchone()) + + res = result.fetchone() + if res: + return Mix.mix_to_dataclass(res) @classmethod def insert_one(cls, mix: Mix): diff --git a/app/models/mix.py b/app/models/mix.py index 5255b5cd..091d4616 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -24,8 +24,7 @@ class Mix: saved: bool = False def to_full_dict(self): - # Limit track mix to 30 tracks - tracks = TrackStore.get_tracks_by_trackhashes(self.tracks) + tracks = TrackStore.get_tracks_by_trackhashes(self.tracks)[:40] serialized_tracks = serialize_tracks(tracks) _dict = asdict(self) @@ -37,12 +36,18 @@ class Mix: _dict["duration"] = seconds_to_time_string(sum(t.duration for t in tracks)) _dict["trackcount"] = len(tracks) + del _dict["extra"]["albums"] + del _dict["extra"]["artists"] + return _dict def to_dict(self): item = asdict(self) del item["tracks"] + del item["extra"]["albums"] + del item["extra"]["artists"] + return item @classmethod diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index d0b57e38..69d60e03 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -8,7 +8,7 @@ import requests from urllib.parse import quote from PIL import Image -from app.db.userdata import SimilarArtistTable +from app.db.userdata import MixTable, SimilarArtistTable from app.lib.colorlib import get_image_colors from app.models.artist import Artist from app.models.mix import Mix @@ -25,6 +25,14 @@ from app.utils.remove_duplicates import remove_duplicates from app.utils.stats import get_artists_in_period +class MixAlreadyExists(Exception): + """ + Raised when a mix with the same sourcehash already exists. + """ + + pass + + class MixesPlugin(Plugin): MAX_TRACKS_TO_FETCH = 5 TRACK_MIX_LENGTH = 30 @@ -42,13 +50,22 @@ class MixesPlugin(Plugin): self.set_active(server_online) def ping_server(self): - try: - requests.get(self.server, timeout=10) - except requests.exceptions.ConnectionError: - print("Failed to connect to the recommendation server") - return False + max_retries = 3 + retry_delay = 2 # seconds - return True + for attempt in range(max_retries): + try: + requests.get(self.server, timeout=10) + return True + except Exception as e: + print( + f"Failed to connect to the recommendation server (attempt {attempt + 1}/{max_retries})" + ) + if attempt < max_retries - 1: + time.sleep(retry_delay) + continue + + return False @plugin_method def get_track_mix(self, tracks: list[Track], with_help: bool = False): @@ -73,16 +90,16 @@ class MixesPlugin(Plugin): ] try: - response = requests.post(f"{self.server}/radio", json=queries, timeout=10) - except requests.exceptions.ConnectionError: + response = requests.post(f"{self.server}/radio", json=queries, timeout=30) + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): print("Failed to connect to recommendation server") - return [] + return [], [], [] try: results = response.json() except json.JSONDecodeError: print("Failed to decode JSON response from recommendation server") - return [] + return [], [], [] trackhashes: list[str] = results["tracks"] @@ -119,7 +136,7 @@ class MixesPlugin(Plugin): # try to balance the mix trackmatches = balance_mix(trackmatches) - return trackmatches + return trackmatches, results["albums"], results["albums"] @plugin_method def get_artist_mix(self, artisthash: str): @@ -136,10 +153,11 @@ class MixesPlugin(Plugin): sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) - # TODO: Check if we already have this mix in the - # database and return that instead + if MixTable.get_by_sourcehash(sourcehash): + raise MixAlreadyExists() - return (self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]), sourcehash) + tracks, albums, artists = self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) + return (tracks, albums, artists, sourcehash) @plugin_method def create_artist_mixes(self, userid: int): @@ -239,25 +257,36 @@ class MixesPlugin(Plugin): """ Given an artist dict, creates an artist mix. """ - _artist = ArtistStore.get_artist_by_hash(artist["artisthash"]) + _artist = ArtistStore.artistmap.get(artist["artisthash"]) if not _artist: return None - mix_tracks, sourcehash = self.get_artist_mix(artist["artisthash"]) + tracks = TrackStore.get_tracks_by_trackhashes(_artist.trackhashes) + tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) + sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] + sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) + + db_mix = MixTable.get_by_sourcehash(sourcehash) + if db_mix: + print(f"🔍 Found existing mix for {_artist.artist.name}") + print(db_mix.title) + return db_mix + + mix_tracks, albums, artists = self.get_track_mix(sourcetracks) if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None # try downloading artist image - mix_image = {"image": _artist.image, "color": _artist.color} - downloaded_img_color = self.download_artist_image(_artist) + mix_image = {"image": _artist.artist.image, "color": _artist.artist.color} + downloaded_img_color = self.download_artist_image(_artist.artist) if downloaded_img_color: - mix_image["image"] = f"{_artist.artisthash}.jpg" + mix_image["image"] = f"{_artist.artist.artisthash}.jpg" mix_image["color"] = downloaded_img_color[0] - return Mix( + mix = Mix( # the a prefix indicates that this is an artist mix id=f"a{artist['artisthash']}", title=artist["artist"] + " Radio", @@ -268,10 +297,18 @@ class MixesPlugin(Plugin): "type": "artist", "artisthash": artist["artisthash"], "image": mix_image, + # NOTE: Save the similar albums and artists + # Related to the source tracks that were used to create the mix + # Will be useful when generating other homepage entries + "albums": albums, + "artists": artists, }, timestamp=int(time.time()), ) + MixTable.insert_one(mix) + return mix + def download_artist_image(self, artist: Artist): try: res = requests.get(f"{self.server}/image?artist={artist.name}") @@ -332,7 +369,6 @@ class MixesPlugin(Plugin): for match in albummatches: if len(mixtracks) >= limit: - print(f"Filled up to {limit} tracks with album tracks") return mixtracks albumtracks = [ @@ -346,9 +382,6 @@ class MixesPlugin(Plugin): sample = random.sample(albumtracks, k=1) mixtracks.extend(sample) - print( - f"Supplement: album track {sample[0].title} from ALBUM: {match.album.og_title}" - ) artistmatches = ( a @@ -358,7 +391,6 @@ class MixesPlugin(Plugin): for match in artistmatches: if len(mixtracks) >= limit: - print(f"Filled up to {limit} tracks with artist tracks") return mixtracks artisttracks = [ @@ -372,11 +404,15 @@ class MixesPlugin(Plugin): sample = random.sample(artisttracks, k=1) mixtracks.extend(sample) - print( - f"Supplement: track {sample[0].title} from ARTIST: {match.artist.name}" - ) return mixtracks def get_mix_from_lastfm_data(self, artisthash: str, limit: int): + """ + Creates a mix from the locally available lastfm similar artists data. + + The resulting mix is definitely expected to be of low quality. + + TODO: Implement this! + """ pass From 9de991dd98405eeba9f2ab14596c925126ddb51c Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 17 Nov 2024 20:06:08 +0100 Subject: [PATCH 15/44] fix keyError --- app/lib/home/recentlyadded.py | 48 ++--------------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index 58ef1184..a4251fd1 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -1,5 +1,4 @@ from datetime import datetime -from time import time from app.lib.playlistlib import get_first_4_images from app.models.playlist import Playlist @@ -9,17 +8,12 @@ from app.store.tracks import TrackStore from app.store.albums import AlbumStore from app.store.artists import ArtistStore -from app.serializers.track import serialize_track -from app.serializers.album import album_serializer -from app.serializers.artist import serialize_for_card - from itertools import groupby from app.utils import flatten from app.utils.dates import ( create_new_date, date_string_to_time_passed, - timestamp_to_time_passed, ) older_albums = set() @@ -65,14 +59,6 @@ def create_track(t: Track): "timestamp": t.last_mod, "help_text": "NEW TRACK", } - # track = serialize_track(t, to_remove={"created_date"}) - # track["help_text"] = "NEW TRACK" - - # return { - # "type": "track", - # "item": track, - # } - # INFO: Keys: folder, tracks, time (timestamp) # group_type = dict[str, str | list[Track] | float] @@ -89,7 +75,7 @@ def check_folder_type(group_: dict): if len(tracks) == 1: entry = create_track(tracks[0]) - entry["item"]["time"] = timestamp_to_time_passed(time) + entry["timestamp"] = time return entry is_album, albumhash, _ = check_is_album_folder(tracks) @@ -100,18 +86,6 @@ def check_folder_type(group_: dict): if entry is None: return None - # album = album_serializer( - # entry.album, - # to_remove={ - # "genres", - # "og_title", - # "date", - # "duration", - # "count", - # "albumartists_hashes", - # "base_title", - # }, - # ) return { "type": "album", "hash": albumhash, @@ -127,7 +101,7 @@ def check_folder_type(group_: dict): if entry is None: return None - + return { "type": "artist", "hash": artisthash, @@ -137,18 +111,6 @@ def check_folder_type(group_: dict): ), } - # artist = serialize_for_card(entry.artist) - # artist["trackcount"] = trackcount - # artist["help_text"] = ( - # "NEW ARTIST" if artisthash not in existing_artist_hashes else "NEW MUSIC" - # ) - # artist["time"] = timestamp_to_time_passed(time) - - # return { - # "type": "artist", - # "item": artist, - # } - is_track_folder = check_is_track_folder(tracks) return ( @@ -159,12 +121,6 @@ def check_folder_type(group_: dict): "hash": key, "timestamp": time, "help_text": "NEW MUSIC", - # "item": { - # "path": key, - # "count": len(tracks), - # "help_text": "NEW MUSIC", - # "time": timestamp_to_time_passed(time), - # }, } ) From 70c2558f929551cdcfacb0280820bc293e0e849a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 21 Nov 2024 12:32:49 +0300 Subject: [PATCH 16/44] fix: /playlists returning wrong playlists + homepage recently played showing wrong user id items on first run --- app/db/userdata.py | 8 ++++++-- app/lib/home/recentlyplayed.py | 2 +- app/plugins/mixes.py | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/db/userdata.py b/app/db/userdata.py index 8bb35d31..41d34226 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -345,8 +345,12 @@ class PlaylistTable(Base): ) @classmethod - def get_all(cls): - result = cls.all() + def get_all(cls, current_user: bool = True): + if current_user: + result = cls.execute(select(cls).where(cls.userid == get_current_userid())) + else: + result = cls.execute(select(cls)) + return playlists_to_dataclasses(result) @classmethod diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index 1797c01c..6aee67d1 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -158,7 +158,7 @@ def get_recently_played( entries = _entries limit = 1 else: - entries = ScrobbleTable.get_all(0, BATCH_SIZE) + entries = ScrobbleTable.get_all(0, BATCH_SIZE, userid=userid) max_iterations = 20 iterations = 0 diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 69d60e03..d4171b17 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -201,6 +201,8 @@ class MixesPlugin(Plugin): }, } + # FIXME: Make sure that different artists don't generate the same mix + for i, period in enumerate(artists.values()): # if previous period has less than its max # add the difference to this period's limit @@ -226,6 +228,7 @@ class MixesPlugin(Plugin): period["created"] += 1 print(f"⭐⭐⭐⭐ Created {len(mixes)} mixes") + print([m.title for m in mixes]) return mixes def get_mix_description(self, tracks: list[Track], artishash: str): From 7be782d91e5c1441b5e4e1ba753e9aec89ca2338 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 21 Nov 2024 13:16:14 +0300 Subject: [PATCH 17/44] add contributors to README.md --- README.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/README.md b/README.md index ed16f4ba..c608a589 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,72 @@ Swing Music is looking for contributors. If you're interested, please join us at This software is provided to you with terms stated in the MIT License. Read the full text in the `LICENSE` file located at the root of this repository. **[MIT License](https://opensource.org/licenses/MIT) | Copyright (c) 2023 Mungai Njoroge** + +### Contributors + +Shout out to the following code contributors who have helped maintain and improve Swing Music: + + From 2aa6b3b1de39b95cac40c661d1545d8a4e463a61 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 21 Nov 2024 13:20:11 +0300 Subject: [PATCH 18/44] reduce image height to 80px --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c608a589..bf5abf6f 100644 --- a/README.md +++ b/README.md @@ -107,49 +107,49 @@ Shout out to the following code contributors who have helped maintain and improv - +
@cwilvx - +
@Ericgacoki - +
@Simonh2o - +
@tcsenpai - +
@jensgrunzer1 - +
@Type-Delta - +
@MarcOrfilaCarreras @@ -158,7 +158,7 @@ Shout out to the following code contributors who have helped maintain and improv - +
@tralph3 From e42ec3afb52884ca0496787da900427546c92955 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 21 Nov 2024 14:40:27 +0300 Subject: [PATCH 19/44] update top streamed cron to show on weekends --- app/lib/recipes/topstreamed.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/lib/recipes/topstreamed.py b/app/lib/recipes/topstreamed.py index 9907d541..e04af347 100644 --- a/app/lib/recipes/topstreamed.py +++ b/app/lib/recipes/topstreamed.py @@ -1,5 +1,4 @@ from gettext import ngettext -from os import name import pendulum from app.crons.cron import CronJob @@ -23,7 +22,7 @@ class TopArtists(CronJob, HomepageRoutine): """ Only valid if it's the middle or last 2 days of this month. - When the duration is "week", it's valid on Saturday and Sunday. + When the duration is "week", it's valid on the weekend. """ if self.duration == "month": now = pendulum.now() @@ -34,7 +33,7 @@ class TopArtists(CronJob, HomepageRoutine): or now.day > now.days_in_month - 2 ) if self.duration == "week": - return pendulum.now().isoweekday() in (6, 7) + return pendulum.now().isoweekday() in (5, 6, 7) return False From 809415ddb4b7be4df9501aa901f92054c14ae2af Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 27 Nov 2024 10:55:11 +0300 Subject: [PATCH 20/44] add because you listened to artist artists + add artists you might like --- app/crons/__init__.py | 2 +- app/crons/mixes.py | 5 +++ app/lib/home/recentlyplayed.py | 4 +-- app/lib/recipes/because.py | 37 +++++++++++++++++++++ app/plugins/mixes.py | 61 +++++++++++++++++++++++++++++++++- app/store/homepage.py | 38 ++++++++++++++++++--- 6 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 app/lib/recipes/because.py diff --git a/app/crons/__init__.py b/app/crons/__init__.py index 4d648e18..8e5c56a0 100644 --- a/app/crons/__init__.py +++ b/app/crons/__init__.py @@ -18,9 +18,9 @@ def start_cron_jobs(): RecentlyAdded() # Initialized CRON jobs - Mixes() TopArtists() TopArtists(duration="week") + Mixes() # Trigger all CRON jobs when the app is started. schedule.run_all() diff --git a/app/crons/mixes.py b/app/crons/mixes.py index 08274896..dbb596c1 100644 --- a/app/crons/mixes.py +++ b/app/crons/mixes.py @@ -1,5 +1,6 @@ from app.crons.cron import CronJob from app.lib.recipes.artistmixes import ArtistMixes +from app.lib.recipes.because import BecauseYouListened class Mixes(CronJob): @@ -19,3 +20,7 @@ class Mixes(CronJob): """ print("⭐⭐⭐⭐ Mixes cron job running") ArtistMixes() + + # INFO: Because you listened to artist items are generated using + # the artist mixes, so run them after the artist mixes are created. + BecauseYouListened() diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index 6aee67d1..a565ab80 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -184,7 +184,7 @@ def get_recently_played( return items -def recover_recently_played_items(items: list[dict]): +def recover_items(items: list[dict]): custom_playlists = [ {"name": "recentlyadded", "handler": get_recently_added_playlist}, {"name": "recentlyplayed", "handler": get_recently_played_playlist}, @@ -235,7 +235,7 @@ def recover_recently_played_items(items: list[dict]): }, } elif item["type"] == "playlist": - if item["is_custom"]: + if item.get("is_custom"): playlist, _ = next( i["handler"]() for i in custom_playlists diff --git a/app/lib/recipes/because.py b/app/lib/recipes/because.py new file mode 100644 index 00000000..2454e5b0 --- /dev/null +++ b/app/lib/recipes/because.py @@ -0,0 +1,37 @@ +from pprint import pprint +from app.db.userdata import UserTable +from app.lib.recipes import HomepageRoutine +from app.lib.recipes.artistmixes import ArtistMixes +from app.models.mix import Mix +from app.plugins.mixes import MixesPlugin +from app.store.homepage import HomepageStore + + +class BecauseYouListened(HomepageRoutine): + store_keys = ["because_you_listened_to_artist", "artists_you_might_like"] + + @property + def is_valid(self): + return MixesPlugin().enabled + + def run(self): + users = UserTable.get_all() + + for user in users: + entry: dict[str, Mix] = HomepageStore.entries.get( + ArtistMixes.store_key + ).items.get(user.id) # type: ignore + + if not entry: + continue + + because_you_listened_to_artist, artists_you_might_like = ( + MixesPlugin().get_because_items(list(entry.values())) + ) + + HomepageStore.entries[self.store_keys[0]].items[ + user.id + ] = because_you_listened_to_artist + HomepageStore.entries[self.store_keys[1]].items[ + user.id + ] = artists_you_might_like diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index d4171b17..cecbab6b 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -1,4 +1,5 @@ import datetime +from gettext import ngettext import json from pprint import pprint import random @@ -136,7 +137,7 @@ class MixesPlugin(Plugin): # try to balance the mix trackmatches = balance_mix(trackmatches) - return trackmatches, results["albums"], results["albums"] + return trackmatches, results["albums"], results["artists"] @plugin_method def get_artist_mix(self, artisthash: str): @@ -419,3 +420,61 @@ class MixesPlugin(Plugin): TODO: Implement this! """ pass + + def get_because_items(self, mixes: list[Mix]): + """ + Given a list of mixes, returns a list of artists that are similar to the + artists in the mixes. + """ + artists: dict[str, list[dict[str, str | int]]] = {} + + for mix in mixes: + mix_artisthash = mix.extra["artisthash"] + artists.setdefault(mix_artisthash, []) + + for artisthash in mix.extra["artists"]: + artist = ArtistStore.artistmap.get(artisthash) + + if not artist: + continue + + artists[mix_artisthash].append( + { + "type": "artist", + "trackcount": artist.artist.trackcount, + "hash": artisthash, + "help_text": str(artist.artist.trackcount) + + ngettext(" track", " tracks", artist.artist.trackcount), + } + ) + + # INFO: Sort artists by trackcount + artists[mix_artisthash] = sorted( + artists[mix_artisthash], + key=lambda x: x["trackcount"], + reverse=True, + ) + + artisthash = mixes[0].extra["artisthash"] + because_you_listened_to_artist = { + "title": "Because you listened to " + + ArtistStore.artistmap[artisthash].artist.name, + "items": artists[artisthash][:15], + } + + # Flatten list of artists and remove duplicates by artisthash + all_artists = [] + seen = set() + + for artist_list in artists.values(): + for artist in artist_list: + if artist["hash"] not in seen: + all_artists.append(artist) + seen.add(artist["hash"]) + + artists_you_might_like = { + "title": "Artists you might like", + "items": random.sample(all_artists, k=min(15, len(all_artists))), + } + + return because_you_listened_to_artist, artists_you_might_like diff --git a/app/store/homepage.py b/app/store/homepage.py index 12fb8808..28bfab69 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -1,7 +1,8 @@ from abc import ABC from dataclasses import dataclass +import pprint from typing import Any -from app.lib.home.recentlyplayed import recover_recently_played_items +from app.lib.home.recentlyplayed import recover_items from app.models.mix import Mix from app.utils.auth import get_current_userid @@ -78,7 +79,7 @@ class RecentlyPlayedHomepageEntry(HomepageEntry): return { "title": self.title, "description": self.description, - "items": recover_recently_played_items(items), + "items": recover_items(items), } @@ -91,7 +92,7 @@ class RecentlyAddedHomepageEntry(RecentlyPlayedHomepageEntry): return super().get_items(0, limit) -class TopStreamedHomepageEntry(RecentlyPlayedHomepageEntry): +class GenericRecoverableEntry(RecentlyPlayedHomepageEntry): """ A homepage entry for top streamed. """ @@ -101,6 +102,25 @@ class TopStreamedHomepageEntry(RecentlyPlayedHomepageEntry): pass +class BecauseYouListenedToArtistHomepageEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for because you listened to artist. + """ + + # SHAPE: {userid: {title: str, items: list[RecoverableItem]}} + items: dict[int, dict[str, Any]] + + def get_items(self, userid: int, limit: int | None = None): + pprint.pprint(self.items) + title = self.items.get(userid, {}).get("title") + items = self.items.get(userid, {}).get("items", [])[:limit] + + return { + "title": title, + "items": recover_items(items), + } + + class HomepageStore: """ Stores the homepage items. @@ -114,14 +134,22 @@ class HomepageStore: title="Artist mixes for you", description="Based on artists you have been listening to", ), - "top_streamed_weekly_artists": TopStreamedHomepageEntry( + "top_streamed_weekly_artists": GenericRecoverableEntry( title="Top artists this week", description="Your most played artists since Monday", ), - "top_streamed_monthly_artists": TopStreamedHomepageEntry( + "top_streamed_monthly_artists": GenericRecoverableEntry( title="Top artists this month", description="Your most played artists since the start of the month", ), + "because_you_listened_to_artist": BecauseYouListenedToArtistHomepageEntry( + title="", + description="Artists similar to the artist you listened to", + ), + "artists_you_might_like": BecauseYouListenedToArtistHomepageEntry( + title="Artists you might like", + description="Artists similar to the artists you have listened to", + ), "recently_added": RecentlyAddedHomepageEntry( title="Recently added", description="New music added to your library", From ee0f6c76462bf577b393eafa6f0eafefabe4f6c9 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 27 Nov 2024 12:35:48 +0300 Subject: [PATCH 21/44] add track mixes --- app/api/plugins/mixes.py | 2 + app/lib/recipes/artistmixes.py | 16 ++++- app/models/mix.py | 4 +- app/plugins/mixes.py | 107 ++++++++++++++++++++++++++++----- app/store/homepage.py | 7 ++- 5 files changed, 116 insertions(+), 20 deletions(-) diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index f79e3762..a22bab83 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -53,6 +53,8 @@ def get_mix(query: MixQuery): match query.mixid[0]: case "a": mixtype = "artist_mixes" + case "t": + mixtype = "custom_mixes" case _: return {"msg": "Invalid mix ID"}, 400 diff --git a/app/lib/recipes/artistmixes.py b/app/lib/recipes/artistmixes.py index ac0d2199..91a9af81 100644 --- a/app/lib/recipes/artistmixes.py +++ b/app/lib/recipes/artistmixes.py @@ -23,5 +23,19 @@ class ArtistMixes(HomepageRoutine): HomepageStore.set_mixes(mixes, entrykey=self.store_key, userid=user.id) + custom_mixes = [] + for _mix in mixes: + custom_mix = mix.get_custom_mix_items(_mix) + + if custom_mix: + custom_mixes.append(custom_mix) + + for index, custom_mix in enumerate(custom_mixes): + custom_mix.title = f"Mix {index + 1}" + + HomepageStore.set_mixes( + custom_mixes, entrykey="custom_mixes", userid=user.id + ) + def __init__(self) -> None: - super().__init__() \ No newline at end of file + super().__init__() diff --git a/app/models/mix.py b/app/models/mix.py index 091d4616..84fe41d5 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -30,8 +30,8 @@ class Mix: _dict = asdict(self) _dict["tracks"] = serialized_tracks - if not self.extra.get("image"): - _dict["images"] = get_first_4_images(tracks) + # if not self.extra.get("image"): + # _dict["images"] = get_first_4_images(tracks) _dict["duration"] = seconds_to_time_string(sum(t.duration for t in tracks)) _dict["trackcount"] = len(tracks) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index cecbab6b..729e8a5d 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -1,16 +1,13 @@ -import datetime from gettext import ngettext import json -from pprint import pprint import random -import string import time import requests -from urllib.parse import quote from PIL import Image -from app.db.userdata import MixTable, SimilarArtistTable +from app.db.userdata import MixTable from app.lib.colorlib import get_image_colors +from app.lib.playlistlib import get_first_4_images from app.models.artist import Artist from app.models.mix import Mix from app.models.track import Track @@ -22,7 +19,6 @@ from app.store.tracks import TrackStore from app.utils.dates import get_date_range, get_duration_ago from app.utils.hashing import create_hash from app.utils.mixes import balance_mix -from app.utils.remove_duplicates import remove_duplicates from app.utils.stats import get_artists_in_period @@ -36,8 +32,8 @@ class MixAlreadyExists(Exception): class MixesPlugin(Plugin): MAX_TRACKS_TO_FETCH = 5 - TRACK_MIX_LENGTH = 30 MIN_TRACK_MIX_LENGTH = 15 + MIX_TRACKS = 40 MIN_DAY_LISTEN_DURATION = 3 * 60 # 3 minutes MIN_WEEK_LISTEN_DURATION = 10 * 60 # 10 minutes @@ -421,16 +417,76 @@ class MixesPlugin(Plugin): """ pass + def get_custom_mix_items(self, mix: Mix): + """ + Given a mix, returns the excess tracks as a custom mix. + """ + + # INFO: If the mix can't have more than 20 tracks, return None + if len(mix.tracks) <= self.MIX_TRACKS + 20: + return None + + tracks = TrackStore.get_tracks_by_trackhashes(mix.tracks[self.MIX_TRACKS :]) + + return Mix( + id=f"t{mix.extra['artisthash']}", + title="", # INFO: Will be filled after all mixes are created. + description=self.get_mix_description(tracks, mix.extra["artisthash"]), + tracks=[t.trackhash for t in tracks], + sourcehash=create_hash(*[t.trackhash for t in tracks]), + extra={ + "type": "track", + "images": self.get_custom_mix_images(tracks), + "artists": None, + "albums": None, + }, + ) + + def get_custom_mix_images(self, tracks: list[Track]): + + first_album = tracks[0].albumhash + first_img = { + "image": first_album + ".webp", + "type": "album", + "color": AlbumStore.albummap[first_album].album.color, + } + + seen = set() + images = [first_img] + + for track in tracks[1:]: + artisthash = track.artists[0]["artisthash"] + + if artisthash in seen: + continue + + seen.add(artisthash) + + image = { + "image": artisthash + ".webp", + "type": "artist", + "color": ArtistStore.get_artist_by_hash(artisthash).color, + } + + images.append(image) + + if len(images) == 3: + break + + return images + def get_because_items(self, mixes: list[Mix]): """ Given a list of mixes, returns a list of artists that are similar to the artists in the mixes. """ artists: dict[str, list[dict[str, str | int]]] = {} + albums: dict[str, list[dict[str, str | int]]] = {} for mix in mixes: mix_artisthash = mix.extra["artisthash"] artists.setdefault(mix_artisthash, []) + albums.setdefault(mix_artisthash, []) for artisthash in mix.extra["artists"]: artist = ArtistStore.artistmap.get(artisthash) @@ -448,6 +504,22 @@ class MixesPlugin(Plugin): } ) + for albumhash in mix.extra["albums"]: + album = AlbumStore.albummap.get(albumhash) + + if not album: + continue + + albums[mix_artisthash].append( + { + "type": "album", + "trackcount": album.album.trackcount, + "hash": albumhash, + "help_text": str(album.album.trackcount) + + ngettext(" track", " tracks", album.album.trackcount), + } + ) + # INFO: Sort artists by trackcount artists[mix_artisthash] = sorted( artists[mix_artisthash], @@ -455,26 +527,33 @@ class MixesPlugin(Plugin): reverse=True, ) + # INFO: Sort albums by trackcount + albums[mix_artisthash] = sorted( + albums[mix_artisthash], + key=lambda x: x["trackcount"], + reverse=True, + ) + artisthash = mixes[0].extra["artisthash"] because_you_listened_to_artist = { "title": "Because you listened to " + ArtistStore.artistmap[artisthash].artist.name, - "items": artists[artisthash][:15], + "items": albums[artisthash][:15], } # Flatten list of artists and remove duplicates by artisthash all_artists = [] seen = set() - for artist_list in artists.values(): - for artist in artist_list: - if artist["hash"] not in seen: - all_artists.append(artist) - seen.add(artist["hash"]) + # for artist_list in artists.values(): + # for artist in artist_list: + # if artist["hash"] not in seen: + # all_artists.append(artist) + # seen.add(artist["hash"]) artists_you_might_like = { "title": "Artists you might like", - "items": random.sample(all_artists, k=min(15, len(all_artists))), + "items": artists[artisthash][:15], } return because_you_listened_to_artist, artists_you_might_like diff --git a/app/store/homepage.py b/app/store/homepage.py index 28bfab69..289c41c9 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -1,6 +1,4 @@ from abc import ABC -from dataclasses import dataclass -import pprint from typing import Any from app.lib.home.recentlyplayed import recover_items from app.models.mix import Mix @@ -111,7 +109,6 @@ class BecauseYouListenedToArtistHomepageEntry(RecentlyPlayedHomepageEntry): items: dict[int, dict[str, Any]] def get_items(self, userid: int, limit: int | None = None): - pprint.pprint(self.items) title = self.items.get(userid, {}).get("title") items = self.items.get(userid, {}).get("items", [])[:limit] @@ -134,6 +131,10 @@ class HomepageStore: title="Artist mixes for you", description="Based on artists you have been listening to", ), + "custom_mixes": MixHomepageEntry( + title="Mixes for you", + description="Because artist mixes alone aren't enough", + ), "top_streamed_weekly_artists": GenericRecoverableEntry( title="Top artists this week", description="Your most played artists since Monday", From 37374d6c82524b16097d48304db712e6498dfbdc Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 1 Dec 2024 22:00:22 +0300 Subject: [PATCH 22/44] use artist tracks in period to generate mix --- app/plugins/mixes.py | 25 +++++++++++++++++-------- app/utils/stats.py | 10 +++++++++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 729e8a5d..174b3dca 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -217,7 +217,13 @@ class MixesPlugin(Plugin): if artist["artisthash"] in indexed: continue - mix = self.create_artist_mix(artist) + # INFO: track['tracks'] is a dict of trackhashes and their counts + # get the trackhashes sorted by count + trackhashes = sorted( + artist["tracks"], key=lambda x: artist["tracks"][x], reverse=True + ) + + mix = self.create_artist_mix(artist, trackhashes[:self.MAX_TRACKS_TO_FETCH]) if mix: mixes.append(mix) @@ -253,7 +259,7 @@ class MixesPlugin(Plugin): return f"Featuring {tracks[0].artists[0]['name']}" - def create_artist_mix(self, artist: dict[str, str]): + def create_artist_mix(self, artist: dict[str, str], trackhashes: list[str]): """ Given an artist dict, creates an artist mix. """ @@ -262,10 +268,12 @@ class MixesPlugin(Plugin): if not _artist: return None - tracks = TrackStore.get_tracks_by_trackhashes(_artist.trackhashes) - tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) - sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] - sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) + tracks = TrackStore.get_tracks_by_trackhashes(trackhashes) + # tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) + # sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] + + # INFO: Sort the trackhashes when creating the sourcehash + sourcehash = create_hash(*sorted(trackhashes, key=lambda x: trackhashes.index(x))) db_mix = MixTable.get_by_sourcehash(sourcehash) if db_mix: @@ -273,7 +281,7 @@ class MixesPlugin(Plugin): print(db_mix.title) return db_mix - mix_tracks, albums, artists = self.get_track_mix(sourcetracks) + mix_tracks, albums, artists = self.get_track_mix(tracks) if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None @@ -296,6 +304,7 @@ class MixesPlugin(Plugin): extra={ "type": "artist", "artisthash": artist["artisthash"], + "sourcetracks": trackhashes, "image": mix_image, # NOTE: Save the similar albums and artists # Related to the source tracks that were used to create the mix @@ -430,7 +439,7 @@ class MixesPlugin(Plugin): return Mix( id=f"t{mix.extra['artisthash']}", - title="", # INFO: Will be filled after all mixes are created. + title="", # INFO: Will be filled after all mixes are created. description=self.get_mix_description(tracks, mix.extra["artisthash"]), tracks=[t.trackhash for t in tracks], sourcehash=create_hash(*[t.trackhash for t in tracks]), diff --git a/app/utils/stats.py b/app/utils/stats.py index 0ed38eb1..50087001 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -15,12 +15,15 @@ def get_artists_in_period( start_time: int | float, end_time: int | float, userid: int | None = None ): scrobbles = ScrobbleTable.get_all_in_period(start_time, end_time, userid) - artists: Any = defaultdict(lambda: {"playcount": 0, "playduration": 0}) + artists: Any = defaultdict( + lambda: {"playcount": 0, "playduration": 0, "tracks": {}} + ) for scrobble in scrobbles: track = TrackStore.get_tracks_by_trackhashes([scrobble.trackhash]) if not track: continue + track = track[0] for artist in track.artists: @@ -31,6 +34,11 @@ def get_artists_in_period( artists[artisthash]["playcount"] += 1 artists[artisthash]["playduration"] += scrobble.duration + # index the track counts too + artists[artisthash]["tracks"][track.trackhash] = ( + artists[artisthash]["tracks"].get(track.trackhash, 0) + 1 + ) + artists = list(artists.values()) return sorted(artists, key=lambda x: x["playduration"], reverse=True) From 8ff283cbcbc1cfb2900f406dcf201929f93b4d8b Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sun, 8 Dec 2024 16:20:58 +0300 Subject: [PATCH 23/44] update mix api endpoint payload shape --- TODO.md | 2 +- app/plugins/mixes.py | 3 ++- app/store/tracks.py | 16 ++++++++++++++++ manage.py | 2 ++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index cd34050f..b668a9db 100644 --- a/TODO.md +++ b/TODO.md @@ -21,7 +21,7 @@ -# DONE + # DONE - Support auth headers - Add recently played playlist diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 174b3dca..0c3a2b29 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -79,7 +79,8 @@ class MixesPlugin(Plugin): """ queries = [ { - "query": f"{track.title} - {','.join(a['name'] for a in track.artists)}", + "title": track.title, + "artists": [a["name"] for a in track.artists], "album": track.og_album, "with_help": with_help, } diff --git a/app/store/tracks.py b/app/store/tracks.py index cc82efc8..7b0524cf 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -1,6 +1,7 @@ # from tqdm import tqdm import itertools +import json from typing import Callable, Iterable from app.db.libdata import TrackTable @@ -313,3 +314,18 @@ class TrackStore: def get_recently_played(cls, limit: int): tracks = cls.get_flat_list() return sorted(tracks, key=lambda x: x.lastplayed, reverse=True)[:limit] + + @classmethod + def export(cls): + path = "tracks.json" + + with open(path, "w") as f: + data = [ + { + "title": t.title, + "album": t.album, + "artists": [a["name"] for a in t.artists], + } + for t in cls.get_flat_list() + ] + json.dump(data, f) diff --git a/manage.py b/manage.py index afdb7a85..d6fe3f3e 100644 --- a/manage.py +++ b/manage.py @@ -27,6 +27,7 @@ from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info from app.setup import load_into_mem, run_setup from app.start_info_logger import log_startup_info +from app.store.tracks import TrackStore from app.utils.filesystem import get_home_res_path from app.utils.paths import getClientFilesExtensions from app.utils.threading import background @@ -227,6 +228,7 @@ if __name__ == "__main__": load_into_mem() run_swingmusic() + TrackStore.export() host = FLASKVARS.get_flask_host() port = FLASKVARS.get_flask_port() From 77485dd0a77feeb55b3ed7e699065313a910abd2 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 11 Dec 2024 14:22:20 +0300 Subject: [PATCH 24/44] adapt to new cloud endpoints + export artists to json --- app/plugins/mixes.py | 60 +++++++++++++++++++++++++------------------- app/store/artists.py | 13 ++++++++++ manage.py | 4 ++- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 0c3a2b29..83259890 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -1,4 +1,5 @@ from gettext import ngettext +from io import BytesIO import json import random import time @@ -6,8 +7,6 @@ import requests from PIL import Image from app.db.userdata import MixTable -from app.lib.colorlib import get_image_colors -from app.lib.playlistlib import get_first_4_images from app.models.artist import Artist from app.models.mix import Mix from app.models.track import Track @@ -42,9 +41,10 @@ class MixesPlugin(Plugin): def __init__(self): super().__init__("mixes", "Mixes") self.server = "https://smcloud.mungaist.com" + # self.server = "http://localhost:1956" - server_online = self.ping_server() - self.set_active(server_online) + # server_online = self.ping_server() + self.set_active(True) def ping_server(self): max_retries = 3 @@ -224,7 +224,9 @@ class MixesPlugin(Plugin): artist["tracks"], key=lambda x: artist["tracks"][x], reverse=True ) - mix = self.create_artist_mix(artist, trackhashes[:self.MAX_TRACKS_TO_FETCH]) + mix = self.create_artist_mix( + artist, trackhashes[: self.MAX_TRACKS_TO_FETCH] + ) if mix: mixes.append(mix) @@ -274,7 +276,9 @@ class MixesPlugin(Plugin): # sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] # INFO: Sort the trackhashes when creating the sourcehash - sourcehash = create_hash(*sorted(trackhashes, key=lambda x: trackhashes.index(x))) + sourcehash = create_hash( + *sorted(trackhashes, key=lambda x: trackhashes.index(x)) + ) db_mix = MixTable.get_by_sourcehash(sourcehash) if db_mix: @@ -289,11 +293,10 @@ class MixesPlugin(Plugin): # try downloading artist image mix_image = {"image": _artist.artist.image, "color": _artist.artist.color} - downloaded_img_color = self.download_artist_image(_artist.artist) + image = self.download_artist_image(_artist.artist) - if downloaded_img_color: - mix_image["image"] = f"{_artist.artist.artisthash}.jpg" - mix_image["color"] = downloaded_img_color[0] + if image: + mix_image["image"] = image mix = Mix( # the a prefix indicates that this is an artist mix @@ -321,30 +324,35 @@ class MixesPlugin(Plugin): def download_artist_image(self, artist: Artist): try: - res = requests.get(f"{self.server}/image?artist={artist.name}") + res = requests.get( + f"{self.server}/mix/image?artist={artist.name}&type=Artist" + ) except requests.exceptions.ConnectionError: return None if res.status_code == 200: - # save to file - with open( - f"{Paths.get_md_mixes_img_path()}/{artist.artisthash}.jpg", "wb" - ) as f: - f.write(res.content) + filename = f"{artist.artisthash}_{int(time.time())}.webp" + path = Paths.get_md_mixes_img_path() + "/" + filename - # resize to 256px width while maintaining aspect ratio - img = Image.open(f"{Paths.get_md_mixes_img_path()}/{artist.artisthash}.jpg") - aspect_ratio = img.width / img.height + image = Image.open(BytesIO(res.content)) + aspect_ratio = image.width / image.height - newwidth = 256 - newheight = int(256 / aspect_ratio) + # resize to 512px + md_width = 512 + md_height = int(md_width / aspect_ratio) - img = img.resize((newwidth, newheight), Image.LANCZOS) - img.save(f"{Paths.get_sm_mixes_img_path()}/{artist.artisthash}.jpg") + image = image.resize((md_width, md_height), Image.LANCZOS) + image.save(path, "webp") - return get_image_colors( - f"{Paths.get_sm_mixes_img_path()}/{artist.artisthash}.jpg" - ) + # resize to 256px + sm_width = 256 + sm_height = int(sm_width / aspect_ratio) + + image = image.resize((sm_width, sm_height), Image.LANCZOS) + small_path = Paths.get_sm_mixes_img_path() + "/" + filename + image.save(small_path, "webp") + + return filename return None diff --git a/app/store/artists.py b/app/store/artists.py index f6b27d41..2ca1554e 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -162,3 +162,16 @@ class ArtistStore: return TrackStore.get_tracks_by_trackhashes(entry.trackhashes) return [] + + @classmethod + def export(cls): + path = "artists.json" + + with open(path, "w") as f: + data = [ + { + "name": a.name, + } + for a in cls.get_flat_list() + ] + json.dump(data, f) diff --git a/manage.py b/manage.py index d6fe3f3e..fb4b1779 100644 --- a/manage.py +++ b/manage.py @@ -27,6 +27,7 @@ from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info from app.setup import load_into_mem, run_setup from app.start_info_logger import log_startup_info +from app.store.artists import ArtistStore from app.store.tracks import TrackStore from app.utils.filesystem import get_home_res_path from app.utils.paths import getClientFilesExtensions @@ -229,7 +230,8 @@ if __name__ == "__main__": load_into_mem() run_swingmusic() TrackStore.export() - + ArtistStore.export() + host = FLASKVARS.get_flask_host() port = FLASKVARS.get_flask_port() From 86afa66aca2965d005d14fb98c74fd0cbb5ee915 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Wed, 11 Dec 2024 18:28:02 +0100 Subject: [PATCH 25/44] bump waitress threads to 100 --- manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.py b/manage.py index fb4b1779..912a4072 100644 --- a/manage.py +++ b/manage.py @@ -239,7 +239,7 @@ if __name__ == "__main__": app, host=host, port=port, - threads=10, + threads=100, ipv6=True, ipv4=True, ) From 98720466aa53d970c6af803ba14895a2ccd15d6a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 17:31:55 +0300 Subject: [PATCH 26/44] implement saving mixes + add: get mixes + handle mixes on recently played + move modules around to fix circular deps --- app/api/home/__init__.py | 2 +- app/api/plugins/mixes.py | 97 ++++++--- app/db/userdata.py | 79 +++++++- app/lib/home/__init__.py | 303 ++++++++++++++++++++++++++++ app/lib/home/create_items.py | 158 +++++++++++++++ app/lib/home/get_recently_played.py | 47 +++++ app/lib/home/recentlyplayed.py | 294 --------------------------- app/lib/recipes/artistmixes.py | 5 +- app/lib/recipes/recents.py | 2 +- app/models/logger.py | 3 +- app/models/mix.py | 11 +- app/plugins/mixes.py | 80 +++++--- app/store/homepage.py | 138 ++----------- app/store/homepageentries.py | 118 +++++++++++ 14 files changed, 860 insertions(+), 477 deletions(-) create mode 100644 app/lib/home/__init__.py create mode 100644 app/lib/home/create_items.py create mode 100644 app/lib/home/get_recently_played.py create mode 100644 app/store/homepageentries.py diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index e7fa192a..095c0f44 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema from app.lib.home.recentlyadded import get_recently_added_items -from app.lib.home.recentlyplayed import get_recently_played +from app.lib.home.get_recently_played import get_recently_played from app.store.homepage import HomepageStore bp_tag = Tag(name="Home", description="Homepage items") diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index a22bab83..168b6a1c 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -1,7 +1,9 @@ +from typing import Literal from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field +from app.db.userdata import MixTable from app.plugins.mixes import MixesPlugin from app.store.homepage import HomepageStore from app.store.tracks import TrackStore @@ -13,37 +15,43 @@ api = APIBlueprint( ) -@api.post("/track") -def get_track_mix(): - """ - Get a track mix - """ - mixes = MixesPlugin() - track = TrackStore.trackhashmap["9eeee292264ad01b"].get_best() - tracks = mixes.get_track_mix([track]) - - return { - "total": len(tracks), - "tracks": tracks, - } +class GetMixesBody(BaseModel): + mixtype: Literal["artists", "tracks"] = Field(description="The type of mix") -@api.post("/artist") -def get_artist_mix(): - mixes = MixesPlugin() - # return mixes.create_artist_mixes() - # tracks = mixes.get_artist_mix("09306be8039b98ad") +@api.get("/") +def get_artist_mixes(path: GetMixesBody): + srcmixes = MixTable.get_all(with_userid=True) + mixes = [] - # return { - # "total": len(tracks), - # "tracks": tracks, - # } + if path.mixtype == "artists": + mixes = [mix.to_dict(convert_timestamp=True) for mix in srcmixes] + elif path.mixtype == "tracks": + plugin = MixesPlugin() - return "hi" + for mix in srcmixes: + custom_mix = plugin.get_track_mix(mix) + if custom_mix: + mixes.append(custom_mix.to_dict(convert_timestamp=True)) + + seen_mixids = set() + + # filter duplicates by trackshash + final_mixes = [] + for mix in mixes: + # INFO: Ignore duplicates for artist mixes + if mix["id"] in seen_mixids and path.mixtype == "tracks": + continue + + final_mixes.append(mix) + seen_mixids.add(mix["id"]) + + return final_mixes class MixQuery(BaseModel): mixid: str = Field(description="The mix id") + sourcehash: str = Field(description="The sourcehash of the mix") @api.get("/") @@ -58,8 +66,45 @@ def get_mix(query: MixQuery): case _: return {"msg": "Invalid mix ID"}, 400 - mix = HomepageStore.get_mix(mixtype, query.mixid[1:]) - if mix: + # INFO: Check if the mix is already in the homepage store + mix = HomepageStore.get_mix(mixtype, query.mixid) + if mix and mix["sourcehash"] == query.sourcehash: return mix, 200 - return {"msg": "Mix not found"}, 404 + # INF0: Get the mix from the db + mix = MixTable.get_by_sourcehash(query.sourcehash) + + if not mix: + return {"msg": "Mix not found"}, 404 + + if mixtype == "custom_mixes": + plugin = MixesPlugin() + mix = plugin.get_track_mix(mix) + + if not mix: + return {"msg": "Mix not found"}, 404 + + return mix.to_full_dict(), 200 + + +class SaveMixRequest(BaseModel): + mixid: str = Field(description="The id of the mix") + type: str = Field(description="The type of mix") + sourcehash: str = Field(description="The sourcehash of the mix") + + +@api.post("/save") +def save_mix(body: SaveMixRequest): + mix_type = body.type + mix_sourcehash = body.sourcehash + + if mix_type == "artist": + state = MixTable.save_artist_mix(mix_sourcehash) + elif mix_type == "track": + state = MixTable.save_track_mix(mix_sourcehash) + + mix = HomepageStore.find_mix(body.mixid) + + if mix: + mix.saved = state + return {"msg": "Mixes saved"}, 200 diff --git a/app/db/userdata.py b/app/db/userdata.py index 41d34226..b1b80afe 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -486,15 +486,22 @@ class MixTable(Base): description: Mapped[str] = mapped_column(String()) timestamp: Mapped[int] = mapped_column(Integer()) sourcehash: Mapped[str] = mapped_column(String(), unique=True, index=True) - tracks: Mapped[list[str]] = mapped_column(JSON(), default_factory=list) + userid: Mapped[int] = mapped_column( + Integer(), ForeignKey("user.id", ondelete="cascade"), index=True + ) saved: Mapped[bool] = mapped_column(Boolean(), default=False) + tracks: Mapped[list[str]] = mapped_column(JSON(), default_factory=list) extra: Mapped[dict[str, Any]] = mapped_column( JSON(), nullable=True, default_factory=dict ) @classmethod - def get_all(cls): - result = cls.execute(select(cls)) + def get_all(cls, with_userid: bool = False): + if with_userid: + result = cls.execute(select(cls).where(cls.userid == get_current_userid())) + else: + result = cls.execute(select(cls)) + return Mix.mixes_to_dataclasses(result.fetchall()) @classmethod @@ -505,6 +512,13 @@ class MixTable(Base): if res: return Mix.mix_to_dataclass(res) + @classmethod + def get_by_mixid(cls, mixid: str): + result = cls.execute(select(cls).where(cls.mixid == mixid)) + res = result.fetchone() + if res: + return Mix.mix_to_dataclass(res) + @classmethod def insert_one(cls, mix: Mix): mixdict = asdict(mix) @@ -512,3 +526,62 @@ class MixTable(Base): del mixdict["id"] return cls.execute(insert(cls).values(mixdict), commit=True) + + @classmethod + def update_one(cls, mixid: str, mix: Mix): + mixdict = asdict(mix) + mixdict["mixid"] = mix.id + del mixdict["id"] + + return cls.execute( + update(cls) + .where( + and_( + cls.mixid == mixid, + cls.sourcehash == mix.sourcehash, + cls.userid == get_current_userid(), + ) + ) + .values(mixdict), + commit=True, + ) + + @classmethod + def save_artist_mix(cls, sourcehash: str): + """ + Toggles the saved status of an artist mix. + """ + + mix = cls.get_by_sourcehash(sourcehash) + + if not mix: + return False + + mix.saved = not mix.saved + cls.update_one(mix.id, mix) + + return mix.saved + + @classmethod + def get_saved_track_mixes(cls): + """ + Return all mixes that have the extra.trackmix_saved set to True. + """ + + result = cls.execute(select(cls).where(cls.extra.c.trackmix_saved == True)) + return Mix.mixes_to_dataclasses(result.fetchall()) + + @classmethod + def save_track_mix(cls, sourcehash: str): + """ + Toggles the property extra.trackmix_saved to True. + """ + + mix = cls.get_by_sourcehash(sourcehash) + if not mix: + return False + + mix.extra["trackmix_saved"] = not mix.extra["trackmix_saved"] + cls.update_one(mix.id, mix) + + return mix.extra["trackmix_saved"] diff --git a/app/lib/home/__init__.py b/app/lib/home/__init__.py new file mode 100644 index 00000000..a762a66d --- /dev/null +++ b/app/lib/home/__init__.py @@ -0,0 +1,303 @@ +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.folder import FolderStore +from app.store.tracks import TrackStore + +from app.models.logger import TrackLog +from app.lib.playlistlib import get_first_4_images +from app.utils.dates import timestamp_to_time_passed +from app.lib.home.recentlyadded import get_recently_added_playlist +from app.db.userdata import FavoritesTable, MixTable, PlaylistTable +from app.lib.home.recentlyplayed import get_recently_played_playlist + +from app.serializers.track import serialize_track +from app.serializers.album import album_serializer +from app.serializers.artist import serialize_for_card +from app.serializers.playlist import serialize_for_card as serialize_playlist + + +def create_items(entries: list[TrackLog], limit: int): + custom_playlists = [ + {"name": "recentlyadded", "handler": get_recently_added_playlist}, + {"name": "recentlyplayed", "handler": get_recently_played_playlist}, + ] + + items = [] + added = set() + + for entry in entries: + if len(items) >= limit: + break + + if entry.source in added: + continue + + added.add(entry.source) + + if entry.type == "mix": + if not entry.type_src: + continue + + # INFO: Get mix from homepage store + from app.store.homepage import HomepageStore + mix = HomepageStore.find_mix(entry.type_src) + + if not mix and entry.type_src.startswith("t"): + # mix is a track mix (not saved in the db) + continue + + if not mix: + # INFO: Get mix from db + mix = MixTable.get_by_mixid(entry.type_src) + + if not mix: + continue + + items.append( + { + "type": "mix", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + ) + continue + + if entry.type == "album": + album = AlbumStore.albummap.get(entry.type_src) + + if album is None: + continue + + item = { + "type": "album", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "artist": + artist = ArtistStore.artistmap.get(entry.type_src) + + if artist is None: + continue + + items.append( + { + "type": "artist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + ) + + continue + + if entry.type == "folder": + folder = entry.type_src + + if not folder: + continue + + if not folder.endswith("/"): + folder += "/" + + is_home_dir = entry.type_src == "$home" + + if is_home_dir: + folder = os.path.expanduser("~") + + item = { + "type": "folder", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "playlist": + is_custom = entry.type_src in [i["name"] for i in custom_playlists] + + if is_custom: + items.append( + { + "type": "playlist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + "is_custom": True, + } + ) + continue + + playlist = PlaylistTable.get_by_id(entry.type_src) + if playlist is None: + continue + + item = { + "type": "playlist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "favorite": + items.append( + { + "type": "favorite", + "timestamp": entry.timestamp, + } + ) + continue + + t = TrackStore.trackhashmap.get(entry.trackhash) + + if t is None: + continue + + item = { + "type": "track", + "hash": entry.trackhash, + "timestamp": entry.timestamp, + } + items.append(item) + + return items + + +def recover_items(items: list[dict]): + custom_playlists = [ + {"name": "recentlyadded", "handler": get_recently_added_playlist}, + {"name": "recentlyplayed", "handler": get_recently_played_playlist}, + ] + recovered = [] + + for item in items: + recovered_item = None + + if item["type"] == "album": + album = AlbumStore.get_album_by_hash(item["hash"]) + if album is None: + continue + + album = album_serializer( + album, + to_remove={ + "genres", + "date", + "count", + "duration", + "albumartists_hashes", + "og_title", + }, + ) + + recovered_item = { + "type": "album", + "item": album, + } + elif item["type"] == "artist": + artist = ArtistStore.get_artist_by_hash(item["hash"]) + if artist is None: + continue + + recovered_item = { + "type": "artist", + "item": serialize_for_card(artist), + } + elif item["type"] == "folder": + count = FolderStore.count_tracks_containing_paths([item["hash"]]) + + recovered_item = { + "type": "folder", + "item": { + "path": item["hash"], + "count": count[0]["trackcount"], + }, + } + elif item["type"] == "playlist": + if item.get("is_custom"): + playlist, _ = next( + i["handler"]() + for i in custom_playlists + if i["name"] == item["hash"] + ) + playlist.images = [i["image"] for i in playlist.images] + + playlist = serialize_playlist( + playlist, to_remove={"settings", "duration"} + ) + recovered_item = { + "type": "playlist", + "item": playlist, + } + else: + playlist = PlaylistTable.get_by_id(item["hash"]) + if playlist is None: + continue + + tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) + playlist.clear_lists() + + if not playlist.has_image: + images = get_first_4_images(tracks) + images = [i["image"] for i in images] + playlist.images = images + + recovered_item = { + "type": "playlist", + "item": serialize_playlist(playlist), + } + elif item["type"] == "favorite": + recovered_item = { + "type": "favorite", + "item": { + "count": FavoritesTable.count(), + }, + } + elif item["type"] == "track": + track = TrackStore.trackhashmap.get(item["hash"]) + if track is None: + continue + + recovered_item = { + "type": "track", + "item": serialize_track(track.get_best()), + } + + elif item["type"] == "mix": + from app.store.homepage import HomepageStore + mix = HomepageStore.find_mix(item["hash"]) + if mix is None: + mix = MixTable.get_by_mixid(item["hash"]) + + if mix is None: + continue + + mix = mix.to_dict() + + recovered_item = { + "type": "mix", + "item": mix, + } + + if recovered_item is not None: + helptext = item.get("help_text") or item.get("type") + secondary_text = item.get("secondary_text") + + if "secondary_text" in item: + secondary_text = item["secondary_text"] + elif "timestamp" in item: + secondary_text = timestamp_to_time_passed(item["timestamp"]) + + if helptext: + recovered_item["item"]["help_text"] = helptext + + if secondary_text: + recovered_item["item"]["time"] = secondary_text + + recovered.append(recovered_item) + + return recovered diff --git a/app/lib/home/create_items.py b/app/lib/home/create_items.py new file mode 100644 index 00000000..d10d134d --- /dev/null +++ b/app/lib/home/create_items.py @@ -0,0 +1,158 @@ +from app.db.userdata import MixTable, PlaylistTable +from app.lib.home.recentlyadded import get_recently_added_playlist +from app.lib.home.recentlyplayed import get_recently_played_playlist +from app.models.logger import TrackLog +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.homepage import HomepageStore +from app.store.tracks import TrackStore + + +def create_items(entries: list[TrackLog], limit: int): + custom_playlists = [ + {"name": "recentlyadded", "handler": get_recently_added_playlist}, + {"name": "recentlyplayed", "handler": get_recently_played_playlist}, + ] + + items = [] + added = set() + + for entry in entries: + if len(items) >= limit: + break + + if entry.source in added: + continue + + added.add(entry.source) + + if entry.type == "mix": + if not entry.type_src: + continue + + # INFO: Get mix from homepage store + mix = HomepageStore.find_mix(entry.type_src) + + if not mix and entry.type_src.startswith("t"): + # mix is a track mix (not saved in the db) + continue + + if not mix: + # INFO: Get mix from db + mix = MixTable.get_by_mixid(entry.type_src) + + if not mix: + continue + + items.append( + { + "type": "mix", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + ) + continue + + if entry.type == "album": + album = AlbumStore.albummap.get(entry.type_src) + + if album is None: + continue + + item = { + "type": "album", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "artist": + artist = ArtistStore.artistmap.get(entry.type_src) + + if artist is None: + continue + + items.append( + { + "type": "artist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + ) + + continue + + if entry.type == "folder": + folder = entry.type_src + + if not folder: + continue + + if not folder.endswith("/"): + folder += "/" + + is_home_dir = entry.type_src == "$home" + + if is_home_dir: + folder = os.path.expanduser("~") + + item = { + "type": "folder", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "playlist": + is_custom = entry.type_src in [i["name"] for i in custom_playlists] + + if is_custom: + items.append( + { + "type": "playlist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + "is_custom": True, + } + ) + continue + + playlist = PlaylistTable.get_by_id(entry.type_src) + if playlist is None: + continue + + item = { + "type": "playlist", + "hash": entry.type_src, + "timestamp": entry.timestamp, + } + + items.append(item) + continue + + if entry.type == "favorite": + items.append( + { + "type": "favorite", + "timestamp": entry.timestamp, + } + ) + continue + + t = TrackStore.trackhashmap.get(entry.trackhash) + + if t is None: + continue + + item = { + "type": "track", + "hash": entry.trackhash, + "timestamp": entry.timestamp, + } + items.append(item) + + return items \ No newline at end of file diff --git a/app/lib/home/get_recently_played.py b/app/lib/home/get_recently_played.py new file mode 100644 index 00000000..79bde7d4 --- /dev/null +++ b/app/lib/home/get_recently_played.py @@ -0,0 +1,47 @@ +from app.db.userdata import ScrobbleTable +from app.lib.home.create_items import create_items +from app.models.logger import TrackLog + + +def get_recently_played( + limit: int, userid: int | None = None, _entries: list[TrackLog] = [] +): + """ + Get the recently played items for the homepage. + + Pass a list of track log entries to use a subset of the scrobble table. + """ + # TODO: Paginate this + items = [] + + BATCH_SIZE = 200 + current_index = 0 + + if len(_entries): + entries = _entries + limit = 1 + else: + entries = ScrobbleTable.get_all(0, BATCH_SIZE, userid=userid) + + max_iterations = 20 + iterations = 0 + + while len(items) < limit and iterations < max_iterations: + items.extend(create_items(entries, limit)) + current_index += BATCH_SIZE + + if len(items) < limit: + entries = ScrobbleTable.get_all( + start=current_index + 1, limit=BATCH_SIZE, userid=userid + ) + if not entries: + break + + iterations += 1 + + if iterations == max_iterations: + print( + f"Warning: Reached maximum iterations ({max_iterations}) while fetching recently played items" + ) + + return items \ No newline at end of file diff --git a/app/lib/home/recentlyplayed.py b/app/lib/home/recentlyplayed.py index a565ab80..bde6816d 100644 --- a/app/lib/home/recentlyplayed.py +++ b/app/lib/home/recentlyplayed.py @@ -1,307 +1,13 @@ from datetime import datetime -import os -from app.db.userdata import FavoritesTable, PlaylistTable, ScrobbleTable -from app.models.logger import TrackLog from app.models.playlist import Playlist -from app.serializers.track import serialize_track -from app.serializers.album import album_serializer from app.lib.playlistlib import get_first_4_images -from app.store.folder import FolderStore from app.utils.dates import ( create_new_date, date_string_to_time_passed, - timestamp_to_time_passed, ) -from app.serializers.artist import serialize_for_card -from app.serializers.playlist import serialize_for_card as serialize_playlist -from app.lib.home.recentlyadded import get_recently_added_playlist -from app.store.albums import AlbumStore from app.store.tracks import TrackStore -from app.store.artists import ArtistStore - - -def get_recently_played( - limit: int, userid: int | None = None, _entries: list[TrackLog] = [] -): - """ - Get the recently played items for the homepage. - - Pass a list of track log entries to use a subset of the scrobble table. - """ - # TODO: Paginate this - items = [] - added = set() - - custom_playlists = [ - {"name": "recentlyadded", "handler": get_recently_added_playlist}, - {"name": "recentlyplayed", "handler": get_recently_played_playlist}, - ] - - def create_items(entries: list[TrackLog]): - for entry in entries: - if len(items) >= limit: - break - - if entry.source in added: - continue - - added.add(entry.source) - - if entry.type == "album": - album = AlbumStore.albummap.get(entry.type_src) - - if album is None: - continue - - item = { - "type": "album", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "artist": - artist = ArtistStore.artistmap.get(entry.type_src) - - if artist is None: - continue - - items.append( - { - "type": "artist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - ) - - continue - - if entry.type == "folder": - folder = entry.type_src - - if not folder: - continue - - if not folder.endswith("/"): - folder += "/" - - is_home_dir = entry.type_src == "$home" - - if is_home_dir: - folder = os.path.expanduser("~") - - item = { - "type": "folder", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "playlist": - is_custom = entry.type_src in [i["name"] for i in custom_playlists] - - if is_custom: - items.append( - { - "type": "playlist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - "is_custom": True, - } - ) - continue - - playlist = PlaylistTable.get_by_id(entry.type_src) - if playlist is None: - continue - - item = { - "type": "playlist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "favorite": - items.append( - { - "type": "favorite", - "timestamp": entry.timestamp, - } - ) - continue - - t = TrackStore.trackhashmap.get(entry.trackhash) - - if t is None: - continue - - item = { - "type": "track", - "hash": entry.trackhash, - "timestamp": entry.timestamp, - } - items.append(item) - - BATCH_SIZE = 200 - current_index = 0 - - if len(_entries): - entries = _entries - limit = 1 - else: - entries = ScrobbleTable.get_all(0, BATCH_SIZE, userid=userid) - - max_iterations = 20 - iterations = 0 - - while len(items) < limit and iterations < max_iterations: - create_items(entries) - current_index += BATCH_SIZE - - if len(items) < limit: - entries = ScrobbleTable.get_all( - start=current_index + 1, limit=BATCH_SIZE, userid=userid - ) - if not entries: - break - - iterations += 1 - - if iterations == max_iterations: - print( - f"Warning: Reached maximum iterations ({max_iterations}) while fetching recently played items" - ) - - return items - - -def recover_items(items: list[dict]): - custom_playlists = [ - {"name": "recentlyadded", "handler": get_recently_added_playlist}, - {"name": "recentlyplayed", "handler": get_recently_played_playlist}, - ] - recovered = [] - - for item in items: - recovered_item = None - - if item["type"] == "album": - album = AlbumStore.get_album_by_hash(item["hash"]) - if album is None: - continue - - album = album_serializer( - album, - to_remove={ - "genres", - "date", - "count", - "duration", - "albumartists_hashes", - "og_title", - }, - ) - - recovered_item = { - "type": "album", - "item": album, - } - elif item["type"] == "artist": - artist = ArtistStore.get_artist_by_hash(item["hash"]) - if artist is None: - continue - - recovered_item = { - "type": "artist", - "item": serialize_for_card(artist), - } - elif item["type"] == "folder": - count = FolderStore.count_tracks_containing_paths([item["hash"]]) - - recovered_item = { - "type": "folder", - "item": { - "path": item["hash"], - "count": count[0]["trackcount"], - }, - } - elif item["type"] == "playlist": - if item.get("is_custom"): - playlist, _ = next( - i["handler"]() - for i in custom_playlists - if i["name"] == item["hash"] - ) - playlist.images = [i["image"] for i in playlist.images] - - playlist = serialize_playlist( - playlist, to_remove={"settings", "duration"} - ) - recovered_item = { - "type": "playlist", - "item": playlist, - } - else: - playlist = PlaylistTable.get_by_id(item["hash"]) - if playlist is None: - continue - - tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) - playlist.clear_lists() - - if not playlist.has_image: - images = get_first_4_images(tracks) - images = [i["image"] for i in images] - playlist.images = images - - recovered_item = { - "type": "playlist", - "item": serialize_playlist(playlist), - } - elif item["type"] == "favorite": - recovered_item = { - "type": "favorite", - "item": { - "count": FavoritesTable.count(), - }, - } - elif item["type"] == "track": - track = TrackStore.trackhashmap.get(item["hash"]) - if track is None: - continue - - recovered_item = { - "type": "track", - "item": serialize_track(track.get_best()), - } - - if recovered_item is not None: - helptext = item.get("help_text") or item.get("type") - secondary_text = item.get("secondary_text") - - if "secondary_text" in item: - secondary_text = item["secondary_text"] - elif "timestamp" in item: - secondary_text = timestamp_to_time_passed(item["timestamp"]) - - if helptext: - recovered_item["item"]["help_text"] = helptext - - if secondary_text: - recovered_item["item"]["time"] = secondary_text - - recovered.append(recovered_item) - - return recovered def get_recently_played_playlist(limit: int = 100): diff --git a/app/lib/recipes/artistmixes.py b/app/lib/recipes/artistmixes.py index 91a9af81..3ee9d6a1 100644 --- a/app/lib/recipes/artistmixes.py +++ b/app/lib/recipes/artistmixes.py @@ -25,14 +25,11 @@ class ArtistMixes(HomepageRoutine): custom_mixes = [] for _mix in mixes: - custom_mix = mix.get_custom_mix_items(_mix) + custom_mix = mix.get_track_mix(_mix) if custom_mix: custom_mixes.append(custom_mix) - for index, custom_mix in enumerate(custom_mixes): - custom_mix.title = f"Mix {index + 1}" - HomepageStore.set_mixes( custom_mixes, entrykey="custom_mixes", userid=user.id ) diff --git a/app/lib/recipes/recents.py b/app/lib/recipes/recents.py index 12ff558d..a9123d3c 100644 --- a/app/lib/recipes/recents.py +++ b/app/lib/recipes/recents.py @@ -1,7 +1,7 @@ import pprint from app.db.userdata import ScrobbleTable, UserTable from app.lib.home.recentlyadded import get_recently_added_items -from app.lib.home.recentlyplayed import get_recently_played +from app.lib.home.get_recently_played import get_recently_played from app.lib.recipes import HomepageRoutine from app.store.homepage import HomepageStore diff --git a/app/models/logger.py b/app/models/logger.py index e3668ea9..5d67e382 100644 --- a/app/models/logger.py +++ b/app/models/logger.py @@ -27,10 +27,11 @@ class TrackLog: def __post_init__(self): prefix_map = { + "mix:": "mix", "al:": "album", "ar:": "artist", - "pl:": "playlist", "fo:": "folder", + "pl:": "playlist", "favorite": "favorite", } diff --git a/app/models/mix.py b/app/models/mix.py index 84fe41d5..097788df 100644 --- a/app/models/mix.py +++ b/app/models/mix.py @@ -5,7 +5,8 @@ from typing import Any from app.lib.playlistlib import get_first_4_images from app.serializers.track import serialize_tracks from app.store.tracks import TrackStore -from app.utils.dates import seconds_to_time_string +from app.utils.dates import seconds_to_time_string, timestamp_to_time_passed +from app.utils.hashing import create_hash @dataclass @@ -15,6 +16,7 @@ class Mix: description: str tracks: list[str] sourcehash: str + userid: int """ A hash of the tracks used to generate the mix. """ @@ -41,8 +43,13 @@ class Mix: return _dict - def to_dict(self): + def to_dict(self, convert_timestamp: bool = False): item = asdict(self) + item["trackshash"] = create_hash(*self.tracks[:40]) + + if convert_timestamp: + item["time"] = timestamp_to_time_passed(item["timestamp"]) + del item["tracks"] del item["extra"]["albums"] diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 83259890..5247f0e0 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -32,7 +32,7 @@ class MixAlreadyExists(Exception): class MixesPlugin(Plugin): MAX_TRACKS_TO_FETCH = 5 MIN_TRACK_MIX_LENGTH = 15 - MIX_TRACKS = 40 + MIX_TRACKS_LENGTH = 40 MIN_DAY_LISTEN_DURATION = 3 * 60 # 3 minutes MIN_WEEK_LISTEN_DURATION = 10 * 60 # 10 minutes @@ -65,7 +65,7 @@ class MixesPlugin(Plugin): return False @plugin_method - def get_track_mix(self, tracks: list[Track], with_help: bool = False): + def get_track_mix_data(self, tracks: list[Track], with_help: bool = False): """ Given a list of tracks, creates a mix by fetching data from the Swing Music Cloud recommendation server. @@ -137,25 +137,25 @@ class MixesPlugin(Plugin): return trackmatches, results["albums"], results["artists"] @plugin_method - def get_artist_mix(self, artisthash: str): - """ - Given an artisthash, creates an artist mix using the - self.MAX_TRACKS_TO_FETCH most listened to tracks. + # def get_artist_mix(self, artisthash: str): + # """ + # Given an artisthash, creates an artist mix using the + # self.MAX_TRACKS_TO_FETCH most listened to tracks. - Returns a tuple of the mix and the sourcehash. - """ - artist = ArtistStore.artistmap[artisthash] - tracks = TrackStore.get_tracks_by_trackhashes(artist.trackhashes) + # Returns a tuple of the mix and the sourcehash. + # """ + # artist = ArtistStore.artistmap[artisthash] + # tracks = TrackStore.get_tracks_by_trackhashes(artist.trackhashes) - tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) - sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] - sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) + # tracks = sorted(tracks, key=lambda x: x.playduration, reverse=True) + # sourcetracks = tracks[: self.MAX_TRACKS_TO_FETCH] + # sourcehash = create_hash(*[t.trackhash for t in sourcetracks]) - if MixTable.get_by_sourcehash(sourcehash): - raise MixAlreadyExists() + # if MixTable.get_by_sourcehash(sourcehash): + # raise MixAlreadyExists() - tracks, albums, artists = self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) - return (tracks, albums, artists, sourcehash) + # tracks, albums, artists = self.get_track_mix(tracks[: self.MAX_TRACKS_TO_FETCH]) + # return (tracks, albums, artists, sourcehash) @plugin_method def create_artist_mixes(self, userid: int): @@ -225,7 +225,7 @@ class MixesPlugin(Plugin): ) mix = self.create_artist_mix( - artist, trackhashes[: self.MAX_TRACKS_TO_FETCH] + artist, trackhashes[: self.MAX_TRACKS_TO_FETCH], userid=userid ) if mix: @@ -262,7 +262,9 @@ class MixesPlugin(Plugin): return f"Featuring {tracks[0].artists[0]['name']}" - def create_artist_mix(self, artist: dict[str, str], trackhashes: list[str]): + def create_artist_mix( + self, artist: dict[str, str], trackhashes: list[str], userid: int + ): """ Given an artist dict, creates an artist mix. """ @@ -286,7 +288,7 @@ class MixesPlugin(Plugin): print(db_mix.title) return db_mix - mix_tracks, albums, artists = self.get_track_mix(tracks) + mix_tracks, albums, artists = self.get_track_mix_data(tracks) if len(mix_tracks) < self.MIN_TRACK_MIX_LENGTH: return None @@ -300,11 +302,12 @@ class MixesPlugin(Plugin): mix = Mix( # the a prefix indicates that this is an artist mix - id=f"a{artist['artisthash']}", + id=f"a{userid}{artist['artisthash']}", title=artist["artist"] + " Radio", description=self.get_mix_description(mix_tracks, artist["artisthash"]), tracks=[t.trackhash for t in mix_tracks], sourcehash=sourcehash, + userid=userid, extra={ "type": "artist", "artisthash": artist["artisthash"], @@ -435,31 +438,47 @@ class MixesPlugin(Plugin): """ pass - def get_custom_mix_items(self, mix: Mix): + def get_track_mix(self, mix: Mix): """ Given a mix, returns the excess tracks as a custom mix. """ # INFO: If the mix can't have more than 20 tracks, return None - if len(mix.tracks) <= self.MIX_TRACKS + 20: + if len(mix.tracks) <= self.MIX_TRACKS_LENGTH + 20: return None - tracks = TrackStore.get_tracks_by_trackhashes(mix.tracks[self.MIX_TRACKS :]) + og_track = TrackStore.trackhashmap.get(mix.tracks[0]) - return Mix( - id=f"t{mix.extra['artisthash']}", - title="", # INFO: Will be filled after all mixes are created. + if not og_track: + return None + + og_track = og_track.get_best() + tracks = [og_track] + TrackStore.get_tracks_by_trackhashes( + mix.tracks[self.MIX_TRACKS_LENGTH :] + ) + + trackmix = Mix( + id=f"t{mix.userid}{mix.extra['artisthash']}", + title=og_track.title, description=self.get_mix_description(tracks, mix.extra["artisthash"]), tracks=[t.trackhash for t in tracks], sourcehash=create_hash(*[t.trackhash for t in tracks]), + userid=mix.userid, extra={ "type": "track", + "og_sourcehash": mix.sourcehash, "images": self.get_custom_mix_images(tracks), "artists": None, "albums": None, }, ) + # INFO: Write track mix save state + if mix.extra.get("trackmix_saved"): + trackmix.saved = True + + return trackmix + def get_custom_mix_images(self, tracks: list[Track]): first_album = tracks[0].albumhash @@ -478,12 +497,17 @@ class MixesPlugin(Plugin): if artisthash in seen: continue + artist = ArtistStore.artistmap.get(artisthash) + + if not artist: + continue + seen.add(artisthash) image = { "image": artisthash + ".webp", "type": "artist", - "color": ArtistStore.get_artist_by_hash(artisthash).color, + "color": artist.artist.color, } images.append(image) diff --git a/app/store/homepage.py b/app/store/homepage.py index 289c41c9..07a8cf80 100644 --- a/app/store/homepage.py +++ b/app/store/homepage.py @@ -1,123 +1,16 @@ -from abc import ABC from typing import Any -from app.lib.home.recentlyplayed import recover_items -from app.models.mix import Mix + +from app.store.homepageentries import ( + BecauseYouListenedToArtistHomepageEntry, + GenericRecoverableEntry, + HomepageEntry, + MixHomepageEntry, + RecentlyAddedHomepageEntry, + RecentlyPlayedHomepageEntry, +) from app.utils.auth import get_current_userid -class HomepageEntry(ABC): - """ - Base class for all homepage entries. - - items is a dict of userid to a dict of stuff. - """ - - title: str - description: str - items: dict[int, Any] - - def __init__(self, title: str, description: str): - self.title = title - self.description = description - - def get_items(self, userid: int, limit: int | None = None): - """ - Return usable items for the homepage. - """ - ... - - -class MixHomepageEntry(HomepageEntry): - """ - A homepage entry for mixes. - self.items is a dict of userid to a dict of mixid to mix. - """ - - items: dict[int, dict[str, Mix]] - - def __init__(self, title: str, description: str): - super().__init__(title, description) - self.items = {} - - def get_items(self, userid: int, limit: int | None = None): - items = [] - - for mix in self.items.get(userid, {}).values(): - if limit and len(items) >= limit: - break - - items.append( - { - "type": "mix", - "item": mix.to_dict(), - } - ) - - return { - "title": self.title, - "description": self.description, - "items": items, - } - - -class RecentlyPlayedHomepageEntry(HomepageEntry): - """ - A homepage entry for recently played. - """ - - items: dict[int, list[dict[str, Any]]] - - def __init__(self, title: str, description: str = ""): - super().__init__(title, description) - self.items = {} - - def get_items(self, userid: int, limit: int | None = None): - items = self.items.get(userid, [])[:limit] - - return { - "title": self.title, - "description": self.description, - "items": recover_items(items), - } - - -class RecentlyAddedHomepageEntry(RecentlyPlayedHomepageEntry): - """ - A homepage entry for recently added. - """ - - def get_items(self, userid: int, limit: int | None = None): - return super().get_items(0, limit) - - -class GenericRecoverableEntry(RecentlyPlayedHomepageEntry): - """ - A homepage entry for top streamed. - """ - - # NOTE: This extends RecentlyPlayedHomepageEntry because - # the shape of the data is the same. - pass - - -class BecauseYouListenedToArtistHomepageEntry(RecentlyPlayedHomepageEntry): - """ - A homepage entry for because you listened to artist. - """ - - # SHAPE: {userid: {title: str, items: list[RecoverableItem]}} - items: dict[int, dict[str, Any]] - - def get_items(self, userid: int, limit: int | None = None): - title = self.items.get(userid, {}).get("title") - items = self.items.get(userid, {}).get("items", [])[:limit] - - return { - "title": title, - "items": recover_items(items), - } - - class HomepageStore: """ Stores the homepage items. @@ -159,7 +52,7 @@ class HomepageStore: @classmethod def set_mixes(cls, items: list[Any], entrykey: str, userid: int | None = None): - idmap = {item.id[1:]: item for item in items} + idmap = {item.id: item for item in items} cls.entries[entrykey].items[userid or get_current_userid()] = idmap @classmethod @@ -175,3 +68,14 @@ class HomepageStore: for entry in cls.entries.keys() if len(cls.entries[entry].items) ] + + @classmethod + def find_mix(cls, mixid: str): + mixentries = ["artist_mixes", "custom_mixes"] + + for entry in mixentries: + mix = cls.entries[entry].items.get(get_current_userid(), {}).get(mixid) + if mix: + return mix + + return None diff --git a/app/store/homepageentries.py b/app/store/homepageentries.py new file mode 100644 index 00000000..7343a884 --- /dev/null +++ b/app/store/homepageentries.py @@ -0,0 +1,118 @@ +from abc import ABC +from typing import Any + +from app.lib.home import recover_items +from app.models.mix import Mix + +class HomepageEntry(ABC): + """ + Base class for all homepage entries. + + items is a dict of userid to a dict of stuff. + """ + + title: str + description: str + items: dict[int, Any] + + def __init__(self, title: str, description: str): + self.title = title + self.description = description + + def get_items(self, userid: int, limit: int | None = None): + """ + Return usable items for the homepage. + """ + ... + + +class MixHomepageEntry(HomepageEntry): + """ + A homepage entry for mixes. + self.items is a dict of userid to a dict of mixid to mix. + """ + + items: dict[int, dict[str, Mix]] + + def __init__(self, title: str, description: str): + super().__init__(title, description) + self.items = {} + + def get_items(self, userid: int, limit: int | None = None): + items = [] + + for mix in self.items.get(userid, {}).values(): + if limit and len(items) >= limit: + break + + items.append( + { + "type": "mix", + "item": mix.to_dict(), + } + ) + + return { + "title": self.title, + "description": self.description, + "items": items, + } + + +class RecentlyPlayedHomepageEntry(HomepageEntry): + """ + A homepage entry for recently played. + """ + + items: dict[int, list[dict[str, Any]]] + + def __init__(self, title: str, description: str = ""): + super().__init__(title, description) + self.items = {} + + def get_items(self, userid: int, limit: int | None = None): + items = self.items.get(userid, [])[:limit] + + return { + "title": self.title, + "description": self.description, + "items": recover_items(items), + } + + +class RecentlyAddedHomepageEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for recently added. + """ + + def get_items(self, userid: int, limit: int | None = None): + return super().get_items(0, limit) + + +class GenericRecoverableEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for top streamed. + """ + + # NOTE: This extends RecentlyPlayedHomepageEntry because + # the shape of the data is the same. + pass + + +class BecauseYouListenedToArtistHomepageEntry(RecentlyPlayedHomepageEntry): + """ + A homepage entry for because you listened to artist. + """ + + # SHAPE: {userid: {title: str, items: list[RecoverableItem]}} + items: dict[int, dict[str, Any]] + + def get_items(self, userid: int, limit: int | None = None): + title = self.items.get(userid, {}).get("title") + items = self.items.get(userid, {}).get("items", [])[:limit] + + return { + "title": title, + "items": recover_items(items), + } + From 016211e419741c0b33edfc433231f2c655a21254 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 18:01:53 +0300 Subject: [PATCH 27/44] fix: keyerror on saving track mix --- app/db/userdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/db/userdata.py b/app/db/userdata.py index b1b80afe..c09ae9a9 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -581,7 +581,7 @@ class MixTable(Base): if not mix: return False - mix.extra["trackmix_saved"] = not mix.extra["trackmix_saved"] + mix.extra["trackmix_saved"] = not mix.extra.get("trackmix_saved", False) cls.update_one(mix.id, mix) return mix.extra["trackmix_saved"] From 24aa34807b165467c0a7d4231a7ecfdc9dcc5e67 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 20:03:36 +0300 Subject: [PATCH 28/44] increase mix count to 15 --- app/plugins/mixes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 5247f0e0..7bf316f3 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -172,26 +172,26 @@ class MixesPlugin(Plugin): artists = { "today": { - "max": 3, + "max": 4, "artists": get_artists_in_period(today_start, today_end, userid), "created": 0, }, "last_2_days": { - "max": 2, + "max": 3, "artists": get_artists_in_period( last_2_days_start, time.time(), userid ), "created": 0, }, "last_7_days": { - "max": 3, + "max": 4, "artists": get_artists_in_period( last_7_days_start, time.time(), userid ), "created": 0, }, "last_1_month": { - "max": 2, + "max": 4, "artists": get_artists_in_period( last_1_month_start, time.time(), userid ), From 94591daa1e68d5a2c60ded4e27abf736775daa26 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 20:55:59 +0300 Subject: [PATCH 29/44] support mix items not in store --- .gitignore | 1 + app/api/plugins/mixes.py | 3 +- app/api/scrobble/__init__.py | 2 + app/lib/home/__init__.py | 315 +++------------------------------ app/lib/home/create_items.py | 22 +-- app/lib/home/recover_items.py | 151 ++++++++++++++++ app/lib/recipes/artistmixes.py | 2 +- app/plugins/mixes.py | 23 +-- app/store/homepageentries.py | 2 +- 9 files changed, 202 insertions(+), 319 deletions(-) create mode 100644 app/lib/home/recover_items.py diff --git a/.gitignore b/.gitignore index b2412028..8b104bf5 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ logs.txt testdata.py test.py nohup.out +*s.json diff --git a/app/api/plugins/mixes.py b/app/api/plugins/mixes.py index 168b6a1c..3fe3f08b 100644 --- a/app/api/plugins/mixes.py +++ b/app/api/plugins/mixes.py @@ -78,8 +78,7 @@ def get_mix(query: MixQuery): return {"msg": "Mix not found"}, 404 if mixtype == "custom_mixes": - plugin = MixesPlugin() - mix = plugin.get_track_mix(mix) + mix = MixesPlugin.get_track_mix(mix) if not mix: return {"msg": "Mix not found"}, 404 diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index 834fe06f..212e7973 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -72,6 +72,8 @@ def log_track(body: LogTrackBody): return {"msg": "Track not found."}, 404 scrobble_data = dict(body) + # REVIEW: Do we need to store the extra info in the database? + # OR .... can we just write it to the backup file on demand? scrobble_data["extra"] = get_extra_info(body.trackhash, "track") ScrobbleTable.add(scrobble_data) diff --git a/app/lib/home/__init__.py b/app/lib/home/__init__.py index a762a66d..00446544 100644 --- a/app/lib/home/__init__.py +++ b/app/lib/home/__init__.py @@ -1,303 +1,30 @@ -from app.store.albums import AlbumStore -from app.store.artists import ArtistStore -from app.store.folder import FolderStore -from app.store.tracks import TrackStore - -from app.models.logger import TrackLog -from app.lib.playlistlib import get_first_4_images -from app.utils.dates import timestamp_to_time_passed -from app.lib.home.recentlyadded import get_recently_added_playlist -from app.db.userdata import FavoritesTable, MixTable, PlaylistTable -from app.lib.home.recentlyplayed import get_recently_played_playlist - -from app.serializers.track import serialize_track -from app.serializers.album import album_serializer -from app.serializers.artist import serialize_for_card -from app.serializers.playlist import serialize_for_card as serialize_playlist +from app.db.userdata import MixTable +from app.plugins.mixes import MixesPlugin -def create_items(entries: list[TrackLog], limit: int): - custom_playlists = [ - {"name": "recentlyadded", "handler": get_recently_added_playlist}, - {"name": "recentlyplayed", "handler": get_recently_played_playlist}, - ] +def find_mix(mixid: str, sourcehash: str): + """ + Find a mix in the homepage store or the db. + """ + from app.store.homepage import HomepageStore - items = [] - added = set() + mixtype = "custom_mixes" if mixid[0] == "t" else "artist_mixes" - for entry in entries: - if len(items) >= limit: - break + # INFO: Try getting the mix from the homepage store + mix = HomepageStore.get_mix(mixtype, mixid) + if mix and mix["sourcehash"] == sourcehash: + return mix - if entry.source in added: - continue + # INFO: Get the mix from the db + mix = MixTable.get_by_sourcehash(sourcehash) - added.add(entry.source) + if not mix: + return None - if entry.type == "mix": - if not entry.type_src: - continue + if mixtype == "custom_mixes": + mix = MixesPlugin.get_track_mix(mix) - # INFO: Get mix from homepage store - from app.store.homepage import HomepageStore - mix = HomepageStore.find_mix(entry.type_src) + if not mix: + return None - if not mix and entry.type_src.startswith("t"): - # mix is a track mix (not saved in the db) - continue - - if not mix: - # INFO: Get mix from db - mix = MixTable.get_by_mixid(entry.type_src) - - if not mix: - continue - - items.append( - { - "type": "mix", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - ) - continue - - if entry.type == "album": - album = AlbumStore.albummap.get(entry.type_src) - - if album is None: - continue - - item = { - "type": "album", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "artist": - artist = ArtistStore.artistmap.get(entry.type_src) - - if artist is None: - continue - - items.append( - { - "type": "artist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - ) - - continue - - if entry.type == "folder": - folder = entry.type_src - - if not folder: - continue - - if not folder.endswith("/"): - folder += "/" - - is_home_dir = entry.type_src == "$home" - - if is_home_dir: - folder = os.path.expanduser("~") - - item = { - "type": "folder", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "playlist": - is_custom = entry.type_src in [i["name"] for i in custom_playlists] - - if is_custom: - items.append( - { - "type": "playlist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - "is_custom": True, - } - ) - continue - - playlist = PlaylistTable.get_by_id(entry.type_src) - if playlist is None: - continue - - item = { - "type": "playlist", - "hash": entry.type_src, - "timestamp": entry.timestamp, - } - - items.append(item) - continue - - if entry.type == "favorite": - items.append( - { - "type": "favorite", - "timestamp": entry.timestamp, - } - ) - continue - - t = TrackStore.trackhashmap.get(entry.trackhash) - - if t is None: - continue - - item = { - "type": "track", - "hash": entry.trackhash, - "timestamp": entry.timestamp, - } - items.append(item) - - return items - - -def recover_items(items: list[dict]): - custom_playlists = [ - {"name": "recentlyadded", "handler": get_recently_added_playlist}, - {"name": "recentlyplayed", "handler": get_recently_played_playlist}, - ] - recovered = [] - - for item in items: - recovered_item = None - - if item["type"] == "album": - album = AlbumStore.get_album_by_hash(item["hash"]) - if album is None: - continue - - album = album_serializer( - album, - to_remove={ - "genres", - "date", - "count", - "duration", - "albumartists_hashes", - "og_title", - }, - ) - - recovered_item = { - "type": "album", - "item": album, - } - elif item["type"] == "artist": - artist = ArtistStore.get_artist_by_hash(item["hash"]) - if artist is None: - continue - - recovered_item = { - "type": "artist", - "item": serialize_for_card(artist), - } - elif item["type"] == "folder": - count = FolderStore.count_tracks_containing_paths([item["hash"]]) - - recovered_item = { - "type": "folder", - "item": { - "path": item["hash"], - "count": count[0]["trackcount"], - }, - } - elif item["type"] == "playlist": - if item.get("is_custom"): - playlist, _ = next( - i["handler"]() - for i in custom_playlists - if i["name"] == item["hash"] - ) - playlist.images = [i["image"] for i in playlist.images] - - playlist = serialize_playlist( - playlist, to_remove={"settings", "duration"} - ) - recovered_item = { - "type": "playlist", - "item": playlist, - } - else: - playlist = PlaylistTable.get_by_id(item["hash"]) - if playlist is None: - continue - - tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) - playlist.clear_lists() - - if not playlist.has_image: - images = get_first_4_images(tracks) - images = [i["image"] for i in images] - playlist.images = images - - recovered_item = { - "type": "playlist", - "item": serialize_playlist(playlist), - } - elif item["type"] == "favorite": - recovered_item = { - "type": "favorite", - "item": { - "count": FavoritesTable.count(), - }, - } - elif item["type"] == "track": - track = TrackStore.trackhashmap.get(item["hash"]) - if track is None: - continue - - recovered_item = { - "type": "track", - "item": serialize_track(track.get_best()), - } - - elif item["type"] == "mix": - from app.store.homepage import HomepageStore - mix = HomepageStore.find_mix(item["hash"]) - if mix is None: - mix = MixTable.get_by_mixid(item["hash"]) - - if mix is None: - continue - - mix = mix.to_dict() - - recovered_item = { - "type": "mix", - "item": mix, - } - - if recovered_item is not None: - helptext = item.get("help_text") or item.get("type") - secondary_text = item.get("secondary_text") - - if "secondary_text" in item: - secondary_text = item["secondary_text"] - elif "timestamp" in item: - secondary_text = timestamp_to_time_passed(item["timestamp"]) - - if helptext: - recovered_item["item"]["help_text"] = helptext - - if secondary_text: - recovered_item["item"]["time"] = secondary_text - - recovered.append(recovered_item) - - return recovered + return mix.to_dict() diff --git a/app/lib/home/create_items.py b/app/lib/home/create_items.py index d10d134d..a6492898 100644 --- a/app/lib/home/create_items.py +++ b/app/lib/home/create_items.py @@ -1,10 +1,11 @@ -from app.db.userdata import MixTable, PlaylistTable +import os +from app.db.userdata import PlaylistTable +from app.lib.home import find_mix from app.lib.home.recentlyadded import get_recently_added_playlist from app.lib.home.recentlyplayed import get_recently_played_playlist from app.models.logger import TrackLog from app.store.albums import AlbumStore from app.store.artists import ArtistStore -from app.store.homepage import HomepageStore from app.store.tracks import TrackStore @@ -30,17 +31,16 @@ def create_items(entries: list[TrackLog], limit: int): if not entry.type_src: continue - # INFO: Get mix from homepage store - mix = HomepageStore.find_mix(entry.type_src) + splits = entry.type_src.split(".") - if not mix and entry.type_src.startswith("t"): - # mix is a track mix (not saved in the db) + try: + mixid = splits[0] + sourcehash = splits[1] + except IndexError: continue - if not mix: - # INFO: Get mix from db - mix = MixTable.get_by_mixid(entry.type_src) - + # INFO: Get mix from homepage store + mix = find_mix(mixid, sourcehash) if not mix: continue @@ -155,4 +155,4 @@ def create_items(entries: list[TrackLog], limit: int): } items.append(item) - return items \ No newline at end of file + return items diff --git a/app/lib/home/recover_items.py b/app/lib/home/recover_items.py new file mode 100644 index 00000000..8e381482 --- /dev/null +++ b/app/lib/home/recover_items.py @@ -0,0 +1,151 @@ +from app.db.userdata import FavoritesTable, MixTable, PlaylistTable +from app.lib.home import find_mix +from app.lib.home.recentlyadded import get_recently_added_playlist +from app.lib.home.recentlyplayed import get_recently_played_playlist +from app.lib.playlistlib import get_first_4_images +from app.serializers.album import album_serializer +from app.serializers.artist import serialize_for_card +from app.serializers.playlist import serialize_for_card as serialize_playlist +from app.serializers.track import serialize_track +from app.store.albums import AlbumStore +from app.store.artists import ArtistStore +from app.store.folder import FolderStore +from app.store.tracks import TrackStore +from app.utils.dates import timestamp_to_time_passed + + +def recover_items(items: list[dict]): + custom_playlists = [ + {"name": "recentlyadded", "handler": get_recently_added_playlist}, + {"name": "recentlyplayed", "handler": get_recently_played_playlist}, + ] + recovered = [] + + for item in items: + recovered_item = None + + if item["type"] == "album": + album = AlbumStore.get_album_by_hash(item["hash"]) + if album is None: + continue + + album = album_serializer( + album, + to_remove={ + "genres", + "date", + "count", + "duration", + "albumartists_hashes", + "og_title", + }, + ) + + recovered_item = { + "type": "album", + "item": album, + } + elif item["type"] == "artist": + artist = ArtistStore.get_artist_by_hash(item["hash"]) + if artist is None: + continue + + recovered_item = { + "type": "artist", + "item": serialize_for_card(artist), + } + elif item["type"] == "folder": + count = FolderStore.count_tracks_containing_paths([item["hash"]]) + + recovered_item = { + "type": "folder", + "item": { + "path": item["hash"], + "count": count[0]["trackcount"], + }, + } + elif item["type"] == "playlist": + if item.get("is_custom"): + playlist, _ = next( + i["handler"]() + for i in custom_playlists + if i["name"] == item["hash"] + ) + playlist.images = [i["image"] for i in playlist.images] + + playlist = serialize_playlist( + playlist, to_remove={"settings", "duration"} + ) + recovered_item = { + "type": "playlist", + "item": playlist, + } + else: + playlist = PlaylistTable.get_by_id(item["hash"]) + if playlist is None: + continue + + tracks = TrackStore.get_tracks_by_trackhashes(playlist.trackhashes) + playlist.clear_lists() + + if not playlist.has_image: + images = get_first_4_images(tracks) + images = [i["image"] for i in images] + playlist.images = images + + recovered_item = { + "type": "playlist", + "item": serialize_playlist(playlist), + } + elif item["type"] == "favorite": + recovered_item = { + "type": "favorite", + "item": { + "count": FavoritesTable.count(), + }, + } + elif item["type"] == "track": + track = TrackStore.trackhashmap.get(item["hash"]) + if track is None: + continue + + recovered_item = { + "type": "track", + "item": serialize_track(track.get_best()), + } + + elif item["type"] == "mix": + try: + splits = item["hash"].split(".") + mixid = splits[0] + sourcehash = splits[1] + except IndexError: + continue + + mix = find_mix(mixid, sourcehash) + if mix is None: + continue + + recovered_item = { + "type": "mix", + "item": mix, + } + + if recovered_item is not None: + helptext = item.get("help_text") or item.get("type") + secondary_text = item.get("secondary_text") + + if "secondary_text" in item: + secondary_text = item["secondary_text"] + elif "timestamp" in item: + secondary_text = timestamp_to_time_passed(item["timestamp"]) + + if helptext: + recovered_item["item"]["help_text"] = helptext + + if secondary_text: + recovered_item["item"]["time"] = secondary_text + + recovered.append(recovered_item) + + return recovered diff --git a/app/lib/recipes/artistmixes.py b/app/lib/recipes/artistmixes.py index 3ee9d6a1..3de2bcdc 100644 --- a/app/lib/recipes/artistmixes.py +++ b/app/lib/recipes/artistmixes.py @@ -25,7 +25,7 @@ class ArtistMixes(HomepageRoutine): custom_mixes = [] for _mix in mixes: - custom_mix = mix.get_track_mix(_mix) + custom_mix = MixesPlugin.get_track_mix(_mix) if custom_mix: custom_mixes.append(custom_mix) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 7bf316f3..8691abc6 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -237,7 +237,8 @@ class MixesPlugin(Plugin): print([m.title for m in mixes]) return mixes - def get_mix_description(self, tracks: list[Track], artishash: str): + @classmethod + def get_mix_description(cls, tracks: list[Track], artishash: str): """ Constructs a description for a mix by putting together the first n=4 artists in the mix tracklist. @@ -434,17 +435,18 @@ class MixesPlugin(Plugin): The resulting mix is definitely expected to be of low quality. - TODO: Implement this! + TODO: Maybe implement this! """ pass - def get_track_mix(self, mix: Mix): + @classmethod + def get_track_mix(cls, mix: Mix): """ Given a mix, returns the excess tracks as a custom mix. """ # INFO: If the mix can't have more than 20 tracks, return None - if len(mix.tracks) <= self.MIX_TRACKS_LENGTH + 20: + if len(mix.tracks) <= cls.MIX_TRACKS_LENGTH + 20: return None og_track = TrackStore.trackhashmap.get(mix.tracks[0]) @@ -454,20 +456,20 @@ class MixesPlugin(Plugin): og_track = og_track.get_best() tracks = [og_track] + TrackStore.get_tracks_by_trackhashes( - mix.tracks[self.MIX_TRACKS_LENGTH :] + mix.tracks[cls.MIX_TRACKS_LENGTH :] ) trackmix = Mix( id=f"t{mix.userid}{mix.extra['artisthash']}", title=og_track.title, - description=self.get_mix_description(tracks, mix.extra["artisthash"]), + description=cls.get_mix_description(tracks, mix.extra["artisthash"]), tracks=[t.trackhash for t in tracks], sourcehash=create_hash(*[t.trackhash for t in tracks]), userid=mix.userid, extra={ "type": "track", "og_sourcehash": mix.sourcehash, - "images": self.get_custom_mix_images(tracks), + "images": cls.get_custom_mix_images(tracks), "artists": None, "albums": None, }, @@ -479,8 +481,8 @@ class MixesPlugin(Plugin): return trackmix - def get_custom_mix_images(self, tracks: list[Track]): - + @classmethod + def get_custom_mix_images(cls, tracks: list[Track]): first_album = tracks[0].albumhash first_img = { "image": first_album + ".webp", @@ -517,7 +519,8 @@ class MixesPlugin(Plugin): return images - def get_because_items(self, mixes: list[Mix]): + @staticmethod + def get_because_items(mixes: list[Mix]): """ Given a list of mixes, returns a list of artists that are similar to the artists in the mixes. diff --git a/app/store/homepageentries.py b/app/store/homepageentries.py index 7343a884..276ccd69 100644 --- a/app/store/homepageentries.py +++ b/app/store/homepageentries.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Any -from app.lib.home import recover_items +from app.lib.home.recover_items import recover_items from app.models.mix import Mix class HomepageEntry(ABC): From ffa4adccbd5c0beac6397d51a094ba0ea7a1ec61 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 21:39:31 +0300 Subject: [PATCH 30/44] qoute artist image --- app/plugins/mixes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 8691abc6..99f1bbf0 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -3,6 +3,7 @@ from io import BytesIO import json import random import time +from urllib.parse import quote import requests from PIL import Image @@ -40,8 +41,8 @@ class MixesPlugin(Plugin): def __init__(self): super().__init__("mixes", "Mixes") - self.server = "https://smcloud.mungaist.com" - # self.server = "http://localhost:1956" + # self.server = "https://smcloud.mungaist.com" + self.server = "http://localhost:1956" # server_online = self.ping_server() self.set_active(True) @@ -329,7 +330,7 @@ class MixesPlugin(Plugin): def download_artist_image(self, artist: Artist): try: res = requests.get( - f"{self.server}/mix/image?artist={artist.name}&type=Artist" + f"{self.server}/mix/image?artist={quote(artist.name)}&type=Artist" ) except requests.exceptions.ConnectionError: return None From 883c845d45d7a9e2a934e5a62f8b7fae53e02422 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Thu, 26 Dec 2024 21:41:41 +0300 Subject: [PATCH 31/44] uncomment local server --- app/plugins/mixes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 99f1bbf0..36924b5b 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -41,8 +41,8 @@ class MixesPlugin(Plugin): def __init__(self): super().__init__("mixes", "Mixes") - # self.server = "https://smcloud.mungaist.com" - self.server = "http://localhost:1956" + self.server = "https://smcloud.mungaist.com" + # self.server = "http://localhost:1956" # server_online = self.ping_server() self.set_active(True) From 872fdf26b488508535d5722e0c72f855ef32c736 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 27 Dec 2024 10:40:27 +0300 Subject: [PATCH 32/44] sort mixes by created date + write og mix date to track mix --- app/db/userdata.py | 8 ++++++-- app/plugins/mixes.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/db/userdata.py b/app/db/userdata.py index c09ae9a9..d8ca569f 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -498,9 +498,13 @@ class MixTable(Base): @classmethod def get_all(cls, with_userid: bool = False): if with_userid: - result = cls.execute(select(cls).where(cls.userid == get_current_userid())) + result = cls.execute( + select(cls) + .where(cls.userid == get_current_userid()) + .order_by(cls.timestamp.desc()) + ) else: - result = cls.execute(select(cls)) + result = cls.execute(select(cls).order_by(cls.timestamp.desc())) return Mix.mixes_to_dataclasses(result.fetchall()) diff --git a/app/plugins/mixes.py b/app/plugins/mixes.py index 36924b5b..999b8ad2 100644 --- a/app/plugins/mixes.py +++ b/app/plugins/mixes.py @@ -475,6 +475,7 @@ class MixesPlugin(Plugin): "albums": None, }, ) + trackmix.timestamp = mix.timestamp # INFO: Write track mix save state if mix.extra.get("trackmix_saved"): From f0cb09d4ff8f8f5fc89fc40fae81305f56f80d9d Mon Sep 17 00:00:00 2001 From: cwilvx Date: Sat, 28 Dec 2024 16:48:05 +0300 Subject: [PATCH 33/44] update requirements --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index 00df6efc..2411f60d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ astroid==2.15.8 attrs==23.1.0 black==22.12.0 blinker==1.6.3 +boto3==1.35.71 +botocore==1.35.71 Brotli==1.1.0 certifi==2023.7.22 charset-normalizer==3.3.0 @@ -29,6 +31,7 @@ iniconfig==2.0.0 isort==5.12.0 itsdangerous==2.1.2 Jinja2==3.1.2 +jmespath==1.0.1 lazy-object-proxy==1.9.0 locust==2.20.1 MarkupSafe==2.1.3 @@ -55,7 +58,10 @@ pyxdg==0.28 pyzmq==25.1.2 rapidfuzz==2.15.2 requests==2.31.0 +requests-ip-rotator==1.0.14 roundrobin==0.0.4 +s3transfer==0.10.4 +schedule==1.2.2 setproctitle==1.3.3 show-in-file-manager==1.1.4 six==1.16.0 From 78224d17b8f227f00a4ed9c136980929521167af Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 30 Dec 2024 13:00:33 +0300 Subject: [PATCH 34/44] fix: index error on empty recents update --- app/lib/recipes/recents.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/lib/recipes/recents.py b/app/lib/recipes/recents.py index a9123d3c..1a5ed760 100644 --- a/app/lib/recipes/recents.py +++ b/app/lib/recipes/recents.py @@ -37,12 +37,15 @@ class RecentlyPlayed(HomepageRoutine): ) item = items[0] - store_entry = HomepageStore.entries[self.store_key].items[ - self.userids[0] - ][0] + try: + store_entry = HomepageStore.entries[self.store_key].items[ + self.userids[0] + ][0] + except IndexError: + store_entry = None if ( - item["type"] + item["hash"] + store_entry and item["type"] + item["hash"] == store_entry["type"] + store_entry["hash"] ): # If the item is the same as the one in the store From 8136e350aaf3787e2d8015e350e61fb58bcd2e65 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 30 Dec 2024 21:00:16 +0300 Subject: [PATCH 35/44] lastfm integration --- app/api/plugins/__init__.py | 40 +++++++++++++++++++ app/api/scrobble/__init__.py | 6 +++ app/config.py | 3 ++ app/plugins/lastfm.py | 76 ++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 app/plugins/lastfm.py diff --git a/app/api/plugins/__init__.py b/app/api/plugins/__init__.py index 9e86e18c..ae4ee5fb 100644 --- a/app/api/plugins/__init__.py +++ b/app/api/plugins/__init__.py @@ -2,7 +2,10 @@ from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.auth import admin_required +from app.config import UserConfig from app.db.userdata import PluginTable +from app.plugins.lastfm import LastFmPlugin +from app.utils.auth import get_current_userid bp_tag = Tag(name="Plugins", description="Manage plugins") api = APIBlueprint("plugins", __name__, url_prefix="/plugins", abp_tags=[bp_tag]) @@ -61,3 +64,40 @@ def update_plugin_settings(body: PluginSettingsBody): plugin = PluginTable.get_by_name(plugin) return {"status": "success", "settings": plugin.settings} + + +class LastFmSessionBody(BaseModel): + token: str = Field(description="The token to use to create the session") + + +@api.post("/lastfm/session/create") +def create_lastfm_session(body: LastFmSessionBody): + """ + Create a Last.fm session + """ + if not body.token: + return {"error": "Missing token"}, 400 + + lastfm = LastFmPlugin() + session_key = lastfm.get_session_key(body.token) + + if session_key: + config = UserConfig() + current_user = get_current_userid() + config.lastfmSessionKeys[str(current_user)] = session_key + config.lastfmSessionKeys = config.lastfmSessionKeys + + return {"status": "success", "session_key": session_key} + + +@api.post("/lastfm/session/delete") +def delete_lastfm_session(): + """ + Delete the Last.fm session + """ + config = UserConfig() + current_user = get_current_userid() + config.lastfmSessionKeys[str(current_user)] = "" + config.lastfmSessionKeys = config.lastfmSessionKeys + + return {"status": "success"} diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index 212e7973..d0396465 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -13,6 +13,7 @@ from app.lib.recipes.recents import RecentlyPlayed from app.models.album import Album from app.models.stats import StatItem from app.models.track import Track +from app.plugins.lastfm import LastFmPlugin from app.serializers.artist import serialize_for_card from app.serializers.album import serialize_for_card as serialize_for_album_card from app.serializers.track import serialize_track, serialize_tracks @@ -97,6 +98,11 @@ def log_track(body: LogTrackBody): if track: track.increment_playcount(duration, timestamp) + lastfm = LastFmPlugin() + + if lastfm.enabled: + lastfm.scrobble(trackentry.tracks[0], timestamp) + return {"msg": "recorded"}, 201 diff --git a/app/config.py b/app/config.py index b365527c..39b1d27c 100644 --- a/app/config.py +++ b/app/config.py @@ -48,6 +48,9 @@ class UserConfig: # plugins enablePlugins: bool = True + lastfmApiKey: str = "5e5306fbf3e8e3bc92f039b6c6c4bd4e" + lastfmApiSecret: str = "0553005e93f9a4b4819d835182181806" + lastfmSessionKeys: dict[str, str] = field(default_factory=dict) def __post_init__(self): """ diff --git a/app/plugins/lastfm.py b/app/plugins/lastfm.py new file mode 100644 index 00000000..9db8be76 --- /dev/null +++ b/app/plugins/lastfm.py @@ -0,0 +1,76 @@ +import requests +from typing import Any +from hashlib import md5 +from urllib.parse import quote_plus + +from app.config import UserConfig +from app.models.track import Track +from app.utils.auth import get_current_userid +from app.utils.threading import background +from app.plugins import Plugin, plugin_method + + +class LastFmPlugin(Plugin): + def __init__(self): + self.config = UserConfig() + super().__init__("lastfm", "Last.fm scrobbler") + self.set_active( + bool( + self.config.lastfmApiKey + and self.config.lastfmApiSecret + and self.config.lastfmSessionKeys.get(str(get_current_userid())) + ) + ) + + def get_api_signature(self, data: dict[str, Any]) -> str: + params = {k: v for k, v in data.items()} + + signature = "".join(f"{k}{v}" for k, v in sorted(params.items())) + signature += self.config.lastfmApiSecret + + return md5(signature.encode("utf-8")).hexdigest() + + def post(self, data: dict[str, Any], useSessionKey: bool = True): + url = "http://ws.audioscrobbler.com/2.0/?format=json" + data["api_key"] = self.config.lastfmApiKey + if useSessionKey: + data["sk"] = self.config.lastfmSessionKeys.get(str(get_current_userid())) + + data["api_sig"] = self.get_api_signature(data) + + final_url = ( + url + "&" + "&".join(f"{k}={quote_plus(str(v))}" for k, v in data.items()) + ) + + return requests.post(final_url) + + def get_session_key(self, token: str): + data = { + "method": "auth.getSession", + "token": token, + } + + try: + res = self.post(data, useSessionKey=False) + return res.json()["session"]["key"] + except Exception as e: + print("get_session_key error", e) + return None + + @plugin_method + @background + def scrobble(self, track: Track, timestamp: int): + print("Last.fm: logging track: ", track.title, "-", track.artists[0]["name"]) + data = { + "method": "track.scrobble", + "artist": track.artists[0]["name"], + "track": track.title, + "timestamp": timestamp, + "album": track.album, + "albumArtist": track.albumartists[0]["name"], + } + + try: + self.post(data) + except Exception as e: + print("scrobble error", e) From e0a14e52db015019249cd520006cb791064a255a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 30 Dec 2024 21:07:44 +0300 Subject: [PATCH 36/44] hide lastfm session keys for other users --- app/api/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/api/settings.py b/app/api/settings.py index c09e6b6e..ead9ecdc 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -9,6 +9,7 @@ from app.db.userdata import PluginTable from app.lib.index import index_everything from app.settings import Info from app.config import UserConfig +from app.utils.auth import get_current_userid bp_tag = Tag(name="Settings", description="Customize stuff") api = APIBlueprint("settings", __name__, url_prefix="/notsettings", abp_tags=[bp_tag]) @@ -98,6 +99,10 @@ def get_all_settings(): config["plugins"] = plugins config["version"] = Info.SWINGMUSIC_APP_VERSION + # hide lastfmSessionKeys for other users + current_user = get_current_userid() + config["lastfmSessionKey"] = config["lastfmSessionKeys"].get(str(current_user), "") + del config["lastfmSessionKeys"] return config From c64f40e54bc470852ea8faae75a265fac1f2f7a3 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 30 Dec 2024 21:13:25 +0300 Subject: [PATCH 37/44] switch api keys --- app/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config.py b/app/config.py index 39b1d27c..3411d0ba 100644 --- a/app/config.py +++ b/app/config.py @@ -48,8 +48,8 @@ class UserConfig: # plugins enablePlugins: bool = True - lastfmApiKey: str = "5e5306fbf3e8e3bc92f039b6c6c4bd4e" - lastfmApiSecret: str = "0553005e93f9a4b4819d835182181806" + lastfmApiKey: str = "0553005e93f9a4b4819d835182181806" + lastfmApiSecret: str = "5e5306fbf3e8e3bc92f039b6c6c4bd4e" lastfmSessionKeys: dict[str, str] = field(default_factory=dict) def __post_init__(self): From df4c9113711e2e571fde8fe949ee9f88b0df1cc6 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 31 Dec 2024 01:01:48 +0300 Subject: [PATCH 38/44] add logs --- app/lib/recipes/recents.py | 9 ++++++--- app/plugins/lastfm.py | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/lib/recipes/recents.py b/app/lib/recipes/recents.py index 1a5ed760..a42083bb 100644 --- a/app/lib/recipes/recents.py +++ b/app/lib/recipes/recents.py @@ -36,17 +36,20 @@ class RecentlyPlayed(HomepageRoutine): limit=self.ITEM_LIMIT, userid=self.userids[0], _entries=[last_entry] ) - item = items[0] try: + item = items[0] store_entry = HomepageStore.entries[self.store_key].items[ self.userids[0] ][0] except IndexError: store_entry = None + item = None if ( - store_entry and item["type"] + item["hash"] - == store_entry["type"] + store_entry["hash"] + store_entry + and item + and store_entry["type"] + store_entry["hash"] + == item["type"] + item["hash"] ): # If the item is the same as the one in the store # only update the timestamp diff --git a/app/plugins/lastfm.py b/app/plugins/lastfm.py index 9db8be76..1c4587a4 100644 --- a/app/plugins/lastfm.py +++ b/app/plugins/lastfm.py @@ -70,7 +70,11 @@ class LastFmPlugin(Plugin): "albumArtist": track.albumartists[0]["name"], } + print("scrobble data:", data) + try: - self.post(data) + res = self.post(data) + print("scrobble response:", res.text) + print("scrobble response json:", res.json()) except Exception as e: print("scrobble error", e) From cbfe37a4acf0de9692b0393ed485aa70ec5debf8 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 31 Dec 2024 13:43:13 +0300 Subject: [PATCH 39/44] fix: show albums with a single track as singles --- app/models/album.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/album.py b/app/models/album.py index afe4f80b..34ecbac7 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -173,7 +173,7 @@ class Album: # REVIEW: Reading from the config file in a for loop will be slow # TODO: Find a - if singleTrackAsSingle and len(tracks) == 1: + if singleTrackAsSingle and self.trackcount == 1: return True if ( From 2cc063ad7643259d8f1b52d41dce8f8b40a7c396 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 31 Dec 2024 13:56:40 +0300 Subject: [PATCH 40/44] logger stuff --- app/logger.py | 2 +- app/plugins/lastfm.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/logger.py b/app/logger.py index 3df7cd7d..847680bd 100644 --- a/app/logger.py +++ b/app/logger.py @@ -35,7 +35,7 @@ class CustomFormatter(logging.Formatter): return formatter.format(record) -log = logging.getLogger("swing") +log = logging.getLogger("SWING MUSIC") log.propagate = False log.setLevel(logging.DEBUG) diff --git a/app/plugins/lastfm.py b/app/plugins/lastfm.py index 1c4587a4..b25609ea 100644 --- a/app/plugins/lastfm.py +++ b/app/plugins/lastfm.py @@ -9,6 +9,7 @@ from app.utils.auth import get_current_userid from app.utils.threading import background from app.plugins import Plugin, plugin_method +from app.logger import log class LastFmPlugin(Plugin): def __init__(self): @@ -60,7 +61,7 @@ class LastFmPlugin(Plugin): @plugin_method @background def scrobble(self, track: Track, timestamp: int): - print("Last.fm: logging track: ", track.title, "-", track.artists[0]["name"]) + log.info(f"Last.fm: logging track: {track.title} - {track.artists[0]['name']}") data = { "method": "track.scrobble", "artist": track.artists[0]["name"], @@ -70,11 +71,11 @@ class LastFmPlugin(Plugin): "albumArtist": track.albumartists[0]["name"], } - print("scrobble data:", data) + log.info(f"scrobble data: {data}") try: res = self.post(data) - print("scrobble response:", res.text) - print("scrobble response json:", res.json()) + log.info("scrobble response:" + str(res.text)) + log.info("scrobble response json:" + str(res.json())) except Exception as e: - print("scrobble error", e) + log.info("scrobble error" + str(e)) From 2a12487220e00bab019c1c6ab331f6d68350a51a Mon Sep 17 00:00:00 2001 From: cwilvx Date: Mon, 6 Jan 2025 00:18:17 +0300 Subject: [PATCH 41/44] lastfm: dump failed scrobbles locally + bump tinytag to v2.0.0 and refactor taglib.py + add explicit flag to track model --- app/api/artist.py | 8 +- app/config.py | 2 + app/db/userdata.py | 2 +- app/lib/colorlib.py | 10 +- app/lib/taglib.py | 247 ++++++++++++------------------------------ app/logger.py | 7 +- app/models/track.py | 3 + app/plugins/lastfm.py | 91 +++++++++++++++- poetry.lock | 14 +-- pyproject.toml | 2 +- 10 files changed, 188 insertions(+), 198 deletions(-) diff --git a/app/api/artist.py b/app/api/artist.py index e630dd8c..79c4c432 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -7,6 +7,7 @@ from pprint import pprint import random from datetime import datetime from itertools import groupby +from typing import Any from flask_openapi3 import APIBlueprint, Tag from pydantic import Field @@ -72,6 +73,7 @@ def get_artist(path: ArtistHashSchema, query: GetArtistQuery): except ValueError: year = 0 + genres = [*artist.genres] decade = None if year: @@ -79,7 +81,7 @@ def get_artist(path: ArtistHashSchema, query: GetArtistQuery): decade = str(decade)[2:] + "s" if decade: - artist.genres.insert(0, {"name": decade, "genrehash": decade}) + genres.insert(0, {"name": decade, "genrehash": decade}) stats = get_track_group_stats(tracks) duration = sum(t.duration for t in tracks) if tracks else 0 @@ -105,7 +107,7 @@ def get_artist(path: ArtistHashSchema, query: GetArtistQuery): "duration": duration, "trackcount": tcount, "albumcount": artist.albumcount, - "genres": artist.genres, + "genres": genres, "is_favorite": artist.is_favorite, }, "tracks": tracks, @@ -150,7 +152,7 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): albums = [a for a in albumdict.values()] all_albums = sorted(albums, key=lambda a: a.date, reverse=True) - res = { + res: dict[str, Any] = { "albums": [], "appearances": [], "compilations": [], diff --git a/app/config.py b/app/config.py index 3411d0ba..bdbcaed5 100644 --- a/app/config.py +++ b/app/config.py @@ -27,6 +27,8 @@ class UserConfig: "AC/DC", "Bob marley & the wailers", "Crosby, Stills, Nash & Young", + "Smith & Thell", + "Peter, Paul & Mary", } ) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) diff --git a/app/db/userdata.py b/app/db/userdata.py index d8ca569f..a5c66887 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -465,7 +465,7 @@ class LibDataTable(Base): @classmethod def find_one(cls, hash: str, type: Literal["album", "artist"]): result = cls.execute( - select(cls).where((cls.itemhash == hash) & (cls.itemtype == type)) + select(cls).where((cls.itemhash == type + hash) & (cls.itemtype == type)) ) return result.fetchone() diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index 5b56dbe0..8f5587ce 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -124,9 +124,15 @@ class ProcessArtistColors: artist.set_color(colors[0]) # INFO: Write to the database. + print("RECORD") + print(record) if record is None: LibDataTable.insert_one( - {"itemhash": artisthash, "color": colors[0], "itemtype": "artist"} + { + "itemhash": "artist" + artisthash, + "color": colors[0], + "itemtype": "artist", + } ) else: - LibDataTable.update_one(artisthash, {"color": colors[0]}) + LibDataTable.update_one("artist" + artisthash, {"color": colors[0]}) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 820715f7..028b9c27 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -155,35 +155,54 @@ def get_tags(filepath: str, config: UserConfig): return None try: - tags: Any = TinyTag.get(filepath) - except: # noqa: E722 + tags = TinyTag.get(filepath) + except Exception as e: # noqa: E722 return None + metadata: dict[str, Any] = { + "album": tags.album, + "albumartists": tags.albumartist, + "artists": tags.artist, + "title": tags.title, + "last_mod": last_mod, + "filepath": win_replace_slash(filepath), + "folder": win_replace_slash(os.path.dirname(filepath)), + "bitrate": tags.bitrate, + "duration": tags.duration, + "track": tags.track, + "disc": tags.disc, + "genres": tags.genre, + "copyright": " ".join(tags.other.get("copyright", [])), + "extra": {}, + } + no_albumartist: bool = (tags.albumartist == "") or (tags.albumartist is None) no_artist: bool = (tags.artist == "") or (tags.artist is None) if no_albumartist and not no_artist: - tags.albumartist = tags.artist + # INFO: If no albumartist, use the artist + metadata["albumartists"] = tags.artist if no_artist and not no_albumartist: - tags.artist = tags.albumartist + # INFO: If no artist, use the albumartist + metadata["artist"] = tags.albumartist parse_data = None + # INFO: If title or album is empty, extract the album and title from the filename to_filename = ["title", "album"] for tag in to_filename: - p = getattr(tags, tag) + p = metadata[tag] if p == "" or p is None: parse_data = extract_artist_title(filename, config) title = parse_data.title.replace("_", " ") - setattr(tags, tag, title) + metadata[tag] = title - # tags.title = tags.title.replace("_", " ") - # tags.album = tags.album.replace("_", " ") - - parse = ["artist", "albumartist"] + # INFO: If artist or albumartist is empty + # extract the artist and albumartist from the filename + parse = ["artists", "albumartists"] for tag in parse: - p = getattr(tags, tag) + p = metadata[tag] if p == "" or p is None: if not parse_data: @@ -192,194 +211,68 @@ def get_tags(filepath: str, config: UserConfig): artist = parse_data.artist if artist: - setattr(tags, tag, ", ".join(artist)) + metadata[tag] = ", ".join(artist) else: - setattr(tags, tag, "Unknown") + metadata[tag] = "Unknown" - # TODO: Move parsing title, album and artist to startup. (Maybe!) - - to_check = ["album", "year", "albumartist"] + # INFO: If these are empty, set to "Unknown" + to_check = ["album", "albumartists"] for prop in to_check: - p = getattr(tags, prop) + p = metadata[prop] if (p is None) or (p == ""): - setattr(tags, prop, "Unknown") + metadata[prop] = "Unknown" + # INFO: Round the bitrate and duration to_round = ["bitrate", "duration"] for prop in to_round: try: - setattr(tags, prop, math.floor(getattr(tags, prop))) + metadata[prop] = math.floor(getattr(tags, prop)) except TypeError: - setattr(tags, prop, 0) + metadata[prop] = 0 + # INFO: Convert these to int to_int = ["track", "disc"] for prop in to_int: try: - setattr(tags, prop, int(getattr(tags, prop))) + metadata[prop] = int(getattr(tags, prop)) except (ValueError, TypeError): - setattr(tags, prop, 1) + metadata[prop] = 1 - try: - tags.copyright = tags.extra["copyright"] - except KeyError: - tags.copyright = None + # INFO: Extract copyright from extra data + metadata["date"] = parse_date(tags.year or "") or int(last_mod) - # tags.image = f"{tags.albumhash}.webp" - tags.folder = win_replace_slash(os.path.dirname(filepath)) - - tags.date = parse_date(tags.year) or int(last_mod) - tags.filepath = win_replace_slash(filepath) - tags.last_mod = last_mod - - tags.artists = tags.artist - tags.albumartists = tags.albumartist - - # split_artist = split_artists(tags.artist, separators=config.artistSeparators) - # split_albumartists = split_artists(tags.albumartist, separators=config.artistSeparators) - # new_title = tags.title - - # TODO: Figure out which is the best spot to create these hashes # create albumhash using og_album - tags.albumhash = create_hash(tags.album or "", tags.albumartist) + metadata["albumhash"] = create_hash( + tags.album or "", metadata.get("albumartists", "") + ) - # extract featured artists - # if config.extractFeaturedArtists: - # feat, new_title = parse_feat_from_title( - # tags.title, separators=config.artistSeparators - # ) - # original_lower = "-".join([create_hash(a) for a in split_artist]) - # split_artist.extend(a for a in feat if create_hash(a) not in original_lower) + metadata["trackhash"] = create_hash( + metadata.get("artist", ""), metadata.get("album", ""), metadata.get("title", "") + ) - # if no albumartist, assign to the first artist - if not tags.albumartist: - tags.albumartist = split_artists(tags.artist, config)[:1] - - # create json objects for artists and albumartists - # tags.artists = [ - # { - # "artisthash": create_hash(a, decode=True), - # "name": a, - # } - # for a in split_artist - # ] - - # tags.albumartists = [ - # { - # "artisthash": create_hash(a, decode=True), - # "name": a, - # } - # for a in split_albumartists - # ] - - # tags.artisthashes = list( - # {a["artisthash"] for a in tags.artists} - # ) - - # remove prod by - # if config.removeProdBy: - # new_title = remove_prod(new_title) - - # if track is a single, ie. - # if og_title == album, rename album to new_title - # if tags.title == tags.album: - # tags.album = new_title - - # remove remaster from track title - # if config.removeRemasterInfo: - # new_title = clean_title(new_title) - - # save final title - # tags.og_title = tags.title - # tags.title = new_title - # tags.og_album = tags.album - - # clean album title - # if config.cleanAlbumTitle: - # tags.album, _ = get_base_title_and_versions(tags.album, get_versions=False) - - # merge album versions - # if config.mergeAlbums: - # tags.albumhash = create_hash( - # tags.album, *(a["name"] for a in tags.albumartists) - # ) - - # process genres - # if tags.genre: - # src_genres: str = tags.genre - # src_genres = src_genres.lower() - # # separators = {"/", ";", "&"} - # separators = set(config.genreSeparators) - - # contains_rnb = "r&b" in src_genres - # contains_rock = "rock & roll" in src_genres - - # if contains_rnb: - # src_genres = src_genres.replace("r&b", "RnB") - - # if contains_rock: - # src_genres = src_genres.replace("rock & roll", "rock") - - # for s in separators: - # src_genres = src_genres.replace(s, ",") - - # genres_list: list[str] = src_genres.split(",") - # tags.genres = [ - # {"name": g.strip(), "genrehash": create_hash(g.strip())} - # for g in genres_list - # ] - # tags.genrehashes = [g["genrehash"] for g in tags.genres] - # else: - # tags.genres = [] - # tags.genrehashes = [] - - tags.genres = tags.genre - - # sub underscore with space - # tags.title = tags.title.replace("_", " ") - # tags.album = tags.album.replace("_", " ") - tags.trackhash = create_hash(tags.artists, tags.album, tags.title) - - more_extra = { - "audio_offset": tags.audio_offset, - "bitdepth": tags.bitdepth, - "composer": tags.composer, - "channels": tags.channels, - "comment": tags.comment, - "disc_total": tags.disc_total, - "filesize": tags.filesize, - "samplerate": tags.samplerate, - "track_total": tags.track_total, - "hashinfo": { - "algo": "sha1", - "format": "[:5]+[-5:]", # first 5 + last 5 chars - }, + extra: dict[str, Any] = { + k: v for k, v in tags.as_dict().items() if metadata.get(k, "meh") == "meh" } - tags.extra = {**tags.extra, **more_extra} + extra["hashinfo"] = { + "algo": "sha1", + "format": "[:5]+[-5:]", # first 5 + last 5 chars + } - tags = tags.__dict__ + to_pop = ["filename", "artist", "albumartist", "year"] - # delete all tag properties that start with _ (tinytag internals) - for tag in list(tags): - if tag.startswith("_"): - del tags[tag] + # REMOVE EMPTY VALUES + for key, value in extra.items(): + if ( + value is None + or value == "" + # INFO: If value is a list, check if it's empty or if the first element is empty + or (type(value) is list and "".join(value) == "") + ): + to_pop.append(key) - to_delete = [ - "filesize", - "audio_offset", - "channels", - "comment", - "composer", - "disc_total", - "samplerate", - "track_total", - "year", - "bitdepth", - "artist", - "albumartist", - "genre", - ] + for key in to_pop: + extra.pop(key, None) - for tag in to_delete: - del tags[tag] - - return tags + metadata["extra"] = extra + return metadata diff --git a/app/logger.py b/app/logger.py index 847680bd..48b4c53f 100644 --- a/app/logger.py +++ b/app/logger.py @@ -16,10 +16,9 @@ class CustomFormatter(logging.Formatter): red = "\033[41m" bold_red = "\x1b[31;1m" reset = "\x1b[0m" - # format = ( - # "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" - # ) - format_ = "%(message)s" + # format_ = "[%(asctime)s] %(name)s %(levelname)s %(message)s (%(filename)s:%(lineno)d)" + format_ = "[%(asctime)s] [%(levelname)s] %(message)s (%(filename)s:%(lineno)d)\n" + # format_ = "%(message)s" FORMATS = { logging.DEBUG: grey + format_ + reset, diff --git a/app/models/track.py b/app/models/track.py index 5598b9e4..e0a17673 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -50,6 +50,7 @@ class Track: _pos: int = 0 _ati: str = "" image: str = "" + explicit: bool = False fav_userids: list[int] = field(default_factory=list) @property @@ -78,6 +79,8 @@ class Track: self.og_album = self.album self.folder = self.folder + "/" self.weakhash = create_hash(self.title, self.artists) + explicit_tag = self.extra.get("explicit", ["0"]) + self.explicit = int(explicit_tag[0]) == 1 self.image = self.albumhash + ".webp" self.extra = { diff --git a/app/plugins/lastfm.py b/app/plugins/lastfm.py index b25609ea..6948ca86 100644 --- a/app/plugins/lastfm.py +++ b/app/plugins/lastfm.py @@ -1,3 +1,6 @@ +import json +from pathlib import Path +import time import requests from typing import Any from hashlib import md5 @@ -5,13 +8,21 @@ from urllib.parse import quote_plus from app.config import UserConfig from app.models.track import Track +from app.settings import Paths from app.utils.auth import get_current_userid from app.utils.threading import background from app.plugins import Plugin, plugin_method from app.logger import log + class LastFmPlugin(Plugin): + """ + Last.fm scrobbler plugin. + """ + + UPLOADING_DUMPS = False + def __init__(self): self.config = UserConfig() super().__init__("lastfm", "Last.fm scrobbler") @@ -71,11 +82,85 @@ class LastFmPlugin(Plugin): "albumArtist": track.albumartists[0]["name"], } + success = self.post_scrobble_data({**data}) + + if not success: + self.dump_scrobble(data) + else: + self.upload_dumps() + + return success + + def post_scrobble_data(self, data: dict[str, Any]): + """ + Uploads the scrobble data and handles the + response from the lastfm scrobble endpoint. + """ log.info(f"scrobble data: {data}") try: res = self.post(data) - log.info("scrobble response:" + str(res.text)) - log.info("scrobble response json:" + str(res.json())) except Exception as e: - log.info("scrobble error" + str(e)) + log.warn("scrobble response error" + str(e)) + return False + + log.info("scrobble response text: " + str(res.text)) + log.info("scrobble response json: " + str(res.json())) + + res_json: dict[str, Any] = res.json() + + if res_json.get("error"): + log.error("LASTFM: scrobble error" + str(res_json)) + + if res_json["error"] == 9: + log.error("LAST.FM: Invalid session key") + # Invalid session key + self.config.lastfmSessionKeys.pop(str(get_current_userid())) + self.config.lastfmSessionKeys = self.config.lastfmSessionKeys + return False + + if res_json.get("scrobbles", {}).get("@attr", {}).get("accepted") == 1: + log.info("scrobble accepted") + return True + + return False + + # SECTION: Persistence + def dump_scrobble(self, data: dict[str, Any]): + """ + Dumps the scrobble data to a file in the lastfm plugin directory. + """ + dump_dir = Path(Paths.get_plugins_path(), "lastfm") + if not dump_dir.exists(): + dump_dir.mkdir(parents=True, exist_ok=True) + + path = dump_dir / f"{int(time.time())}.json" + + log.info(f"Dumping scrobble to {path}") + with open(path, "w") as f: + json.dump(data, f) + + def upload_dumps(self): + """ + Uploads the scrobble dumps to the lastfm api. + """ + if self.UPLOADING_DUMPS: + return + + self.UPLOADING_DUMPS = True + dump_dir = Path(Paths.get_plugins_path(), "lastfm") + + if not dump_dir.exists(): + return + + try: + for file in dump_dir.iterdir(): + log.info(f"Uploading dump: {file}") + with open(file, "r") as f: + data = json.load(f) + success = self.post_scrobble_data(data) + + if success: + file.unlink() + finally: + self.UPLOADING_DUMPS = False diff --git a/poetry.lock b/poetry.lock index 4d60be39..6f6c63dd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2323,17 +2323,17 @@ widechars = ["wcwidth"] [[package]] name = "tinytag" -version = "1.10.1" -description = "Read music meta data and length of MP3, OGG, OPUS, MP4, M4A, FLAC, WMA and Wave files" +version = "2.0.0" +description = "Read audio file metadata" optional = false -python-versions = ">=2.7" +python-versions = ">=3.7" files = [ - {file = "tinytag-1.10.1-py3-none-any.whl", hash = "sha256:e437654d04c966fbbbdbf807af61eb9759f1d80e4173a7d26202506b37cfdaf0"}, - {file = "tinytag-1.10.1.tar.gz", hash = "sha256:122a63b836f85094aacca43fc807aaee3290be3de17d134f5f4a08b509ae268f"}, + {file = "tinytag-2.0.0-py3-none-any.whl", hash = "sha256:971b9dceae2d1de73b5e8300639ea0b41454633b899426e702aed15f0e72a9b4"}, + {file = "tinytag-2.0.0.tar.gz", hash = "sha256:d041f53d15553bb148549bfbc7feab445caf7105ba95fa2ecb9827bb06b62275"}, ] [package.extras] -tests = ["flake8", "pytest", "pytest-cov"] +tests = ["coverage", "mypy", "pycodestyle", "pylint", "pytest"] [[package]] name = "tomli" @@ -2775,4 +2775,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "85f8932739522e7b53b4fe5bbecc3c10a30bb690e25bf9404209c57ec71e88d3" +content-hash = "733ca957831c695560fe292a6dfdad13c3fc905695f473cd48cf13bfba8defdc" diff --git a/pyproject.toml b/pyproject.toml index d2f32de0..1515ac38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ Pillow = "^9.0.1" "colorgram.py" = "^1.2.0" tqdm = "^4.65.0" rapidfuzz = "^2.13.7" -tinytag = "^1.10.1" +tinytag = ">=2.0.0" Unidecode = "^1.3.6" psutil = "^5.9.4" show-in-file-manager = "^1.1.4" From fe39cadfdcf991f75652d4313b1edf7c909ac79f Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 7 Jan 2025 23:13:19 +0300 Subject: [PATCH 42/44] feat: use thumbnails from folders + cache failed lastfm scrobbles + implement lastfm scrobble filter + change /home to /nothome --- app/api/home/__init__.py | 2 +- app/api/imgserver.py | 117 ++++++++++++++++++++++++++++++++--- app/api/scrobble/__init__.py | 20 ++++-- app/config.py | 1 + app/lib/tagger.py | 1 + app/models/album.py | 1 + app/models/track.py | 2 + app/settings.py | 4 ++ app/store/albums.py | 12 ---- app/utils/stats.py | 7 +-- 10 files changed, 136 insertions(+), 31 deletions(-) diff --git a/app/api/home/__init__.py b/app/api/home/__init__.py index 095c0f44..00748ab7 100644 --- a/app/api/home/__init__.py +++ b/app/api/home/__init__.py @@ -8,7 +8,7 @@ from app.lib.home.get_recently_played import get_recently_played from app.store.homepage import HomepageStore bp_tag = Tag(name="Home", description="Homepage items") -api = APIBlueprint("home", __name__, url_prefix="/home", abp_tags=[bp_tag]) +api = APIBlueprint("home", __name__, url_prefix="/nothome", abp_tags=[bp_tag]) @api.get("/recents/added") diff --git a/app/api/imgserver.py b/app/api/imgserver.py index a3aa8d48..f635f175 100644 --- a/app/api/imgserver.py +++ b/app/api/imgserver.py @@ -1,3 +1,4 @@ +from fileinput import filename from pathlib import Path from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint @@ -5,6 +6,10 @@ from pydantic import BaseModel, Field from flask import send_from_directory from app.settings import Defaults, Paths +from app.store.albums import AlbumStore +from app.store.tracks import TrackStore +from app.utils.threading import background +from PIL import Image bp_tag = Tag( name="Images", description="Image filenames are constructured as '{itemhash}.webp'" @@ -12,6 +17,74 @@ bp_tag = Tag( api = APIBlueprint("imgserver", __name__, url_prefix="/img", abp_tags=[bp_tag]) +@background +def cache_thumbnails(filepath: Path, trackhash: str): + """ + Resizes the image and stores it in the cache directory. + """ + image = Image.open(filepath) + path = Path(Paths.get_image_cache_path()) + aspect_ratio = image.width / image.height + + sizes = { + "xsmall": 64, + "small": 96, + "medium": 256, + "large": 512, + } + + for size, width in sizes.items(): + width = min(width, image.width) + height = int(width / aspect_ratio) + + resized_path = path / size / (trackhash + ".webp") + resized_path.parent.mkdir(parents=True, exist_ok=True) + image.resize((width, height)).save(resized_path, format="webp") + + +def find_thumbnail(albumhash: str, pathhash: str): + # entry = TrackStore.trackhashmap.get(albumhash) + entry = AlbumStore.albummap.get(albumhash) + + if entry is None: + return None, None, "" + + track_file = None + + tracks = TrackStore.get_tracks_by_trackhashes(entry.trackhashes) + for track in tracks: + if track.pathhash == pathhash: + track_file = track + break + + if track_file is None: + return None, None, "" + + folder = Path(track_file.folder) + + # INFO: Check if the folder has image files + extensions = [".jpg", ".jpeg", ".png", ".webp"] + hierarchy = ["cover", "front", "back", "folder", "album", "artwork"] + + images: list[Path] = [] + for item in folder.iterdir(): + if item.suffix in extensions: + images.append(item) + + if len(images) == 0: + return None, None, "" + + # INFO: Check if the folder has image files in the hierarchy + for item in hierarchy: + for image in images: + if image.name.lower().startswith(item.lower()): + return image.parent, image.name, track_file.albumhash + + # INFO: If no image falls in the hierarchy, return the first image + first_image = images[0] + return first_image.parent, first_image.name, track_file.albumhash + + def send_fallback_img(filename: str = "default.webp"): """ Returns the fallback image from the assets folder. @@ -25,7 +98,9 @@ def send_fallback_img(filename: str = "default.webp"): return send_from_directory(folder, filename) -def send_file_or_fallback(folder: str, filename: str, fallback: str = "default.webp"): +def send_file_or_fallback( + folder: str, filename: str, fallback: str = "default.webp", pathhash: str = "" +): """ Returns the file from the folder or the fallback image. """ @@ -34,6 +109,22 @@ def send_file_or_fallback(folder: str, filename: str, fallback: str = "default.w if fpath.exists(): return send_from_directory(folder, filename) + if pathhash != "": + # INFO: Check if the image is in the cache + cache_path = Path(Paths.get_image_cache_path()) / fpath.parent.name / filename + if cache_path.exists(): + return send_from_directory(cache_path.parent, cache_path.name) + + # INFO: Find the thumbnail + parent, file, albumhash = find_thumbnail( + filename.replace(".webp", ""), pathhash + ) + + # INFO: Cache and send the thumbnail + if file is not None and parent is not None: + cache_thumbnails(parent / file, albumhash) + return send_from_directory(parent, file) + return send_fallback_img(fallback) @@ -44,6 +135,13 @@ class ImagePath(BaseModel): ) +class ImageQuery(BaseModel): + pathhash: str = Field( + description="The path hash used to find the thumbnail", + default="", + ) + + # @api.get("/t/o/") # def send_original_thumbnail(path: ImagePath): # """ @@ -60,39 +158,39 @@ class ImagePath(BaseModel): # TRACK THUMBNAILS @api.get("/thumbnail/") -def send_lg_thumbnail(path: ImagePath): +def send_lg_thumbnail(path: ImagePath, query: ImageQuery): """ Get large thumbnail (500 x 500) """ folder = Paths.get_lg_thumb_path() - return send_file_or_fallback(folder, path.imgpath) + return send_file_or_fallback(folder, path.imgpath, pathhash=query.pathhash) @api.get("/thumbnail/xsmall/") -def send_xsm_thumbnail(path: ImagePath): +def send_xsm_thumbnail(path: ImagePath, query: ImageQuery): """ Get extra small thumbnail (64px) """ folder = Paths.get_xsm_thumb_path() - return send_file_or_fallback(folder, path.imgpath) + return send_file_or_fallback(folder, path.imgpath, pathhash=query.pathhash) @api.get("/thumbnail/small/") -def send_sm_thumbnail(path: ImagePath): +def send_sm_thumbnail(path: ImagePath, query: ImageQuery): """ Get small thumbnail (96px) """ folder = Paths.get_sm_thumb_path() - return send_file_or_fallback(folder, path.imgpath) + return send_file_or_fallback(folder, path.imgpath, pathhash=query.pathhash) @api.get("/thumbnail/medium/") -def send_md_thumbnail(path: ImagePath): +def send_md_thumbnail(path: ImagePath, query: ImageQuery): """ Get medium thumbnail (256px) """ folder = Paths.get_md_thumb_path() - return send_file_or_fallback(folder, path.imgpath) + return send_file_or_fallback(folder, path.imgpath, pathhash=query.pathhash) # ARTISTS @@ -141,6 +239,7 @@ def send_playlist_image(path: PlaylistImagePath): folder = Paths.get_playlist_img_path() return send_file_or_fallback(folder, path.imgpath, "playlist.svg") + # MIXES @api.get("/mix/medium/") def send_md_mix_image(path: ImagePath): diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index d0396465..b2d60ccf 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -94,13 +94,19 @@ def log_track(body: LogTrackBody): if artist: artist.increment_playcount(duration, timestamp) - track = TrackStore.trackhashmap.get(body.trackhash) - if track: - track.increment_playcount(duration, timestamp) + trackentry.increment_playcount(duration, timestamp) + track = trackentry.tracks[0] lastfm = LastFmPlugin() - if lastfm.enabled: + print(track.duration / 2, 240, body.duration, "\n") + + if ( + lastfm.enabled + and track.duration > 30 + and body.duration >= min(track.duration / 2, 240) + # SEE: https://www.last.fm/api/scrobbling#when-is-a-scrobble-a-scrobble + ): lastfm.scrobble(trackentry.tracks[0], timestamp) return {"msg": "recorded"}, 201 @@ -350,7 +356,11 @@ def get_stats(): if len(tracks) > 0 else "—" ), - tracks[0].image if len(tracks) > 0 else None, + ( + tracks[0].image + "?pathhash=" + tracks[0].pathhash + if len(tracks) > 0 + else None + ), ) fav_count = FavoritesTable.count_favs_in_period(start_time, end_time) diff --git a/app/config.py b/app/config.py index bdbcaed5..178a233f 100644 --- a/app/config.py +++ b/app/config.py @@ -29,6 +29,7 @@ class UserConfig: "Crosby, Stills, Nash & Young", "Smith & Thell", "Peter, Paul & Mary", + "Simon & Garfunkel", } ) genreSeparators: set[str] = field(default_factory=lambda: {"/", ";", "&"}) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 8999d2ba..7358cb92 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -179,6 +179,7 @@ def create_albums(_trackhashes: list[str] = []) -> list[tuple[Album, set[str]]]: "playduration": track.playduration, "title": track.album, "tracks": {track.trackhash}, + "pathhash": track.pathhash, "extra": {}, } else: diff --git a/app/models/album.py b/app/models/album.py index 34ecbac7..68147aa3 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -30,6 +30,7 @@ class Album: playcount: int playduration: int extra: dict + pathhash: str = "" id: int = -1 type: str = "album" diff --git a/app/models/track.py b/app/models/track.py index e0a17673..e706487c 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -52,6 +52,7 @@ class Track: image: str = "" explicit: bool = False fav_userids: list[int] = field(default_factory=list) + pathhash: str = "" @property def is_favorite(self): @@ -81,6 +82,7 @@ class Track: self.weakhash = create_hash(self.title, self.artists) explicit_tag = self.extra.get("explicit", ["0"]) self.explicit = int(explicit_tag[0]) == 1 + self.pathhash = create_hash(self.folder) self.image = self.albumhash + ".webp" self.extra = { diff --git a/app/settings.py b/app/settings.py index df06e3d9..e9de5d7b 100644 --- a/app/settings.py +++ b/app/settings.py @@ -123,6 +123,10 @@ class Paths: def get_sm_mixes_img_path(cls): return join(cls.get_mixes_img_path(), "small") + @classmethod + def get_image_cache_path(cls): + return join(cls.get_img_path(), "cache") + # defaults class Defaults: diff --git a/app/store/albums.py b/app/store/albums.py index ed1860f1..68f2ec2d 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -38,20 +38,8 @@ class AlbumMapEntry: class AlbumStore: - # albums: list[Album] = CustomList() albummap: dict[str, AlbumMapEntry] = {} - # @staticmethod - # def create_album(track: Track): - # """ - # Creates album object from a track - # """ - # return Album( - # albumhash=track.albumhash, - # albumartists=track.albumartists, # type: ignore - # title=track.og_album, - # ) - @classmethod def load_albums(cls, instance_key: str): """ diff --git a/app/utils/stats.py b/app/utils/stats.py index 50087001..689a4550 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -233,7 +233,7 @@ def get_track_group_stats(tracks: list[Track], is_album: bool = False): "toptrack", f"top track ({seconds_to_time_string(top_track.playduration)} listened)", f"{top_track.title}", - top_track.image, + top_track.image + "?pathhash=" + top_track.pathhash if top_track else None, ) if top_track else StatItem( @@ -251,7 +251,7 @@ def get_track_group_stats(tracks: list[Track], is_album: bool = False): "playcount": 0, "playduration": 0, "title": track.album, - "image": track.image, + "image": track.image + "?pathhash=" + track.pathhash if track.image else None, } albums_map[track.albumhash]["playcount"] += 1 @@ -268,8 +268,7 @@ def get_track_group_stats(tracks: list[Track], is_album: bool = False): "topalbum", f"top album ({seconds_to_time_string(top_album['playduration'])} listened)", f"{top_album['title']}", - top_album["image"], - ) + top_album["image"]) if top_album else StatItem( "topalbum", From ec9f392d737243a3c8d513719de5219609f69014 Mon Sep 17 00:00:00 2001 From: cwilvx Date: Tue, 7 Jan 2025 23:21:31 +0300 Subject: [PATCH 43/44] write path hash on image property --- app/api/scrobble/__init__.py | 2 +- app/models/album.py | 2 +- app/models/track.py | 8 +++++--- app/utils/stats.py | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/api/scrobble/__init__.py b/app/api/scrobble/__init__.py index b2d60ccf..65e8b093 100644 --- a/app/api/scrobble/__init__.py +++ b/app/api/scrobble/__init__.py @@ -357,7 +357,7 @@ def get_stats(): else "—" ), ( - tracks[0].image + "?pathhash=" + tracks[0].pathhash + tracks[0].image if len(tracks) > 0 else None ), diff --git a/app/models/album.py b/app/models/album.py index 68147aa3..5a5714bc 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -54,7 +54,7 @@ class Album: self.fav_userids.append(userid) def __post_init__(self): - self.image = self.albumhash + ".webp" + self.image = self.albumhash + ".webp" + "?pathhash=" + self.pathhash self.populate_versions() self.weakhash = create_hash( self.og_title, ",".join(a["name"] for a in self.albumartists) diff --git a/app/models/track.py b/app/models/track.py index e706487c..9d6958ba 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -52,12 +52,15 @@ class Track: image: str = "" explicit: bool = False fav_userids: list[int] = field(default_factory=list) - pathhash: str = "" @property def is_favorite(self): return get_current_userid() in self.fav_userids + @property + def pathhash(self): + return create_hash(self.folder) + def toggle_favorite_user(self, userid: int): """ Toggles the favorite status of the track for a given user. @@ -82,9 +85,8 @@ class Track: self.weakhash = create_hash(self.title, self.artists) explicit_tag = self.extra.get("explicit", ["0"]) self.explicit = int(explicit_tag[0]) == 1 - self.pathhash = create_hash(self.folder) - self.image = self.albumhash + ".webp" + self.image = self.albumhash + ".webp" + "?pathhash=" + self.pathhash self.extra = { "disc_total": self.extra.get("disc_total", 0), "track_total": self.extra.get("track_total", 0), diff --git a/app/utils/stats.py b/app/utils/stats.py index 689a4550..97e388ff 100644 --- a/app/utils/stats.py +++ b/app/utils/stats.py @@ -233,7 +233,7 @@ def get_track_group_stats(tracks: list[Track], is_album: bool = False): "toptrack", f"top track ({seconds_to_time_string(top_track.playduration)} listened)", f"{top_track.title}", - top_track.image + "?pathhash=" + top_track.pathhash if top_track else None, + top_track.image if top_track else None, ) if top_track else StatItem( @@ -251,7 +251,7 @@ def get_track_group_stats(tracks: list[Track], is_album: bool = False): "playcount": 0, "playduration": 0, "title": track.album, - "image": track.image + "?pathhash=" + track.pathhash if track.image else None, + "image": track.image if track.image else None, } albums_map[track.albumhash]["playcount"] += 1 From 82e303f2df181480684debe2034d257153eb385d Mon Sep 17 00:00:00 2001 From: cwilvx Date: Fri, 10 Jan 2025 14:33:34 +0300 Subject: [PATCH 44/44] bug fixes --- .github/changelog.md | 43 ++++--------------------------------------- app/api/favorites.py | 6 ++++-- app/lib/populate.py | 2 +- app/lib/tagger.py | 2 +- app/lib/taglib.py | 7 ++++++- 5 files changed, 16 insertions(+), 44 deletions(-) diff --git a/.github/changelog.md b/.github/changelog.md index 09a6b020..a8bc00d7 100644 --- a/.github/changelog.md +++ b/.github/changelog.md @@ -1,40 +1,5 @@ -# What's New? +# Bug fixes - - -- Auth -- New artists/albums Sort by: last played, no. of streams, total stream duration -- Option to show now playing track info on tab title. Go to Settings > Appearance to enable -- You can select which disc to play in an album -- Internal Backup and restore - -## Improvements - -- The context menu now doesn't take forever to open up -- Merged "Save as Playlist" with "Add to Playlist" > "New Playlist" - -## Bug fixes - -- Add to queue adding to last index -1 - -## Development - -- Rewritten the whole DB layer to move stores from memory to the database. - -## THE BIG ONE API CHANGES - -- genre is no longer a string, but a struct: - -```ts -interface Genre { - name: str; - genrehash: str; -} -``` - -- Pairing via QR Code has been split into 2 endpoint: - - 1. `/getpaircode` - 2. `/pair` - -- +- Embedded thumbnails are now used when found in tracks +- Handle `AttributeError` on indexing tracks +- Add print on error in favorites diff --git a/app/api/favorites.py b/app/api/favorites.py index 52ea01ac..68dd2e28 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -78,7 +78,8 @@ def toggle_favorite(body: FavoritesAddBody): FavoritesTable.insert_item( {"hash": body.hash, "type": body.type, "extra": extra} ) - except: + except Exception as e: + print(e) return {"msg": "Failed! An error occured"}, 500 toggle_fav(body.type, body.hash) @@ -93,7 +94,8 @@ def remove_favorite(body: FavoritesAddBody): """ try: FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) - except: + except Exception as e: + print(e) return {"msg": "Failed! An error occured"}, 500 toggle_fav(body.type, body.hash) diff --git a/app/lib/populate.py b/app/lib/populate.py index 703b9a76..de7201d8 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -86,7 +86,7 @@ def get_image(_map: tuple[str, Album]): matching_tracks = AlbumStore.get_album_tracks(album.albumhash) for track in matching_tracks: - if extract_thumb(track.filepath, track.image): + if extract_thumb(track.filepath, track.albumhash + ".webp"): break diff --git a/app/lib/tagger.py b/app/lib/tagger.py index 7358cb92..69356391 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -69,7 +69,7 @@ class IndexTracks: for track in tracks: try: extract_thumb( - track["filepath"], track["trackhash"] + ".webp", overwrite=True + track["filepath"], track["albumhash"] + ".webp", overwrite=True ) except FileNotFoundError: continue diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 028b9c27..58d2c535 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -159,6 +159,11 @@ def get_tags(filepath: str, config: UserConfig): except Exception as e: # noqa: E722 return None + try: + other = tags.other + except AttributeError: + other = {} + metadata: dict[str, Any] = { "album": tags.album, "albumartists": tags.albumartist, @@ -172,7 +177,7 @@ def get_tags(filepath: str, config: UserConfig): "track": tags.track, "disc": tags.disc, "genres": tags.genre, - "copyright": " ".join(tags.other.get("copyright", [])), + "copyright": " ".join(other.get("copyright", [])), "extra": {}, }