TDvorak/swingmusic-extended
1
0
Fork 0
mirror of https://github.com/Dvorinka/swingmusic-extended.git synced 2026-06-05 04:53:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

port search to stores

Browse Source 
+ fix favorites
This commit is contained in:
cwilvx
2024-07-27 21:44:33 +03:00
parent 5d32536758
commit b0e904c84f
25 changed files with 428 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/lib/colorlib.py
+20 -22
View File
@@ -12,9 +12,10 @@ from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb
from app.db.sqlite.artistcolors import SQLiteArtistMethods as adb
from app.db.sqlite.utils import SQLiteManager
# from app.store.artists import ArtistStore
from app.db.userdata import ArtistData
from app.logger import log
from app.lib.errors import PopulateCancelledError
from app.store.artists import ArtistStore
from app.utils.progressbar import tqdm
PROCESS_ALBUM_COLORS_KEY = ""
@@ -100,32 +101,29 @@ class ProcessArtistColors:
"""
def __init__(self, instance_key: str) -> None:
# all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0]
all_artists = ArtistStore.get_flat_list()
global PROCESS_ARTIST_COLORS_KEY
PROCESS_ARTIST_COLORS_KEY = instance_key
with SQLiteManager() as cur:
try:
for artist in tqdm(
all_artists, desc="Processing missing artist colors"
):
if PROCESS_ARTIST_COLORS_KEY != instance_key:
raise PopulateCancelledError(
"A newer 'ProcessArtistColors' instance is running. Stopping this one."
)
try:
for artist in tqdm(all_artists, desc="Processing missing artist colors"):
if PROCESS_ARTIST_COLORS_KEY != instance_key:
raise PopulateCancelledError(
"A newer 'ProcessArtistColors' instance is running. Stopping this one."
)
exists = adb.exists(artist.artisthash, cur=cur)
# exists = adb.exists(artist.artisthash, cur=cur)
artist = ArtistData.find_one(artist.artisthash)
if artist and artist.color is not None:
continue
if exists:
continue
colors = process_color(artist.artisthash, is_album=False)
colors = process_color(artist.artisthash, is_album=False)
if colors is None:
continue
if colors is None:
continue
artist.set_colors(colors)
adb.insert_one_artist(cur, artist.artisthash, colors)
finally:
cur.close()
artist.set_colors(colors)
adb.insert_one_artist(cur, artist.artisthash, colors)
finally:
cur.close()
app/lib/index.py
+25
View File
@@ -0,0 +1,25 @@
from app.lib.mapstuff import map_favorites, map_scrobble_data
from app.lib.populate import CordinateMedia
from app.lib.tagger import IndexTracks
from app.store.folder import FolderStore
import gc
from time import time
from app.utils.threading import background
class IndexEverything:
def __init__(self) -> None:
IndexTracks(instance_key=time())
FolderStore.load_filepaths()
map_scrobble_data()
map_favorites()
# CordinateMedia(instance_key=str(time()))
gc.collect()
@background
def index_everything():
return IndexEverything()
app/lib/mapstuff.py
+68
View File
@@ -0,0 +1,68 @@
from app.db.userdata import FavoritesTable, ScrobbleTable
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from typing import Any
def map_scrobble_data():
"""
Maps scrobble data to the in-memory stores.
The scrobble data is loaded from the database and grouped by trackhash.
The album and artist scrobble data (for those tracks) are then incremented based on the data.
"""
records = ScrobbleTable.get_all(0, None)
# group records by trackhash
grouped: dict[str, dict[str, Any]] = {}
for record in records:
# aggregate playcount, playduration and lastplayed
item = grouped.setdefault(record.trackhash, {})
item["playcount"] = item.get("playcount", 0) + 1
item["playduration"] = item.get("playduration", 0) + record.duration
item["lastplayed"] = max(item.get("lastplayed", 0), record.timestamp)
# increment playcount, playduration and lastplayed for albums and artists
for trackhash, data in grouped.items():
track = TrackStore.trackhashmap.get(trackhash)
if track is None:
continue
track.increment_playcount(data["playduration"], data["lastplayed"])
album = AlbumStore.albummap.get(track.tracks[0].albumhash)
if album:
album.increment_playcount(data["playduration"], data["lastplayed"])
for artisthash in track.tracks[0].artisthashes:
artist = ArtistStore.artistmap.get(artisthash)
if artist:
artist.increment_playcount(data["playduration"], data["lastplayed"])
def map_favorites():
"""
Maps favorites data to the in-memory stores.
"""
favorites = FavoritesTable.get_all()
for entry in favorites:
if entry.type == "album":
album = AlbumStore.albummap.get(entry.hash)
if album:
album.toggle_favorite_user(entry.userid)
elif entry.type == "artist":
artist = ArtistStore.artistmap.get(entry.hash)
if artist:
artist.toggle_favorite_user(entry.userid)
elif entry.type == "track":
track = TrackStore.trackhashmap.get(entry.hash)
if track:
track.toggle_favorite_user(entry.userid)
app/lib/playlistlib.py
+3 -3
View File
@@ -10,8 +10,9 @@ from typing import Any
from PIL import Image, ImageSequence
from app import settings
from app.db.libdata import AlbumTable, TrackTable
from app.db.libdata import TrackTable
from app.models.track import Track
from app.store.albums import AlbumStore
def create_thumbnail(image: Any, img_path: str) -> str:
"""
@@ -115,8 +116,7 @@ def get_first_4_images(
if len(albums) == 4:
break
# albums = AlbumStore.get_albums_by_hashes(albums)
albums = AlbumTable.get_albums_by_albumhashes(albums)
albums = AlbumStore.get_albums_by_hashes(albums)
images = [
{
"image": album.image,
app/lib/populate.py
+11 -165
View File
@@ -1,28 +1,22 @@
from dataclasses import asdict
import os
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from typing import Generator
from requests import ConnectionError as RequestConnectionError
from requests import ReadTimeout
from app import settings
from app.db.libdata import ArtistTable
from app.db.libdata import AlbumTable, TrackTable
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
# from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
from app.db.sqlite.tracks import SQLiteTrackMethods
from app.lib.artistlib import CheckArtistImages
from app.lib.colorlib import ProcessArtistColors
from app.lib.errors import PopulateCancelledError
from app.lib.taglib import extract_thumb
from app.logger import log
from app.models import Album, Artist, Track
from app.models import Album, Artist
from app.models.lastfm import SimilarArtist
from app.requests.artists import fetch_similar_artists
from app.utils.filesystem import run_fast_scandir
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.utils.network import has_connection
from app.utils.progressbar import tqdm
@@ -35,46 +29,6 @@ remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths
POPULATE_KEY = ""
class Populate:
"""
Populates the database with all songs in the music directory
checks if the song is in the database, if not, it adds it
also checks if the album art exists in the image path, if not tries to extract it.
"""
# def __init__(self, instance_key: str) -> None:
# return
# if len(dirs_to_scan) == 0:
# log.warning(
# (
# "The root directory is not configured. "
# + "Open the app in your webbrowser to configure."
# )
# )
# return
# try:
# if dirs_to_scan[0] == "$home":
# dirs_to_scan = [settings.Paths.USER_HOME_DIR]
# except IndexError:
# pass
# files = set()
# for _dir in dirs_to_scan:
# files = files.union(run_fast_scandir(_dir, full=True)[1])
# unmodified, modified_tracks = self.remove_modified(tracks)
# untagged = files - unmodified
# if len(untagged) != 0:
# self.tag_untagged(untagged, instance_key)
# self.extract_thumb_with_overwrite(modified_tracks)
class CordinateMedia:
"""
Cordinates the extracting of thumbnails
@@ -117,102 +71,6 @@ class CordinateMedia:
log.warn(e)
return
# @staticmethod
# def remove_modified(tracks: Generator[TrackTable, None, None]):
# """
# Removes tracks from the database that have been modified
# since they were added to the database.
# """
# unmodified_paths = set()
# modified_tracks: list[TrackTable] = []
# modified_paths = set()
# for track in tracks:
# try:
# if track.last_mod == round(os.path.getmtime(track.filepath)):
# unmodified_paths.add(track.filepath)
# continue
# except (FileNotFoundError, OSError) as e:
# log.warning(e) # REVIEW More informations = good
# TrackStore.remove_track_obj(track)
# remove_tracks_by_filepaths(track.filepath)
# modified_paths.add(track.filepath)
# modified_tracks.append(track)
# TrackStore.remove_tracks_by_filepaths(modified_paths)
# remove_tracks_by_filepaths(modified_paths)
# return unmodified_paths, modified_tracks
# @staticmethod
# def tag_untagged(untagged: set[str], key: str):
# pass
# for file in tqdm(untagged, desc="Reading files"):
# if POPULATE_KEY != key:
# log.warning("'Populate.tag_untagged': Populate key changed")
# return
# tags = get_tags(file)
# if tags is not None:
# TrackTable.insert_one(tags)
# =============================================
# log.info("Found %s new tracks", len(untagged))
# # tagged_tracks: deque[dict] = deque()
# # tagged_count = 0
# favs = favdb.get_fav_tracks()
# records = dict()
# for fav in favs:
# r = records.setdefault(fav[1], set())
# r.add(fav[4])
# tagged_tracks.append(tags)
# track = Track(**tags)
# track.fav_userids = list(records.get(track.trackhash, set()))
# TrackStore.add_track(track)
# if not AlbumStore.album_exists(track.albumhash):
# AlbumStore.add_album(AlbumStore.create_album(track))
# for artist in track.artists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# for artist in track.albumartists:
# if not ArtistStore.artist_exists(artist.artisthash):
# ArtistStore.add_artist(Artist(artist.name))
# tagged_count += 1
# else:
# log.warning("Could not read file: %s", file)
# if len(tagged_tracks) > 0:
# log.info("Adding %s tracks to database", len(tagged_tracks))
# insert_many_tracks(tagged_tracks)
# log.info("Added %s/%s tracks", tagged_count, len(untagged))
# @staticmethod
# def extract_thumb_with_overwrite(tracks: list[TrackTable]):
# """
# Extracts the thumbnail from a list of filepaths,
# overwriting the existing thumbnail if it exists,
# for modified files.
# """
# for track in tracks:
# try:
# extract_thumb(track.filepath, track.image, overwrite=True)
# except FileNotFoundError:
# continue
def get_image(_map: tuple[str, Album]):
"""
@@ -228,25 +86,11 @@ def get_image(_map: tuple[str, Album]):
if POPULATE_KEY != instance_key:
raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed")
matching_tracks = filter(
lambda t: t.albumhash == album.albumhash,
TrackTable.get_tracks_by_albumhash(album.albumhash),
)
matching_tracks = AlbumStore.get_album_tracks(album.albumhash)
try:
track = next(matching_tracks)
extracted = extract_thumb(track.filepath, track.image)
while not extracted:
try:
track = next(matching_tracks)
extracted = extract_thumb(track.filepath, track.image)
except StopIteration:
break
return
except StopIteration:
pass
for track in matching_tracks:
if extract_thumb(track.filepath, track.image):
break
def get_cpu_count():
@@ -274,7 +118,7 @@ class ProcessTrackThumbnails:
# filter out albums that already have thumbnails
albums = filter(
lambda album: album.albumhash not in processed, AlbumTable.get_all()
lambda album: album.albumhash not in processed, AlbumStore.get_flat_list()
)
albums = list(albums)
@@ -330,7 +174,9 @@ class FetchSimilarArtistsLastFM:
processed = ".".join(a.artisthash for a in processed)
# filter out artists that already have similar artists
artists = filter(lambda a: a.artisthash not in processed, ArtistTable.get_all())
artists = filter(
lambda a: a.artisthash not in processed, ArtistStore.get_flat_list()
)
artists = list(artists)
# process the rest
app/lib/searchlib.py
+16 -25
View File
@@ -9,7 +9,8 @@ from unidecode import unidecode
from app import models
from app.config import UserConfig
from app.db.libdata import AlbumTable, ArtistTable, TrackTable
# from app.db.libdata import AlbumTable, ArtistTable, TrackTable
# from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.models.enums import FavType
@@ -19,9 +20,9 @@ from app.serializers.album import serialize_for_card_many as serialize_albums
from app.serializers.artist import serialize_for_cards
from app.serializers.track import serialize_track, serialize_tracks
# from app.store.albums import AlbumStore
# from app.store.artists import ArtistStore
# from app.store.tracks import TrackStore
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from app.utils.remove_duplicates import remove_duplicates
@@ -54,8 +55,7 @@ class Limit:
class SearchTracks:
def __init__(self, query: str) -> None:
self.query = query
# self.tracks = TrackStore.tracks
self.tracks = TrackTable.get_all()
self.tracks = TrackStore.get_flat_list()
def __call__(self) -> List[models.Track]:
"""
@@ -78,8 +78,7 @@ class SearchTracks:
class SearchArtists:
def __init__(self, query: str) -> None:
self.query = query
# self.artists = ArtistStore.artists
self.artists = ArtistTable.get_all()
self.artists = ArtistStore.get_flat_list()
def __call__(self):
"""
@@ -101,8 +100,7 @@ class SearchArtists:
class SearchAlbums:
def __init__(self, query: str) -> None:
self.query = query
# self.albums = AlbumStore.albums
self.albums = AlbumTable.get_all()
self.albums = AlbumStore.get_flat_list()
def __call__(self) -> List[models.Album]:
"""
@@ -169,9 +167,9 @@ class TopResults:
def collect_all():
all_items: list[_type] = []
all_items.extend(ArtistTable.get_all())
all_items.extend(TrackTable.get_all())
all_items.extend(AlbumTable.get_all())
all_items.extend(ArtistStore.get_flat_list())
all_items.extend(TrackStore.get_flat_list())
all_items.extend(TrackStore.get_flat_list())
return all_items, get_titles(all_items)
@@ -194,7 +192,7 @@ class TopResults:
return {"type": "track", "item": item}
if isinstance(item, models.Album):
tracks = TrackTable.get_tracks_by_albumhash(item.albumhash)
tracks = TrackStore.get_tracks_by_albumhash(item.albumhash)
tracks = remove_duplicates(tracks)
try:
@@ -212,19 +210,13 @@ class TopResults:
track_count = 0
duration = 0
tracks = TrackTable.get_tracks_by_artisthash(item.artisthash)
tracks = TrackStore.get_tracks_by_artisthash(item.artisthash)
tracks = remove_duplicates(tracks)
for track in tracks:
track_count += 1
duration += track.duration
# album_count = AlbumStore.count_albums_by_artisthash(item.artisthash)
# item.set_trackcount(track_count)
# item.set_albumcount(album_count)
# item.set_duration(duration)
return {"type": "artist", "item": item}
@staticmethod
@@ -235,8 +227,7 @@ class TopResults:
tracks.extend(SearchTracks(query)())
if item["type"] == "album":
t = TrackTable.get_tracks_by_albumhash(item["item"].albumhash)
# t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash)
t = TrackStore.get_tracks_by_albumhash(item["item"].albumhash)
t.sort(key=lambda x: x.last_mod)
# if there are less than the limit, get more tracks
@@ -249,7 +240,7 @@ class TopResults:
if item["type"] == "artist":
# t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash)
t = TrackTable.get_tracks_by_artisthash(item["item"].artisthash)
t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash)
# if there are less than the limit, get more tracks
if len(t) < limit:
@@ -271,7 +262,7 @@ class TopResults:
if item["type"] == "artist":
# albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash)
albums = AlbumTable.get_albums_by_artisthash(item["item"].artisthash)
albums = AlbumStore.get_albums_by_artisthash(item["item"].artisthash)
# if there are less than the limit, get more albums
if len(albums) < limit:
app/lib/tagger.py
+6 -34
View File
@@ -1,12 +1,9 @@
import gc
import os
from pprint import pprint
from time import time
from app import settings
from app.config import UserConfig
from app.db.libdata import ArtistTable
from app.db.libdata import TrackTable
from app.lib.populate import CordinateMedia
# from app.lib.populate import CordinateMedia
from app.lib.taglib import extract_thumb, get_tags
from app.models.album import Album
from app.models.artist import Artist
@@ -17,7 +14,6 @@ from app.utils.parsers import get_base_album_title
from app.utils.progressbar import tqdm
from app.logger import log
from app.utils.threading import background
POPULATE_KEY: float = 0
@@ -142,7 +138,6 @@ class IndexTracks:
print("Done")
# class IndexAlbums:
def create_albums():
albums = dict()
all_tracks: list[Track] = TrackTable.get_all()
@@ -165,7 +160,7 @@ def create_albums():
"playduration": track.playduration,
"title": track.album,
"trackcount": 1,
"extra": {}
"extra": {},
}
else:
album = albums[track.albumhash]
@@ -187,13 +182,11 @@ def create_albums():
genres.append(genre)
album["genres"] = genres
album["genrehashes"] = " ".join([g['genrehash'] for g in genres])
album["genrehashes"] = " ".join([g["genrehash"] for g in genres])
album["base_title"], _ = get_base_album_title(album["og_title"])
del genres
# AlbumTable.remove_all()
# AlbumTable.insert_many(list(albums.values()))
return [Album(**album) for album in albums.values()]
@@ -227,9 +220,7 @@ def create_artists():
"playduration": track.playduration,
"trackcount": None,
"tracks": (
{track.trackhash}
if thisartist.get("in_track", True)
else set()
{track.trackhash} if thisartist.get("in_track", True) else set()
),
"extra": {},
}
@@ -261,7 +252,7 @@ def create_artists():
genres.append(genre)
artist["genres"] = genres
artist["genrehashes"] = " ".join([g['genrehash'] for g in genres])
artist["genrehashes"] = " ".join([g["genrehash"] for g in genres])
artist["name"] = sorted(artist["names"])[0]
# INFO: Delete temporary keys
@@ -272,25 +263,6 @@ def create_artists():
# INFO: Delete local variables
del genres
# ArtistTable.remove_all()
# ArtistTable.insert_many(list(artists.values()))
# del artists
return [Artist(**artist) for artist in artists.values()]
class IndexEverything:
def __init__(self) -> None:
IndexTracks(instance_key=time())
# IndexAlbums()
# IndexArtists()
FolderStore.load_filepaths()
# pass
# CordinateMedia(instance_key=str(time()))
gc.collect()
@background
def index_everything():
return IndexEverything()