TDvorak/swingmusic-extended
1
0
Fork 0
mirror of https://github.com/Dvorinka/swingmusic-extended.git synced 2026-06-04 20:43:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

load settings from db, use api to change settings

Browse Source 
+ add route to get all settings
+ add route to set any setting
+ add untested migration to add settings into settings db
+ compress json in api responses using FlaskCompress
+ serve gziped assets if browser accepts encoded files
+ misc
This commit is contained in:
mungai-njoroge
2023-08-24 15:52:09 +03:00
parent e3a61c109b
commit 71cab5f5ea
22 changed files with 437 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+19 -3
View File
@@ -3,18 +3,34 @@ This module combines all API blueprints into a single Flask app instance.
"""
from flask import Flask
from flask_compress import Compress
from flask_cors import CORS
from app.api import (album, artist, favorites, folder, imgserver, playlist,
search, settings, track, colors)
from app.api import (
album,
artist,
colors,
favorites,
folder,
imgserver,
playlist,
search,
settings,
track,
)
def create_api():
"""
Creates the Flask instance, registers modules and registers all the API blueprints.
"""
app = Flask(__name__, static_url_path="")
app = Flask(__name__)
CORS(app)
Compress(app)
app.config["COMPRESS_MIMETYPES"] = [
"application/json",
]
with app.app_context():
app.register_blueprint(album.api)
app/api/artist.py
+5 -5
View File
@@ -7,8 +7,7 @@ from collections import deque
from flask import Blueprint, request
from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
from app.db.sqlite.lastfm.similar_artists import \
SQLiteLastFMSimilarArtists as fmdb
from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb
from app.models import Album, FavType, Track
from app.serializers.album import serialize_for_card_many
from app.serializers.track import serialize_tracks
@@ -49,8 +48,8 @@ class ArtistsCache:
artists: deque[CacheEntry] = deque(maxlen=1)
# THE ABOVE IS SET TO MAXLEN=1 TO AVOID A BUG THAT I WAS TOO LAZY TO INVESTIGATE
# ARTIST TRACKS SOMEHOW DISAPPEARED FOR SOME REASON I COULDN'T UNDERSTAND. BY
# DISAPPEARING I MEAN AN ARTIST YOU ARE SURE HAS 150 TRACKS ONLY SHOWING LIKE 3 IN
# THE ARTIST PAGE. 🤷🏿 (TODO: MAYBE FIX THIS BUG?)
# DISAPPEARING I MEAN AN ARTIST YOU ARE SURE HAS 150 TRACKS ONLY SHOWING
# LIKE 3 IN THE ARTIST PAGE. 🤷🏿 (TODO: MAYBE FIX THIS BUG?)
@classmethod
def get_albums_by_artisthash(cls, artisthash: str) -> tuple[list[Album], int]:
@@ -325,4 +324,5 @@ def get_similar_artists(artisthash: str):
return {"artists": similar[:limit]}
# TODO: Rewrite this file using generators where possible
# TODO: Rewrite this file using generators where possible
app/api/settings.py
+103 -16
View File
@@ -1,17 +1,16 @@
from flask import Blueprint, request
from app import settings
from app.logger import log
from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.lib import populate
from app.lib.watchdogg import Watcher as WatchDog
from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.logger import log
from app.settings import ParserFlags, Paths, set_flag
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from app.utils.generators import get_random_str
from app.utils.threading import background
from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
from app.store.artists import ArtistStore
api = Blueprint("settings", __name__, url_prefix="/")
@@ -21,13 +20,24 @@ def get_child_dirs(parent: str, children: list[str]):
return [_dir for _dir in children if _dir.startswith(parent) and _dir != parent]
def reload_everything():
def reload_everything(instance_key: str):
"""
Reloads all stores using the current database items
"""
TrackStore.load_all_tracks()
AlbumStore.load_albums()
ArtistStore.load_artists()
try:
TrackStore.load_all_tracks(instance_key)
except Exception as e:
log.error(e)
try:
AlbumStore.load_albums(instance_key=instance_key)
except Exception as e:
log.error(e)
try:
ArtistStore.load_artists(instance_key)
except Exception as e:
log.error(e)
@background
@@ -35,15 +45,16 @@ def rebuild_store(db_dirs: list[str]):
"""
Restarts the watchdog and rebuilds the music library.
"""
instance_key = get_random_str()
log.info("Rebuilding library...")
TrackStore.remove_tracks_by_dir_except(db_dirs)
reload_everything()
reload_everything(instance_key)
key = get_random_str()
try:
populate.Populate(key=key)
populate.Populate(instance_key=instance_key)
except populate.PopulateCancelledError:
reload_everything()
reload_everything(instance_key)
return
WatchDog().restart()
@@ -51,6 +62,7 @@ def rebuild_store(db_dirs: list[str]):
log.info("Rebuilding library... ✅")
# I freaking don't know what this function does anymore
def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]):
"""
Params:
@@ -96,7 +108,7 @@ def add_root_dirs():
sdb.remove_root_dirs(db_dirs)
if incoming_home:
finalize([_h], [], [settings.Paths.USER_HOME_DIR])
finalize([_h], [], [Paths.USER_HOME_DIR])
return {"root_dirs": [_h]}
# ---
@@ -127,3 +139,78 @@ def get_root_dirs():
dirs = sdb.get_root_dirs()
return {"dirs": dirs}
# maps settings to their parser flags
mapp = {
"artist_separators": ParserFlags.ARTIST_SEPARATORS,
"extract_feat": ParserFlags.EXTRACT_FEAT,
"remove_prod": ParserFlags.REMOVE_PROD,
"clean_album_title": ParserFlags.CLEAN_ALBUM_TITLE,
"remove_remaster": ParserFlags.REMOVE_REMASTER_FROM_TRACK,
"merge_albums": ParserFlags.MERGE_ALBUM_VERSIONS,
}
@api.route("/settings/", methods=["GET"])
def get_all_settings():
"""
Get all settings from the database.
"""
settings = sdb.get_all_settings()
key_list = list(mapp.keys())
s = {}
for key in key_list:
val_index = key_list.index(key)
try:
s[key] = settings[val_index]
if type(s[key]) == int:
s[key] = bool(s[key])
if type(s[key]) == str:
s[key] = str(s[key]).split(",")
except IndexError:
s[key] = None
root_dirs = sdb.get_root_dirs()
s["root_dirs"] = root_dirs
return {
"settings": s,
}
@background
def reload_all_for_set_setting():
reload_everything(get_random_str())
@api.route("/settings/set", methods=["POST"])
def set_setting():
key = request.get_json().get("key")
value = request.get_json().get("value")
if key is None or value is None or key == "root_dirs":
return {"msg": "Invalid arguments!"}, 400
root_dir = sdb.get_root_dirs()
if not root_dir:
return {"msg": "No root directories set!"}, 400
if key not in mapp:
return {"msg": "Invalid key!"}, 400
sdb.set_setting(key, value)
if mapp[key] is not False:
flag = mapp[key]
set_flag(flag, value)
reload_all_for_set_setting()
return {"result": value}
app/config.py
-61
View File
@@ -1,61 +0,0 @@
"""
Module for managing the JSON config file.
"""
import json
from enum import Enum
from typing import Type
from app.settings import Db
class ConfigKeys(Enum):
ROOT_DIRS = ("root_dirs", list[str])
PLAYLIST_DIRS = ("playlist_dirs", list[str])
USE_ART_COLORS = ("use_art_colors", bool)
DEFAULT_ART_COLOR = ("default_art_color", str)
SHUFFLE_MODE = ("shuffle_mode", str)
REPEAT_MODE = ("repeat_mode", str)
AUTOPLAY_ON_START = ("autoplay_on_start", bool)
VOLUME = ("volume", int)
def __init__(self, key_name: str, data_type: Type):
self.key_name = key_name
self.data_type = data_type
def get_data_type(self) -> Type:
return self.data_type
class ConfigManager:
def __init__(self, config_file_path: str):
self.config_file_path = config_file_path
def read_config(self):
try:
with open(self.config_file_path) as f:
return json.load(f)
except FileNotFoundError:
return {}
# in case of errors, return an empty dict
def write_config(self, config_data):
with open(self.config_file_path, "w") as f:
json.dump(config_data, f, indent=4)
def get_value(self, key: ConfigKeys):
config_data = self.read_config()
value = config_data.get(key.key_name)
if value is not None:
return key.get_data_type()(value)
def set_value(self, key: ConfigKeys, value):
config_data = self.read_config()
config_data[key.key_name] = value
self.write_config(config_data)
settings = ConfigManager(Db.get_json_config_path())
a = settings.get_value(ConfigKeys.ROOT_DIRS)
app/db/sqlite/queries.py
+6 -1
View File
@@ -26,7 +26,12 @@ CREATE TABLE IF NOT EXISTS settings (
id integer PRIMARY KEY,
root_dirs text NOT NULL,
exclude_dirs text,
artist_separators text
artist_separators text NOT NULL default '/,;,&',
extract_feat integer NOT NULL DEFAULT 1,
remove_prod integer NOT NULL DEFAULT 1,
clean_album_title integer NOT NULL DEFAULT 1,
remove_remaster integer NOT NULL DEFAULT 1,
merge_albums integer NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS lastfm_similar_artists (
app/db/sqlite/settings.py
+52 -18
View File
@@ -1,5 +1,8 @@
from pprint import pprint
from typing import Any
from app.db.sqlite.utils import SQLiteManager
from app.utils.wintools import win_replace_slash
from app.settings import FromFlags
class SettingsSQLMethods:
@@ -7,6 +10,28 @@ class SettingsSQLMethods:
Methods for interacting with the settings table.
"""
@staticmethod
def get_all_settings():
"""
Gets all settings from the database.
"""
sql = "SELECT * FROM settings WHERE id = 1"
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql)
settings = cur.fetchone()
cur.close()
# if root_dirs not set
if settings is None:
return []
# print
# omit id, root_dirs, and exclude_dirs
return settings[3:]
@staticmethod
def get_root_dirs() -> list[str]:
"""
@@ -90,25 +115,34 @@ class SettingsSQLMethods:
return [_dir[0] for _dir in dirs]
@staticmethod
def add_artist_separators(seps: set[str]):
"""
Adds a set of artist separators to the userdata table.
"""
# TODO: Implement
def get_settings() -> dict[str, Any]:
pass
@staticmethod
def get_artist_separators() -> set[str]:
"""
Gets a set of artist separators from the userdata table.
"""
# TODO: Implement
pass
def set_setting(key: str, value: Any):
sql = f"UPDATE settings SET {key} = :value WHERE id = 1"
@staticmethod
def remove_artist_separators(seps: set[str]):
"""
Removes a set of artist separators from the userdata table.
"""
# TODO: Implement
pass
if type(value) == bool:
value = str(int(value))
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql, {"value": value})
def load_settings():
s = SettingsSQLMethods.get_all_settings()
# artist separators
db_separators: str = s[0]
db_separators = db_separators.replace(" ", "")
separators = db_separators.split(",")
separators = set(db_separators)
FromFlags.ARTIST_SEPARATORS = separators
# boolean settings
FromFlags.EXTRACT_FEAT = bool(s[1])
FromFlags.REMOVE_PROD = bool(s[2])
FromFlags.CLEAN_ALBUM_TITLE = bool(s[3])
FromFlags.REMOVE_REMASTER_FROM_TRACK = bool(s[4])
FromFlags.MERGE_ALBUM_VERSIONS = bool(s[5])
app/db/sqlite/utils.py
+3 -3
View File
@@ -8,7 +8,7 @@ import time
from typing import Optional
from app.models import Album, Playlist, Track
from app.settings import Db
from app import settings
def tuple_to_track(track: tuple):
@@ -88,10 +88,10 @@ class SQLiteManager:
if self.test_db_path:
db_path = self.test_db_path
else:
db_path = Db.get_app_db_path()
db_path = settings.Db.get_app_db_path()
if self.userdata_db:
db_path = Db.get_userdata_db_path()
db_path = settings.Db.get_userdata_db_path()
self.conn = sqlite3.connect(
db_path,
app/lib/populate.py
+16 -11
View File
@@ -45,9 +45,9 @@ class Populate:
also checks if the album art exists in the image path, if not tries to extract it.
"""
def __init__(self, key: str) -> None:
def __init__(self, instance_key: str) -> None:
global POPULATE_KEY
POPULATE_KEY = key
POPULATE_KEY = instance_key
validate_tracks()
validate_albums()
@@ -80,7 +80,7 @@ class Populate:
untagged = files - unmodified
if len(untagged) != 0:
self.tag_untagged(untagged, key)
self.tag_untagged(untagged, instance_key)
self.extract_thumb_with_overwrite(modified_tracks)
@@ -110,7 +110,7 @@ class Populate:
if Ping()():
FetchSimilarArtistsLastFM()
ArtistStore.load_artists()
ArtistStore.load_artists(instance_key)
@staticmethod
def remove_modified(tracks: Generator[Track, None, None]):
@@ -273,12 +273,17 @@ class FetchSimilarArtistsLastFM:
artists = ArtistStore.artists
with Pool(processes=cpu_count()) as pool:
results = list(
tqdm(
pool.imap_unordered(save_similar_artists, artists),
total=len(artists),
desc="Fetching similar artists",
try:
results = list(
tqdm(
pool.imap_unordered(save_similar_artists, artists),
total=len(artists),
desc="Fetching similar artists",
)
)
)
list(results)
list(results)
# any exception that can be raised by the pool
except:
pass
app/lib/searchlib.py
-1
View File
@@ -242,7 +242,6 @@ class TopResults:
if item["type"] == "artist":
t = TrackStore.get_tracks_by_artisthash(item["item"].artisthash)
t.sort(key=lambda x: x.last_mod)
# if there are less than the limit, get more tracks
if len(t) < limit:
app/migrations/__init__.py
+1
View File
@@ -25,6 +25,7 @@ migrations: list[list[Migration]] = [
v1_3_0.AddLastUpdatedToTrackTable,
v1_3_0.MovePlaylistsAndFavoritesTo10BitHashes,
v1_3_0.RemoveAllTracks,
v1_3_0.UpdateAppSettingsTable,
]
]
app/migrations/v1_3_0/__init__.py
+23
View File
@@ -277,3 +277,26 @@ class RemoveAllTracks(Migration):
with SQLiteManager() as cur:
cur.execute(sql)
cur.close()
class UpdateAppSettingsTable(Migration):
@staticmethod
def migrate():
drop_table_sql = "DROP TABLE settings"
create_table_sql = """
CREATE TABLE IF NOT EXISTS settings (
id integer PRIMARY KEY,
root_dirs text NOT NULL,
exclude_dirs text,
artist_separators text NOT NULL default '/,;,&',
extract_feat integer NOT NULL DEFAULT 1,
remove_prod integer NOT NULL DEFAULT 1,
clean_album_title integer NOT NULL DEFAULT 1,
remove_remaster integer NOT NULL DEFAULT 1,
merge_albums integer NOT NULL DEFAULT 0
);
"""
with SQLiteManager(userdata_db=True) as cur:
cur.execute(drop_table_sql)
cur.execute(create_table_sql)
app/models/album.py
+2 -2
View File
@@ -124,7 +124,7 @@ class Album:
if "various artists" in artists:
return True
substrings = [
substrings = {
"the essential",
"best of",
"greatest hits",
@@ -136,7 +136,7 @@ class Album:
"great hits",
"biggest hits",
"the hits",
]
}
for substring in substrings:
if substring in self.title.lower():
app/periodic_scan.py
+7 -12
View File
@@ -3,14 +3,10 @@ This module contains functions for the server
"""
import time
from app.logger import log
from app.lib.populate import Populate, PopulateCancelledError
from app.utils.generators import get_random_str
from app.utils.network import Ping
from app.utils.threading import background
from app.settings import ParserFlags, get_flag, get_scan_sleep_time
from app.utils.generators import get_random_str
from app.utils.threading import background
@background
@@ -21,14 +17,13 @@ def run_periodic_scans():
# ValidateAlbumThumbs()
# ValidatePlaylistThumbs()
try:
Populate(key=get_random_str())
except PopulateCancelledError:
pass
run_periodic_scan = True
while run_periodic_scan:
run_periodic_scan = get_flag(ParserFlags.DO_PERIODIC_SCANS)
while get_flag(ParserFlags.DO_PERIODIC_SCANS):
try:
Populate(key=get_random_str())
Populate(instance_key=get_random_str())
except PopulateCancelledError:
pass
app/settings.py
+9 -3
View File
@@ -2,6 +2,7 @@
Contains default configs
"""
import os
from typing import Any
join = os.path.join
@@ -162,29 +163,34 @@ class FromFlags:
CLEAN_ALBUM_TITLE = True
REMOVE_REMASTER_FROM_TRACK = True
SHOW_ALBUM_VERSION = True
DO_PERIODIC_SCANS = True
PERIODIC_SCAN_INTERVAL = 300 # seconds
MERGE_ALBUM_VERSIONS = False
ARTIST_SEPARATORS = {",", "/", ";", "&"}
class ParserFlags():
# TODO: Find a way to eliminate this class without breaking typings
class ParserFlags:
EXTRACT_FEAT = "EXTRACT_FEAT"
REMOVE_PROD = "REMOVE_PROD"
CLEAN_ALBUM_TITLE = "CLEAN_ALBUM_TITLE"
SHOW_ALBUM_VERSION = "SHOW_ALBUM_VERSION"
REMOVE_REMASTER_FROM_TRACK = "REMOVE_REMASTER_FROM_TRACK"
DO_PERIODIC_SCANS = "DO_PERIODIC_SCANS"
PERIODIC_SCAN_INTERVAL = "PERIODIC_SCAN_INTERVAL"
MERGE_ALBUM_VERSIONS = "MERGE_ALBUM_VERSIONS"
ARTIST_SEPARATORS = "ARTIST_SEPARATORS"
def get_flag(flag: ParserFlags) -> bool:
return getattr(FromFlags, flag)
def set_flag(flag: ParserFlags, value: Any):
setattr(FromFlags, flag, value)
def get_scan_sleep_time() -> int:
return FromFlags.PERIODIC_SCAN_INTERVAL
app/setup/__init__.py
+13 -3
View File
@@ -1,11 +1,13 @@
"""
Prepares the server for use.
"""
from app.db.sqlite.settings import load_settings
from app.setup.files import create_config_dir
from app.setup.sqlite import run_migrations, setup_sqlite
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
from app.store.tracks import TrackStore
from app.utils.generators import get_random_str
def run_setup():
@@ -13,6 +15,14 @@ def run_setup():
setup_sqlite()
run_migrations()
TrackStore.load_all_tracks()
AlbumStore.load_albums()
ArtistStore.load_artists()
try:
load_settings()
except IndexError:
# settings table is empty
pass
instance_key = get_random_str()
TrackStore.load_all_tracks(instance_key)
AlbumStore.load_albums(instance_key)
ArtistStore.load_artists(instance_key)
app/store/albums.py
+21 -6
View File
@@ -9,6 +9,8 @@ from app.models import Album, Track
from ..utils.hashing import create_hash
from .tracks import TrackStore
ALBUM_LOAD_KEY = ""
class AlbumStore:
albums: list[Album] = []
@@ -25,16 +27,21 @@ class AlbumStore:
)
@classmethod
def load_albums(cls):
def load_albums(cls, instance_key: str):
"""
Loads all albums from the database into the store.
"""
global ALBUM_LOAD_KEY
ALBUM_LOAD_KEY = instance_key
cls.albums = []
albumhashes = set(t.albumhash for t in TrackStore.tracks)
for albumhash in tqdm(albumhashes, desc="Loading albums"):
for albumhash in tqdm(albumhashes, desc=f"Loading {instance_key}"):
if instance_key != ALBUM_LOAD_KEY:
return
for track in TrackStore.tracks:
if track.albumhash == albumhash:
cls.albums.append(cls.create_album(track))
@@ -67,15 +74,21 @@ class AlbumStore:
@classmethod
def get_albums_by_albumartist(
cls, artisthash: str, limit: int, exclude: str
cls, artisthash: str, limit: int, exclude: str
) -> list[Album]:
"""
Returns N albums by the given albumartist, excluding the specified album.
"""
albums = [album for album in cls.albums if artisthash in album.albumartists_hashes]
albums = [
album for album in cls.albums if artisthash in album.albumartists_hashes
]
albums = [album for album in albums if create_hash(album.base_title) != create_hash(exclude)]
albums = [
album
for album in albums
if create_hash(album.base_title) != create_hash(exclude)
]
if len(albums) > limit:
random.shuffle(albums)
@@ -110,7 +123,9 @@ class AlbumStore:
"""
Returns all albums by the given artist.
"""
return [album for album in cls.albums if artisthash in album.albumartists_hashes]
return [
album for album in cls.albums if artisthash in album.albumartists_hashes
]
@classmethod
def count_albums_by_artisthash(cls, artisthash: str):
app/store/artists.py
+9 -1
View File
@@ -10,20 +10,28 @@ from app.utils.bisection import UseBisection
from .albums import AlbumStore
from .tracks import TrackStore
ARTIST_LOAD_KEY = ""
class ArtistStore:
artists: list[Artist] = []
@classmethod
def load_artists(cls):
def load_artists(cls, instance_key: str):
"""
Loads all artists from the database into the store.
"""
global ARTIST_LOAD_KEY
ARTIST_LOAD_KEY = instance_key
cls.artists = get_all_artists(TrackStore.tracks, AlbumStore.albums)
# db_artists: list[tuple] = list(ardb.get_all_artists())
for artist in tqdm(ardb.get_all_artists(), desc="Loading artists"):
if instance_key != ARTIST_LOAD_KEY:
return
cls.map_artist_color(artist)
@classmethod
app/store/tracks.py
+12 -3
View File
@@ -6,15 +6,19 @@ from app.models import Track
from app.utils.bisection import UseBisection
from app.utils.remove_duplicates import remove_duplicates
TRACKS_LOAD_KEY = ""
class TrackStore:
tracks: list[Track] = []
@classmethod
def load_all_tracks(cls):
def load_all_tracks(cls, instance_key: str):
"""
Loads all tracks from the database into the store.
"""
global TRACKS_LOAD_KEY
TRACKS_LOAD_KEY = instance_key
cls.tracks = list(tdb.get_all_tracks())
@@ -22,6 +26,9 @@ class TrackStore:
fav_hashes = " ".join([t[1] for t in fav_hashes])
for track in tqdm(cls.tracks, desc="Loading tracks"):
if instance_key != TRACKS_LOAD_KEY:
return
if track.trackhash in fav_hashes:
track.is_favorite = True
@@ -153,12 +160,14 @@ class TrackStore:
return remove_duplicates(tracks)
@classmethod
def get_tracks_by_artisthash(cls, artisthash: str) -> list[Track]:
def get_tracks_by_artisthash(cls, artisthash: str):
"""
Returns all tracks matching the given artist. Duplicate tracks are removed.
"""
tracks = [t for t in cls.tracks if artisthash in t.artist_hashes]
return remove_duplicates(tracks)
tracks = remove_duplicates(tracks)
tracks.sort(key=lambda x: x.last_mod)
return tracks
@classmethod
def get_tracks_in_path(cls, path: str):
app/utils/parsers.py
+3 -2
View File
@@ -1,13 +1,14 @@
import re
from app.enums.album_versions import AlbumVersionEnum
from app.settings import get_flag, ParserFlags
def split_artists(src: str, custom_seps: set[str] = {}):
def split_artists(src: str):
"""
Splits a string of artists into a list of artists.
"""
separators = {",", ";", "/"}.union(custom_seps)
separators = get_flag(ParserFlags.ARTIST_SEPARATORS)
for sep in separators:
src = src.replace(sep, "߸")