remove deprecated db mappings

+ fix: cli password reset
+ delete old migrations
This commit is contained in:
cwilvx
2024-08-04 10:19:11 +03:00
parent 04946831ce
commit c77d0927c7
31 changed files with 40 additions and 2111 deletions
+1 -4
View File
@@ -5,13 +5,10 @@ Reads and applies the latest database migrations.
"""
import inspect
import sys
from types import ModuleType
# from app.db.sqlite.migrations import MigrationManager
from app.db.metadata import MigrationTable
from app.logger import log
from app.migrations import v1_3_0, v1_4_9
from app.migrations.base import Migration
@@ -41,7 +38,7 @@ def apply_migrations():
migrations past that index are applied and the new length
is stored as the new migration index.
"""
modules = [v1_3_0, v1_4_9]
modules = []
migrations = [get_all_migrations(m) for m in modules]
# index = MigrationManager.get_index()
-306
View File
@@ -1,306 +0,0 @@
import json
import os
import shutil
import time
from collections import OrderedDict
from sqlite3 import OperationalError
from typing import Generator
from app.db.sqlite.utils import SQLiteManager
from app.migrations.base import Migration
from app.settings import Paths
from app.utils.decorators import coroutine
from app.utils.hashing import create_hash
# playlists table
# ---------------
# 0: id
# 1: banner_pos
# 2: has_gif
# 3: image
# 4: last_updated
# 5: name
# 6: trackhashes
class m1_RemoveSmallThumbnailFolder(Migration):
"""
Removes the small thumbnail folder.
Because we are added a new folder "original" in the same directory, and the small thumbs folder is used to check if an album's thumbnail is already extracted.
So we need to remove it, to force the app to extract thumbnails for all albums.
"""
@staticmethod
def migrate():
thumbs_sm_path = Paths.get_sm_thumb_path()
thumbs_lg_path = Paths.get_lg_thumb_path()
for path in [thumbs_sm_path, thumbs_lg_path]:
if os.path.exists(path):
shutil.rmtree(path)
for path in [thumbs_sm_path, thumbs_lg_path]:
os.makedirs(path, exist_ok=True)
class m2_RemovePlaylistArtistHashes(Migration):
"""
removes the artisthashes column from the playlists table.
"""
@staticmethod
def migrate():
# remove artisthashes column
sql = "ALTER TABLE playlists DROP COLUMN artisthashes"
with SQLiteManager(userdata_db=True) as cur:
try:
cur.execute(sql)
except OperationalError:
pass
cur.close()
class m3_AddSettingsToPlaylistTable(Migration):
"""
adds the settings column and removes the banner_pos and has_gif columns
to the playlists table.
"""
@staticmethod
def migrate():
select_playlists_sql = "SELECT * FROM playlists"
with SQLiteManager(userdata_db=True) as cur:
create_playlist_table_sql = """CREATE TABLE IF NOT EXISTS playlists (
id integer PRIMARY KEY,
image text,
last_updated text not null,
name text not null,
settings text,
trackhashes text
);"""
insert_playlist_sql = """INSERT INTO playlists(
image,
last_updated,
name,
settings,
trackhashes
) VALUES(:image, :last_updated, :name, :settings, :trackhashes)
"""
cur.execute(select_playlists_sql)
# load all playlists
playlists = cur.fetchall()
# drop old playlists table
cur.execute("DROP TABLE playlists")
# create new playlists table
cur.execute(create_playlist_table_sql)
def transform_playlists(pipeline: Generator, playlists: tuple):
for playlist in playlists:
# create dict that matches the new schema
p = {
"id": playlist[0],
"name": playlist[5],
"image": playlist[3],
"trackhashes": playlist[6],
"last_updated": playlist[4],
"settings": json.dumps(
{
"has_gif": False,
"banner_pos": playlist[1],
"square_img": False,
"pinned": False,
}
),
}
pipeline.send(p)
@coroutine
def insert_playlist():
while True:
playlist = yield
p = OrderedDict(sorted(playlist.items()))
cur.execute(insert_playlist_sql, p)
# insert playlists using a coroutine
# (my first coroutine)
pipeline = insert_playlist()
transform_playlists(pipeline, playlists)
pipeline.close()
cur.close()
class m4_AddLastUpdatedToTrackTable(Migration):
"""
adds the last modified column to the tracks table.
"""
@staticmethod
def migrate():
# add last_mod column and default to current timestamp
timestamp = time.time()
sql = f"ALTER TABLE tracks ADD COLUMN last_mod text not null DEFAULT '{timestamp}'"
with SQLiteManager() as cur:
try:
cur.execute(sql)
except OperationalError:
pass
cur.close()
class m5_MovePlaylistsAndFavoritesTo10BitHashes(Migration):
"""
moves the playlists and favorites to 10 bit hashes.
"""
@staticmethod
def migrate():
def get_track_data_by_hash(trackhash: str, tracks: list[tuple]) -> tuple:
for track in tracks:
# trackhash is the 15th bit hash
if track[15] == trackhash:
# return artist, album, title
return track[4], track[1], track[13]
def get_track_by_albumhash(albumhash: str, tracks: list[tuple]) -> tuple:
for track in tracks:
# albumhash is the 3rd bit hash
if track[3] == albumhash:
# return album, albumartist
return track[1], track[2]
_base = "SELECT * FROM"
fetch_playlists_sql = f"{_base} playlists"
fetch_tracks_sql = f"{_base} tracks"
update_playlist_hashes_sql = (
"UPDATE playlists SET trackhashes = :trackhashes WHERE id = :id"
)
fetch_favorites_sql = f"{_base} favorites"
update_fav_sql = "UPDATE favorites SET hash = :hash WHERE id = :id"
remove_fav_sql = "DELETE FROM favorites WHERE id = :id"
db_tracks = []
# read tracks from db
with SQLiteManager() as cur:
cur.execute(fetch_tracks_sql)
db_tracks.extend(cur.fetchall())
cur.close()
# update playlists
with SQLiteManager(userdata_db=True) as cur:
cur.execute(fetch_playlists_sql)
playlists = cur.fetchall()
# for each playlist
for p in playlists:
pid = p[0]
# load trackhashes
trackhashes: list[str] = json.loads(p[5])
for index, t in enumerate(trackhashes):
(artist, album, title) = get_track_data_by_hash(t, db_tracks)
# create new hash
new_hash = create_hash(artist, album, title, decode=True, limit=10)
trackhashes[index] = new_hash
# convert to string
trackhashes = json.dumps(trackhashes)
# save to db
cur.execute(
update_playlist_hashes_sql, {"trackhashes": trackhashes, "id": pid}
)
cur.close()
# update favorites
with SQLiteManager(userdata_db=True) as cur:
cur.execute(fetch_favorites_sql)
favorites = cur.fetchall()
# for each favorite
for f in favorites:
fid = f[0]
fhash: str = f[1]
ftype: str = f[2] # "track" || "album"
if ftype == "album":
(album, albumartist) = get_track_by_albumhash(fhash, db_tracks)
# create new hash
new_hash = create_hash(album, albumartist, decode=True, limit=10)
# save to db
cur.execute(update_fav_sql, {"hash": new_hash, "id": fid})
continue
if ftype == "track":
(artist, album, title) = get_track_data_by_hash(fhash, db_tracks)
# create new hash
new_hash = create_hash(artist, album, title, decode=True, limit=10)
# save to db
cur.execute(update_fav_sql, {"hash": new_hash, "id": fid})
continue
# remove favorites that are not track or album. ie. artists
cur.execute(remove_fav_sql, {"id": fid})
cur.close()
class m6_RemoveAllTracks(Migration):
"""
removes all tracks from the tracks table.
"""
@staticmethod
def migrate():
sql = "DELETE FROM tracks"
with SQLiteManager() as cur:
cur.execute(sql)
cur.close()
class m7_UpdateAppSettingsTable(Migration):
@staticmethod
def migrate():
drop_table_sql = "DROP TABLE settings"
create_table_sql = """
CREATE TABLE IF NOT EXISTS settings (
id integer PRIMARY KEY,
root_dirs text NOT NULL,
exclude_dirs text,
artist_separators text NOT NULL default '/,;',
extract_feat integer NOT NULL DEFAULT 1,
remove_prod integer NOT NULL DEFAULT 1,
clean_album_title integer NOT NULL DEFAULT 1,
remove_remaster integer NOT NULL DEFAULT 1,
merge_albums integer NOT NULL DEFAULT 0,
show_albums_as_singles integer NOT NULL DEFAULT 0
);
"""
with SQLiteManager(userdata_db=True) as cur:
cur.execute(drop_table_sql)
cur.execute(create_table_sql)
-405
View File
@@ -1,405 +0,0 @@
import os
import shutil
import sqlite3
from time import time
from app.db.sqlite.utils import SQLiteManager
from app.migrations.base import Migration
from app.settings import Paths
import hashlib
from unidecode import unidecode
from app.db.sqlite.tracks import SQLiteTrackMethods as tdb
from app.db.sqlite.playlists import SQLitePlaylistMethods as pdb
from app.db.sqlite.logger.tracks import SQLiteTrackLogger as ldb
from app.utils.hashing import create_hash
def create_sha256_hash(*args: str, decode=False, limit=10) -> str:
"""
This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments.
Example use case:
- Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID.
:param args: The arguments to hash.
:param decode: Whether to decode the arguments before hashing.
:param limit: The number of characters to return.
:return: The hash.
"""
def remove_non_alnum(token: str) -> str:
token = token.lower().strip().replace(" ", "")
t = "".join(t for t in token if t.isalnum())
if t == "":
return token
return t
str_ = "".join(remove_non_alnum(t) for t in args)
if decode:
str_ = unidecode(str_)
str_ = str_.encode("utf-8")
str_ = hashlib.sha256(str_).hexdigest()
return str_[-limit:]
def create_sha1_hash(*args: str, decode=False, limit=10) -> str:
"""
This function creates a case-insensitive, non-alphanumeric chars ignoring hash from the given arguments.
Example use case:
- Creating computable IDs for duplicate artists. eg. Juice WRLD and Juice Wrld should have the same ID.
:param args: The arguments to hash.
:param decode: Whether to decode the arguments before hashing.
:param limit: The number of characters to return.
:return: The hash.
"""
def remove_non_alnum(token: str) -> str:
token = token.lower().strip().replace(" ", "")
t = "".join(t for t in token if t.isalnum())
if t == "":
return token
return t
str_ = "".join(remove_non_alnum(t) for t in args)
if decode:
str_ = unidecode(str_)
str_ = str_.encode("utf-8")
str_ = hashlib.sha1(str_).hexdigest()
return (
str_[: limit // 2] + str_[-limit // 2 :]
if limit % 2 == 0
else str_[: limit // 2] + str_[-limit // 2 - 1 :]
)
class _1AddTimestampToFavoritesTable(Migration):
"""
Adds a timestamp column to the favorites table.
"""
@staticmethod
def migrate():
# INFO: add timestamp column with automatic current timestamp
sql = f"ALTER TABLE favorites ADD COLUMN timestamp INTEGER NOT NULL DEFAULT 0"
# INFO: execute the sql
with SQLiteManager(userdata_db=True) as cur:
table_exists = cur.execute(
"select count(*) from pragma_table_info('favorites') where name = 'timestamp'"
)
table_exists = table_exists.fetchone()
if table_exists[0] == 1:
return
# INFO: Add the timestamp column to the favorites table
timestamp = int(time())
cur.execute(sql)
cur.execute(f"UPDATE favorites SET timestamp = {timestamp}")
class _2DeleteOriginalThumbnails(Migration):
"""
Original thumbnails are too large and are not needed.
"""
# TODO: Implement this migration
@staticmethod
def migrate():
imgpath = Paths.get_thumbs_path()
og_imgpath = os.path.join(imgpath, "original")
if os.path.exists(og_imgpath):
shutil.rmtree(og_imgpath)
class _3MoveScrobbleToUserId1(Migration):
"""
Updates all track logs from user id = 0 to user id = 1
"""
@staticmethod
def migrate():
sql = """
UPDATE track_logger SET userid = 1 WHERE userid = 0;
ALTER TABLE track_logger RENAME TO _track_logger;
CREATE TABLE IF NOT EXISTS track_logger (
id integer PRIMARY KEY,
trackhash text NOT NULL,
duration integer NOT NULL,
timestamp integer NOT NULL,
source text,
userid integer NOT NULL DEFAULT 1,
constraint fk_users foreign key (userid) references users(id) on delete cascade
);
INSERT INTO track_logger SELECT * FROM _track_logger;
DROP TABLE _track_logger;
"""
# INFO: Move the scrobble table to the user id 1
with SQLiteManager(userdata_db=True) as cur:
cur.executescript(sql)
cur.close()
class _4AddUserIdToFavoritesTable(Migration):
"""
Adds a userid column to the favorites table.
"""
@staticmethod
def migrate():
# check if userid column exists
exists_sql = (
"select count(*) from pragma_table_info('favorites') where name = 'userid'"
)
sql = """
ALTER TABLE favorites ADD userid INTEGER NOT NULL DEFAULT 1;
ALTER TABLE favorites RENAME TO _favorites;
CREATE TABLE IF NOT EXISTS favorites (
id integer PRIMARY KEY,
hash text not null,
type text not null,
timestamp integer not null default 0,
userid integer not null,
constraint fk_users foreign key (userid) references users(id) on delete cascade
);
INSERT INTO favorites SELECT * FROM _favorites;
DROP TABLE _favorites;
"""
with SQLiteManager(userdata_db=True) as cur:
data = cur.execute(exists_sql)
data = data.fetchone()
if data[0] == 1:
return # INFO: column already exists
cur.executescript(sql)
class _5AddUserIdToPlaylistsTable(Migration):
"""
Adds a userid column to the playlists table.
"""
@staticmethod
def migrate():
# check if userid column exists
exists_sql = (
"select count(*) from pragma_table_info('playlists') where name = 'userid'"
)
# Add the userid column to the playlists table
# Rename the old table to _playlists
# Create a new playlists table with the userid column
# Then, copy the data from the old table to the new table
# Finally, drop the old table
sql = """
ALTER TABLE playlists ADD userid INTEGER NOT NULL DEFAULT 1;
ALTER TABLE playlists RENAME TO _playlists;
CREATE TABLE IF NOT EXISTS playlists (
id integer PRIMARY KEY,
image text,
last_updated text not null,
name text not null,
settings text,
trackhashes text,
userid integer not null,
constraint fk_users foreign key (userid) references users(id) on delete cascade
);
INSERT INTO playlists SELECT * FROM _playlists;
DROP TABLE _playlists;
"""
with SQLiteManager(userdata_db=True) as cur:
# INFO: Check if the column already exists
data = cur.execute(exists_sql)
data = data.fetchone()
# INFO: If the column already exists, return
if data[0] == 1:
return # INFO: column already exists
# INFO: Execute the sql
cur.executescript(sql)
class _6MoveHashesToSha1(Migration):
"""
Moves the 10 bit item hashes from sha256 to sha1 which is
faster and more lenient on less powerful devices.
Thanks to [@tcsenpai](https:github.com/tcsenpai) for the contribution.
"""
# enabled: bool = False
# pass
# INFO: Apparentlly, every single table is affected by this migration.
# NOTE: Use generators to avoid memory issues.
@classmethod
def port_track(cls, trackhash: str):
# get the track with the track hash
track = tdb.get_track_by_trackhash(trackhash)
if track is None:
return
title = track.og_title
if track.trackhash != trackhash:
# raise ValueError("Track hash mismatch")
print("Track hash mismatch")
title = track.title
else:
print("Porting track: ", track.title)
# return the new hash
finalhash = create_sha1_hash(
", ".join(a.name for a in track.artists),
track.og_album,
title,
)
if finalhash != create_hash(
", ".join(a.name for a in track.artists), track.og_album, title
):
raise ValueError("Hash mismatch")
@classmethod
def port_album(cls, albumhash: str):
# get the first track with the album hash
track = tdb.get_track_by_albumhash(albumhash)
if track is None:
return
# return the new hash
return create_sha1_hash(
track.og_album,
", ".join(a.name for a in track.albumartists),
)
@classmethod
def port_artist(cls, artisthash: str):
# find all tracks with the artist hash
tracks = [t for t in cls.tracks if artisthash in t.artist_hashes]
if len(tracks) == 0:
return
# find the artist name
artist = [
a.name
for a in tracks[0].artists
if create_sha256_hash(a.name, decode=True) == artisthash
][0]
# return the new hash
return create_sha1_hash(artist, decode=True)
@classmethod
def migrate_favorites(cls):
with SQLiteManager(userdata_db=True) as cur:
# read all favorites
data = cur.execute("SELECT * FROM favorites")
data = data.fetchall()
for track in cls.tracks:
track.artist_hashes = "-".join(
[create_sha256_hash(a.name, decode=True) for a in track.artists]
)
for entry in data:
# hash is the 2nd column in the table
hash = entry[1]
# entry type is the 3rd column in the table
if entry[2] == "track":
newhash = cls.port_track(hash)
if newhash:
cur.execute(
f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'track'"
)
elif entry[2] == "album":
newhash = cls.port_album(hash)
if newhash:
cur.execute(
f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'album'"
)
elif entry[2] == "artist":
newhash = cls.port_artist(hash)
if newhash:
cur.execute(
f"UPDATE favorites SET hash = '{newhash}' WHERE hash = '{hash}' AND type = 'artist'"
)
@classmethod
def migrate_playlists(cls):
playlists = pdb.get_all_playlists()
for playlist in playlists:
# remove previous hashes
to_remove = [
{"trackhash": trackhash, "index": index}
for index, trackhash in enumerate(playlist.trackhashes)
]
pdb.remove_tracks_from_playlist(playlist.id, to_remove)
# add new hashes
newhashes = [
cls.port_track(trackhash) for trackhash in playlist.trackhashes
]
newhashes = [h for h in newhashes if h is not None]
pdb.add_tracks_to_playlist(playlist.id, newhashes)
print("Ported playlist: ", playlist.name)
print("Total tracks: ", len(newhashes))
@classmethod
def migrate_scrobble(cls):
# read all logs
logs = ldb.get_all()
with SQLiteManager(userdata_db=True) as cur:
# for each log, port the hash
for log in logs:
newhash = cls.port_track(log[1])
if newhash:
cur.execute(
f"UPDATE track_logger SET trackhash = '{newhash}' WHERE trackhash = '{log[1]}'"
)
@classmethod
def migrate(cls):
cls.tracks = list(tdb.get_all_tracks())
cls.migrate_favorites()
# cls.migrate_playlists()
# cls.migrate_scrobble()