diff --git a/server/app/functions.py b/server/app/functions.py index ccc040a9..6ad8e2ab 100644 --- a/server/app/functions.py +++ b/server/app/functions.py @@ -1,27 +1,16 @@ """ This module contains functions for the server """ -import datetime import os -from pprint import pprint -import random import time -from dataclasses import asdict from io import BytesIO -from typing import List, Type import requests from app import api from app import helpers -from app import instances -from app import models from app import settings -from app.lib import albumslib -from app.lib import folderslib from app.lib import watchdoge -from app.lib.taglib import get_tags -from app.lib.taglib import return_album_art -from app.logger import Log +from app.lib.populate import Populate from PIL import Image from progress.bar import Bar @@ -31,6 +20,7 @@ def reindex_tracks(): """ Checks for new songs every 5 minutes. """ + is_underway = False while True: populate() @@ -48,129 +38,11 @@ def start_watchdog(): def populate(): - """ - Populate the database with all songs in the music directory - - checks if the song is in the database, if not, it adds it - also checks if the album art exists in the image path, if not tries to - extract it. - """ - start = time.time() - db_tracks = instances.tracks_instance.get_all_tracks() - tagged_tracks = [] - albums = [] - folders = set() - - files = helpers.run_fast_scandir(settings.HOME_DIR, [".flac", ".mp3"], full=True)[1] - - _bar = Bar("Checking files", max=len(files)) - for track in db_tracks: - if track["filepath"] in files: - files.remove(track["filepath"]) - _bar.next() - - _bar.finish() - - Log(f"Found {len(files)} untagged files") - - _bar = Bar("Tagging files", max=len(files)) - for file in files: - tags = get_tags(file) - foldername = os.path.dirname(file) - folders.add(foldername) - - if tags is not None: - tagged_tracks.append(tags) - api.DB_TRACKS.append(tags) - - _bar.next() - _bar.finish() - - Log(f"Tagged {len(tagged_tracks)} tracks") - - pre_albums = [] - - for t in tagged_tracks: - a = { - "title": t["album"], - "artist": t["albumartist"], - } - - if a not in pre_albums: - pre_albums.append(a) - - exist_count = 0 - _bar = Bar("Creating albums", max=len(pre_albums)) - for aa in pre_albums: - albumindex = albumslib.find_album(aa["title"], aa["artist"]) - - if albumindex is None: - track = [ - track - for track in tagged_tracks - if track["album"] == aa["title"] - and track["albumartist"] == aa["artist"] - ][0] - - album = albumslib.create_album(track) - api.ALBUMS.append(album) - albums.append(album) - - instances.album_instance.insert_album(asdict(album)) - - else: - exist_count += 1 - - _bar.next() - - _bar.finish() - - Log(f"{exist_count} of {len(albums)} were already in the database") - - _bar = Bar("Creating tracks", max=len(tagged_tracks)) - for track in tagged_tracks: - try: - album_index = albumslib.find_album(track["album"], track["albumartist"]) - album = api.ALBUMS[album_index] - - track["image"] = album.image - upsert_id = instances.tracks_instance.insert_song(track) - - track["_id"] = {"$oid": str(upsert_id)} - api.TRACKS.append(models.Track(track)) - except TypeError: - # Bug: some albums are not found although they exist in `api.ALBUMS`. It has something to do with the bisection method used or sorting. Not sure yet. - pass - - _bar.next() - - _bar.finish() - - Log(f"Added {len(tagged_tracks)} new tracks and {len(albums)} new albums") - - _bar = Bar("Creating folders", max=len(folders)) - for folder in folders: - if folder not in api.VALID_FOLDERS: - api.VALID_FOLDERS.add(folder) - fff = folderslib.create_folder(folder) - api.FOLDERS.append(fff) - - _bar.next() - - _bar.finish() - - Log(f"Created {len(api.FOLDERS)} folders") - - end = time.time() - - print( - str(datetime.timedelta(seconds=round(end - start))) - + " elapsed for " - + str(len(files)) - + " files" - ) + pop = Populate() + pop.run() +@helpers.background def fetch_image_path(artist: str) -> str or None: """ Returns a direct link to an artist image. @@ -189,6 +61,7 @@ def fetch_image_path(artist: str) -> str or None: return None +@helpers.background def fetch_artist_images(): """Downloads the artists images""" @@ -203,9 +76,8 @@ def fetch_artist_images(): _bar = Bar("Processing images", max=len(artists)) for artist in artists: - file_path = ( - helpers.app_dir + "/images/artists/" + artist.replace("/", "::") + ".webp" - ) + file_path = (helpers.app_dir + "/images/artists/" + + artist.replace("/", "::") + ".webp") if not os.path.exists(file_path): img_path = fetch_image_path(artist) @@ -227,8 +99,7 @@ def fetch_album_bio(title: str, albumartist: str): Returns the album bio for a given album. """ last_fm_url = "http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={}&artist={}&album={}&format=json".format( - settings.LAST_FM_API_KEY, albumartist, title - ) + settings.LAST_FM_API_KEY, albumartist, title) try: response = requests.get(last_fm_url) @@ -237,8 +108,14 @@ def fetch_album_bio(title: str, albumartist: str): return None try: - bio = data["album"]["wiki"]["summary"].split('") +def send_thumbnail(path: str): + fpath = join(THUMB_PATH, path) + exists = os.path.exists(fpath) + + if exists: + return send_from_directory(THUMB_PATH, path) + + return {"msg": "Not found"}, 404 + + +@app.route("/artist/") +def send_artist_image(path: str): + print(ARTIST_PATH) + fpath = join(ARTIST_PATH, path) + exists = os.path.exists(fpath) + + if exists: + return send_from_directory(ARTIST_PATH, path) + + return {"msg": "Not found"}, 404 + + +if __name__ == "__main__": + app.run(threaded=True, port=9877) diff --git a/server/app/lib/albumslib.py b/server/app/lib/albumslib.py index 3433bff0..46d6f4ae 100644 --- a/server/app/lib/albumslib.py +++ b/server/app/lib/albumslib.py @@ -10,12 +10,11 @@ from app import api from app import functions from app import instances from app import models +from app import settings +from app.lib import taglib from app.lib import trackslib from progress.bar import Bar -from app.lib import taglib -from app import settings - def get_all_albums() -> List[models.Album]: """ @@ -66,7 +65,8 @@ def find_album(albumtitle: str, artist: str) -> int or None: iter += 1 mid = (left + right) // 2 - if api.ALBUMS[mid].title == albumtitle and api.ALBUMS[mid].artist == artist: + if api.ALBUMS[mid].title == albumtitle and api.ALBUMS[ + mid].artist == artist: return mid if api.ALBUMS[mid].title < albumtitle: @@ -114,17 +114,15 @@ def get_album_image(album: list) -> str: Gets the image of an album. """ - uri = settings.IMG_THUMB_URI - for track in album: img_p = gen_random_path() exists = taglib.extract_thumb(track["filepath"], webp_path=img_p) if exists: - return uri + img_p + return img_p - return uri + use_defaults() + return use_defaults() def get_album_tracks(album: str, artist: str) -> List: @@ -157,8 +155,7 @@ def create_album(track) -> models.Album: album["date"] = album_tracks[0]["date"] album["artistimage"] = urllib.parse.quote_plus( - album_tracks[0]["albumartist"] + ".webp" - ) + album_tracks[0]["albumartist"] + ".webp") album["image"] = get_album_image(album_tracks) diff --git a/server/app/lib/populate.py b/server/app/lib/populate.py new file mode 100644 index 00000000..4a929471 --- /dev/null +++ b/server/app/lib/populate.py @@ -0,0 +1,171 @@ +from dataclasses import asdict +from os import path + +from app import api +from app import settings +from app.helpers import run_fast_scandir +from app.instances import album_instance +from app.instances import tracks_instance +from app.lib import folderslib +from app.lib.albumslib import create_album +from app.lib.albumslib import find_album +from app.lib.taglib import get_tags +from app.logger import Log +from app.models import Track +from progress.bar import Bar + + +class Populate: + """ + Populate the database with all songs in the music directory + + checks if the song is in the database, if not, it adds it + also checks if the album art exists in the image path, if not tries to + extract it. + """ + + def __init__(self) -> None: + self.files = [] + self.db_tracks = [] + self.tagged_tracks = [] + self.folders = set() + self.pre_albums = [] + self.albums = [] + + self.files = run_fast_scandir(settings.HOME_DIR, [".flac", ".mp3"])[1] + self.db_tracks = tracks_instance.get_all_tracks() + + def run(self): + self.check_untagged() + + if len(self.files) == 0: + return + + self.tag_files() + self.create_pre_albums() + self.create_albums() + self.create_tracks() + self.create_folders() + + def check_untagged(self): + """ + Loops through all the tracks in db tracks removing each + from the list of tagged tracks if it exists. + We will now only have untagged tracks left in `files`. + """ + bar = Bar("Checking untagged", max=len(self.db_tracks)) + for track in self.db_tracks: + if track["filepath"] in self.files: + self.files.remove(track["filepath"]) + bar.next() + + bar.finish() + Log(f"Found {len(self.files)} untagged tracks") + + def tag_files(self): + """ + Loops through all the untagged files and tags them. + """ + bar = Bar("Tagging files", max=len(self.files)) + for file in self.files: + tags = get_tags(file) + folder = path.dirname(file) + self.folders.add(folder) + + if tags is not None: + self.tagged_tracks.append(tags) + api.DB_TRACKS.append(tags) + + bar.next() + bar.finish() + Log(f"Tagged {len(self.tagged_tracks)} files") + + def create_pre_albums(self): + """ + Creates pre-albums for the all tagged tracks. + """ + bar = Bar("Creating pre-albums", max=len(self.tagged_tracks)) + for track in self.tagged_tracks: + album = {"title": track["album"], "artist": track["albumartist"]} + + if album not in self.pre_albums: + self.pre_albums.append(album) + bar.next() + + bar.finish() + Log(f"Created {len(self.pre_albums)} pre-albums") + + def create_albums(self): + """ + Uses the pre-albums to create new albums and add them to the database. + """ + exist_count = 0 + + bar = Bar("Creating albums", max=len(self.pre_albums)) + for album in self.pre_albums: + index = find_album(album["title"], album["artist"]) + + if index is None: + try: + track = [ + track for track in self.tagged_tracks + if track["album"] == album["title"] + and track["albumartist"] == album["artist"] + ][0] + + album = create_album(track) + api.ALBUMS.append(album) + self.albums.append(album) + + album_instance.insert_album(asdict(album)) + + except IndexError: + print("😠\n") + print(album) + + else: + exist_count += 1 + + bar.next() + bar.finish() + Log(f"{exist_count} of {len(self.pre_albums)} albums were already in the database" + ) + + def create_tracks(self): + """ + Loops through all the tagged tracks creating complete track objects using the `models.Track` model. + """ + bar = Bar("Creating tracks", max=len(self.tagged_tracks)) + failed_count = 0 + for track in self.tagged_tracks: + try: + album_index = find_album(track["album"], track["albumartist"]) + album = api.ALBUMS[album_index] + track["image"] = album.image + upsert_id = tracks_instance.insert_song(track) + track["_id"] = {"$oid": str(upsert_id)} + api.TRACKS.append(Track(track)) + except: + # Bug: some albums are not found although they exist in `api.ALBUMS`. It has something to do with the bisection method used or sorting. Not sure yet. + failed_count += 1 + bar.next() + bar.finish() + + Log(f"Added {len(self.tagged_tracks) - failed_count} of {len(self.tagged_tracks)} new tracks and {len(self.albums)} new albums" + ) + + def create_folders(self): + """ + Creates the folder objects for all the tracks. + """ + bar = Bar("Creating folders", max=len(self.folders)) + old_f_count = len(api.FOLDERS) + for folder in self.folders: + api.VALID_FOLDERS.add(folder) + fff = folderslib.create_folder(folder) + api.FOLDERS.append(fff) + bar.next() + + bar.finish() + + Log(f"Created {len(self.folders)} new folders") diff --git a/server/app/lib/trackslib.py b/server/app/lib/trackslib.py index 14751552..1aa25373 100644 --- a/server/app/lib/trackslib.py +++ b/server/app/lib/trackslib.py @@ -26,8 +26,11 @@ def create_all_tracks() -> List[models.Track]: except FileNotFoundError: instances.tracks_instance.remove_song_by_id(track["_id"]["$oid"]) api.DB_TRACKS.remove(track) + try: + tracks.append(models.Track(track)) + except KeyError: + print(track) - tracks.append(models.Track(track)) _bar.next() _bar.finish() diff --git a/server/app/models.py b/server/app/models.py index 862c6f7b..27e8c45b 100644 --- a/server/app/models.py +++ b/server/app/models.py @@ -32,6 +32,7 @@ class Track: discnumber: int def __init__(self, tags): + self.trackid = tags["_id"]["$oid"] self.title = tags["title"] self.artists = tags["artists"].split(", ") diff --git a/server/setup/nginx.setup.sh b/server/setup/nginx.setup.sh old mode 100644 new mode 100755 index 5c9dedd1..10026f6f --- a/server/setup/nginx.setup.sh +++ b/server/setup/nginx.setup.sh @@ -2,4 +2,4 @@ # 2. create symbolic links for each file in sites-available to sites-enable sudo cp ./nginx-sites/* /etc/nginx/sites-available -sudo ln -s /etc/nginx/sites-available/* /etc/nginx/sites-enabled -f \ No newline at end of file +sudo ln -s /etc/nginx/sites-available/* /etc/nginx/sites-enabled -f diff --git a/server/start.sh b/server/start.sh index 5cceeae4..b86f0ca5 100755 --- a/server/start.sh +++ b/server/start.sh @@ -3,6 +3,10 @@ # $ppath manage.py #python manage.py -gpath=$(poetry run which gunicorn) +gpath=$(poetry run which gunicorn) +cd app +"$gpath" -b 0.0.0.0:9877 -w 4 --threads=2 "imgserver:app" & +echo "Booted image server" +cd ../ "$gpath" -b 0.0.0.0:9876 -w 1 --threads=4 "manage:create_app()" #--log-level=debug