From ca31054f488d0b156edafff77850f746b59c2612 Mon Sep 17 00:00:00 2001
From: cwilvx <geoffreymungai45@gmail.com>
Date: Thu, 15 Aug 2024 17:07:34 +0300
Subject: [PATCH] fix: folder endpoint returning same track for different files
 of the same trackhash

+ fix: chunked streaming. return instead of yield chunks
---
 TODO.md               |   2 +-
 app/api/auth.py       |   6 +-
 app/api/stream.py     | 186 ++++++++++++++++++++++++++++++++++++++----
 app/db/userdata.py    |   4 +-
 app/lib/tagger.py     |  50 ++++++++++--
 app/lib/taglib.py     |   3 +-
 app/lib/transcoder.py |  78 ++++++++++++++++++
 app/lib/watchdogg.py  |  95 +++++++++++++--------
 app/models/track.py   |  30 ++++++-
 app/store/albums.py   |  40 +++++----
 app/store/artists.py  | 103 ++++++++++++-----------
 app/store/folder.py   |  10 ++-
 app/utils/__init__.py |   5 +-
 docs/README.md        |   0
 docs/streaming.md     |   3 +
 docs/watchdog.md      |   0
 poetry.lock           |  30 ++++++-
 pyproject.toml        |   1 +
 18 files changed, 508 insertions(+), 138 deletions(-)
 create mode 100644 app/lib/transcoder.py
 create mode 100644 docs/README.md
 create mode 100644 docs/streaming.md
 create mode 100644 docs/watchdog.md

diff --git a/TODO.md b/TODO.md
index ab4ebd22..6edebdd1 100644
--- a/TODO.md
+++ b/TODO.md
@@ -49,4 +49,4 @@
 - Duplicates on search
 - Audio stops on ending
 - Show users on account settings when logged in as admin and show users on login is disabled.
-- 
\ No newline at end of file
+- Save both filepath and trackhash in favorites and playlists
\ No newline at end of file
diff --git a/app/api/auth.py b/app/api/auth.py
index de36e78a..ff593452 100644
--- a/app/api/auth.py
+++ b/app/api/auth.py
@@ -108,14 +108,14 @@ class PairDeviceQuery(BaseModel):
     code: str = Field("", description="The code")
 
 
-@api.post("/pair")
+@api.get("/pair")
 @jwt_required(optional=True)
-def pair_with_code(body: PairDeviceQuery):
+def pair_with_code(query: PairDeviceQuery):
     """
     Get an access token by sending a pair code. NOTE: A code can only be used once!
     """
     global pair_token
-    token = pair_token.get(body.code, None)
+    token = pair_token.get(query.code, None)
 
     if token:
         pair_token = {}
diff --git a/app/api/stream.py b/app/api/stream.py
index b876d5d3..69bb6240 100644
--- a/app/api/stream.py
+++ b/app/api/stream.py
@@ -3,12 +3,16 @@ Contains all the track routes.
 """
 
 import os
+import tempfile
+import time
+from typing import Literal
 
 from flask import send_file, request, Response
 from flask_openapi3 import APIBlueprint, Tag
 from pydantic import BaseModel, Field
 from app.api.apischemas import TrackHashSchema
 from app.lib.trackslib import get_silence_paddings
+from app.lib.transcoder import start_transcoding
 
 from app.store.tracks import TrackStore
 from app.utils.files import guess_mime_type
@@ -17,10 +21,36 @@ bp_tag = Tag(name="File", description="Audio files")
 api = APIBlueprint("track", __name__, url_prefix="/file", abp_tags=[bp_tag])
 
 
+class TransCodeStore:
+    map: dict[str, str] = {}
+
+    @classmethod
+    def add_file(cls, trackhash: str, filepath: str):
+        cls.map[trackhash] = filepath
+
+    @classmethod
+    def remove_file(cls, trackhash: str):
+        del cls.map[trackhash]
+
+    @classmethod
+    def find(cls, trackhash: str):
+        return cls.map.get(trackhash)
+
+
 class SendTrackFileQuery(BaseModel):
     filepath: str = Field(
         description="The filepath to play (if available)", default=None
     )
+    quality: Literal["original", "1411", "800", "600", "320", "256", "128", "96"] = (
+        Field(
+            "320",
+            description="The quality of the audio file. Options: original, 1411, 1024, 512, 320, 256, 128, 96",
+        )
+    )
+    container: Literal["mp3", "aac", "flac", "webm", "ogg"] = Field(
+        "flac",
+        description="The container format of the audio file. Options: mp3, aac, flac, webm, ogg",
+    )
 
 
 @api.get("/<trackhash>/legacy")
@@ -29,6 +59,8 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery):
     Get a playable audio file without Range support
 
     Returns a playable audio file that corresponds to the given filepath. Falls back to track hash if filepath is not found.
+
+    NOTE: Does not support range requests or transcoding.
     """
     trackhash = path.trackhash
     filepath = query.filepath
@@ -37,7 +69,6 @@ def send_track_file_legacy(path: TrackHashSchema, query: SendTrackFileQuery):
     track = None
     tracks = TrackStore.get_tracks_by_filepaths([filepath])
 
-
     if len(tracks) > 0 and os.path.exists(filepath):
         track = tracks[0]
     else:
@@ -66,10 +97,17 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery):
     Get a playable audio file with Range headers support
 
     Returns a playable audio file that corresponds to the given filepath. Falls back to track hash if filepath is not found.
+
+    Transcoding can be done by sending the quality and container query parameters.
+
+    **NOTES:**
+    - Transcoded streams report incorrect duration during playback (idk why! FFMPEG gurus we need your help here).
+    - The quality parameter is the desired bitrate in kbps.
+    - The mp3 container is the best container for upto 320kbps (and has better duration reporting). The flac container allows for higher bitrates but it produces dramatically larger files (when transcoding from lossy formats).
+    - You can get the transcoded bitrate by checking the X-Transcoded-Bitrate header on the first request's response.
     """
     trackhash = path.trackhash
     filepath = query.filepath
-    msg = {"msg": "File Not Found"}
 
     # If filepath is provided, try to send that
     track = None
@@ -91,13 +129,87 @@ def send_track_file(path: TrackHashSchema, query: SendTrackFileQuery):
                     break
 
     if track is not None:
-        audio_type = guess_mime_type(filepath)
-        return send_file_as_chunks(track.filepath, audio_type)
+        if query.quality == "original":
+            return send_file_as_chunks(track.filepath)
 
-    return msg, 404
+        # prevent requesting over transcoding
+        max_bitrate = track.bitrate
+        requested_bitrate = int(query.quality)
+
+        if query.container != "flac":
+            # drop to 320 for non-flac containers
+            requested_bitrate = min(320, requested_bitrate)
+
+        quality = f"{min(max_bitrate, requested_bitrate)}k"
+        return transcode_and_stream(trackhash, track.filepath, quality, query.container)
+
+    return {"msg": "File Not Found"}, 404
 
 
-def send_file_as_chunks(filepath: str, audio_type: str) -> Response:
+def transcode_and_stream(trackhash: str, filepath: str, bitrate: str, container: str):
+    """
+    Initiates transcoding and returns the first chunk of the transcoded file.
+
+    The other chunks are streamed on subsequent requests and are rerouted to `send_file_as_chunks`.
+    """
+    temp_file = TransCodeStore.find(trackhash)
+    if temp_file is not None:
+        return send_file_as_chunks(temp_file)
+
+    format_params = {
+        "mp3": ["-c:a", "libmp3lame"],
+        "aac": ["-c:a", "aac"],
+        "webm": ["-c:a", "libopus"],
+        "ogg": ["-c:a", "libvorbis"],
+        "flac": ["-c:a", "flac"],
+        "wav": ["-c:a", "pcm_s16le"],
+    }
+
+    # Create a temporary file
+    format = f".{container}" if container in format_params.keys() else ".flac"
+    container_args = (
+        format_params[container]
+        if container in format_params.keys()
+        else format_params["flac"]
+    )
+    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=format)
+    temp_filename = temp_file.name
+    temp_file.close()
+
+    TransCodeStore.add_file(trackhash, temp_filename)
+    start_transcoding(filepath, temp_filename, bitrate, container_args)
+
+    chunk_size = 1024 * 512  # 0.5MB
+    file_size = os.path.getsize(filepath)
+
+    def generate():
+        # Poll for the output file
+        while (
+            not os.path.exists(temp_filename)
+            or os.path.getsize(temp_filename) < chunk_size
+        ):
+            print(f"Waiting for transcoding to complete... filename: {temp_filename}")
+            time.sleep(0.1)  # Wait for 100ms before checking again
+
+        with open(temp_filename, "rb") as file:
+            file.seek(0)
+            return file.read(chunk_size)
+
+    audio_type = guess_mime_type(temp_filename)
+    response = Response(
+        generate(),
+        206,
+        mimetype=audio_type,
+        content_type=audio_type,
+        direct_passthrough=True,
+    )
+    response.headers.add("Content-Range", f"bytes {0}-{chunk_size}/{file_size}")
+    response.headers.add("Accept-Ranges", "bytes")
+    response.headers.add("X-Transcoded-Bitrate", bitrate)
+    return response
+
+
+def send_file_as_chunks(filepath: str) -> Response:
     """
     Returns a Response object that streams the file in chunks.
     """
@@ -129,25 +241,69 @@ def send_file_as_chunks(filepath: str, audio_type: str) -> Response:
             file.seek(start)
             remaining_bytes = end - start + 1
 
-            while remaining_bytes > 0:
+            retry_count = 0
+            max_retries = 10  # 5 * 100ms = 500ms total wait time
+
+            while remaining_bytes > 0 or retry_count < max_retries:
+                if retry_count == max_retries:
+                    print("💚 sending final chunk! ...")
+                    return (
+                        file.read(os.path.getsize(filepath) - file.tell()),
+                        file.tell(),
+                        True,
+                    )
+                print("\n\n")
+                print(f"file: {filepath}")
+                print(f"start: {start}")
+                print(f"end: {end}")
+                print(f"filesize: {os.path.getsize(filepath)}")
+                print(f"⭐ (O) Remaining bytes: {remaining_bytes}")
+                print(f"⭐ Remaining bytes: {remaining_bytes}")
+                print(f"⭐ Cursor position: {file.tell()}")
                 # Read the chunk size or all the remaining bytes
+
+                print(f"💚 remaining_bytes: {remaining_bytes}")
+                print(f"💚 retry_count: {retry_count}")
+
+                if remaining_bytes < chunk_size:
+                    time.sleep(0.25)
+                    retry_count += 1
+                    remaining_bytes = os.path.getsize(filepath) - file.tell()
+                    continue
+
                 chunk = file.read(min(chunk_size, remaining_bytes))
-                yield chunk
+                if chunk:
+                    remaining_bytes -= len(chunk)
+                    return chunk, file.tell(), False
+                else:
+                    # If no data is read, wait for 100ms before retrying
+                    time.sleep(0.25)
+                    retry_count += 1
 
-                # Update the remaining bytes
-                remaining_bytes -= len(chunk)
+                    # update remaining bytes
+                    remaining_bytes = os.path.getsize(filepath) - file.tell()
+                    print(f"▶ Remaining bytes: {remaining_bytes}")
 
+            return None, 0, True
+
+    data, position, is_final = generate_chunks()
+
+    audio_type = guess_mime_type(filepath)
     response = Response(
-        generate_chunks(),
-        206,  # Partial Content status code
+        response=data,
+        status=206,  # Partial Content status code
         mimetype=audio_type,
         content_type=audio_type,
         direct_passthrough=True,
     )
-    response.headers.add("Content-Range", f"bytes {start}-{end}/{file_size}")
-    response.headers.add("Accept-Ranges", "bytes")
-    response.headers.add("Content-Length", str(end - start + 1))
 
+    bytes_to_add = chunk_size if not is_final else 0
+    response.headers.add(
+        "Content-Range",
+        f"bytes {start}-{position}/{os.path.getsize(filepath) + bytes_to_add}",
+    )
+    response.headers.add("Accept-Ranges", "bytes")
+    response.headers.add("Content-Length", str(len(data or [])))
     return response
 
 
diff --git a/app/db/userdata.py b/app/db/userdata.py
index 0c5f13bc..2d33ff1e 100644
--- a/app/db/userdata.py
+++ b/app/db/userdata.py
@@ -1,5 +1,5 @@
 import datetime
-from typing import Any
+from typing import Any, Literal
 from sqlalchemy import (
     JSON,
     Boolean,
@@ -409,7 +409,7 @@ class LibDataTable(Base):
         )
 
     @classmethod
-    def find_one(cls, hash: str, type: str):
+    def find_one(cls, hash: str, type: Literal["album", "artist"]):
         result = cls.execute(
             select(cls).where((cls.itemhash == hash) & (cls.itemtype == type))
         )
diff --git a/app/lib/tagger.py b/app/lib/tagger.py
index 92efbf81..4c12cce3 100644
--- a/app/lib/tagger.py
+++ b/app/lib/tagger.py
@@ -8,11 +8,14 @@ from app.models.album import Album
 from app.models.artist import Artist
 from app.models.track import Track
 from app.store.folder import FolderStore
+from app.store.tracks import TrackStore
+from app.utils import flatten
 from app.utils.filesystem import run_fast_scandir
 from app.utils.parsers import get_base_album_title
 from app.utils.progressbar import tqdm
 
 from app.logger import log
+from app.utils.remove_duplicates import remove_duplicates
 
 POPULATE_KEY: float = 0
 
@@ -136,9 +139,27 @@ class IndexTracks:
         print("Done")
 
 
-def create_albums():
+def create_albums(_trackhashes: list[str] = []) -> list[tuple[Album, set[str]]]:
+    """
+    Creates album objects using the indexed tracks. Takes in an optional
+    list of trackhashes to create the albums from. If no list is provided,
+    all tracks are used.
+
+    The trackhashes are passed when creating albums from the watchdogg module.
+
+    Returns a list of tuples containing the album and the trackhashes in the album.
+    ie:
+
+    >>> list[tuple[Album, set[str]]]
+    """
     albums = dict()
-    all_tracks: list[Track] = TrackTable.get_all()
+
+    if _trackhashes:
+        all_tracks: list[Track] = TrackStore.get_tracks_by_trackhashes(_trackhashes)
+    else:
+        all_tracks: list[Track] = TrackStore.get_flat_list()
+
+    all_tracks = remove_duplicates(all_tracks)
 
     for track in all_tracks:
         if track.albumhash not in albums:
@@ -192,9 +213,28 @@ def create_albums():
     return list(albums.values())
 
 
-# class IndexArtists:
-def create_artists():
-    all_tracks: list[Track] = TrackTable.get_all()
+def create_artists(
+    artisthashes: list[str] = [],
+) -> list[tuple[Artist, set[str], set[str]]]:
+    """
+    Creates artist objects using the indexed tracks. Takes in an optional
+    list of artisthashes to create the artists from. If no list is provided,
+    all tracks are used.
+
+    Returns a list of tuples containing the artist, the trackhashes for the artist
+    and the albumhashes for the artist.
+    ie:
+
+    >>> list[tuple[Artist, set[str], set[str]]]
+    """
+    if artisthashes:
+        all_tracks: list[Track] = flatten(
+            [TrackStore.get_tracks_by_artisthash(hash) for hash in artisthashes]
+        )
+    else:
+        all_tracks: list[Track] = TrackStore.get_flat_list()
+
+    all_tracks = remove_duplicates(all_tracks)
     artists = dict()
 
     for track in all_tracks:
diff --git a/app/lib/taglib.py b/app/lib/taglib.py
index 96fce4f3..820715f7 100644
--- a/app/lib/taglib.py
+++ b/app/lib/taglib.py
@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+import math
 import os
 from io import BytesIO
 from pathlib import Path
@@ -206,7 +207,7 @@ def get_tags(filepath: str, config: UserConfig):
     to_round = ["bitrate", "duration"]
     for prop in to_round:
         try:
-            setattr(tags, prop, round(getattr(tags, prop)))
+            setattr(tags, prop, math.floor(getattr(tags, prop)))
         except TypeError:
             setattr(tags, prop, 0)
 
diff --git a/app/lib/transcoder.py b/app/lib/transcoder.py
new file mode 100644
index 00000000..befb978c
--- /dev/null
+++ b/app/lib/transcoder.py
@@ -0,0 +1,78 @@
+from app.utils.threading import background
+
+
+import subprocess
+
+
+@background
+def start_transcoding(
+    input_path: str, output_path: str, bitrate: str, container_args: list[str], compression_level: int = 12
+):
+    """
+    Starts a background transcoding process for an audio file.
+
+    This function uses FFmpeg to transcode an audio file from one format to another,
+    with specified bitrate and container format. It runs as a background task.
+
+    Args:
+        input_path (str): The path to the input audio file.
+        output_path (str): The path where the transcoded file will be saved.
+        bitrate (str): The desired bitrate for the output file (e.g., "128k").
+        container_args (list[str]): FFmpeg arguments specific to the output container format.
+        compression_level (int): Compression level (0-9, default: 6).
+
+    Returns:
+        None
+
+    Note:
+        This function is decorated with @background, which means it runs asynchronously.
+        The actual transcoding process is handled by FFmpeg in a subprocess.
+        The function will print status messages about the transcoding process.
+    """
+    # Base command
+    command = [
+        "ffmpeg",
+        "-i",
+        input_path,
+        "-map_metadata", "0",  # Add this line to copy metadata
+        "-b:a",
+        bitrate,
+        "-vn",
+        "-compression_level",
+        str(compression_level),
+        # REVIEW: Idk what any flag below this point does!
+        "-movflags",
+        "faststart+frag_keyframe+empty_moov",
+        "-write_xing",
+        "0",
+        "-fflags",
+        "+bitexact",
+    ]
+
+    # Add format-specific parameters
+    command.extend(container_args)
+
+    # Add output path and overwrite flag
+    command.extend([output_path, "-y"])
+
+    process = subprocess.Popen(
+        command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
+    print(f"Started transcoding process with PID: {process.pid}")
+
+    try:
+        # Wait for the process to complete
+        process.wait()
+        print(f"Transcoding process (PID: {process.pid}) completed")
+    except KeyboardInterrupt:
+        print(f"Transcoding interrupted. Terminating process (PID: {process.pid})")
+    finally:
+        # Ensure the process is terminated
+        try:
+            process.terminate()
+            process.wait(timeout=5)  # Wait up to 5 seconds for graceful termination
+        except subprocess.TimeoutExpired:
+            print(
+                f"Process (PID: {process.pid}) did not terminate gracefully. Killing..."
+            )
+            process.kill()
\ No newline at end of file
diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py
index 0b577596..b22bbbd1 100644
--- a/app/lib/watchdogg.py
+++ b/app/lib/watchdogg.py
@@ -8,17 +8,20 @@ import sqlite3
 import time
 
 from watchdog.events import PatternMatchingEventHandler
+from watchdog.observers.api import BaseObserverSubclassCallable
 from watchdog.observers import Observer
 
 from app import settings
 from app.config import UserConfig
-from app.db.sqlite.tracks import SQLiteManager
+from app.db.libdata import TrackTable
+from app.db.userdata import LibDataTable
 from app.lib.colorlib import process_color
+from app.lib.tagger import create_albums, create_artists
 from app.lib.taglib import extract_thumb, get_tags
 from app.logger import log
 from app.models import Artist, Track
 from app.store.albums import AlbumStore
-from app.store.artists import ArtistStore
+from app.store.artists import ArtistMapEntry, ArtistStore
 from app.store.tracks import TrackStore
 
 
@@ -27,7 +30,7 @@ class Watcher:
     Contains the methods for initializing and starting watchdog.
     """
 
-    observers: list[Observer] = []
+    observers: list[BaseObserverSubclassCallable] = []
 
     def __init__(self):
         self.observer = Observer()
@@ -126,10 +129,10 @@ class Watcher:
         self.run()
 
 
-def handle_colors(cur: sqlite3.Cursor, albumhash: str):
-    exists = aldb.exists(albumhash, cur)
+def handle_color(albumhash: str):
+    entry = LibDataTable.find_one(albumhash, "album")
 
-    if exists:
+    if entry and entry.color:
         return
 
     colors = process_color(albumhash, is_album=True)
@@ -137,7 +140,12 @@ def handle_colors(cur: sqlite3.Cursor, albumhash: str):
     if colors is None:
         return
 
-    aldb.insert_one_album(cur=cur, albumhash=albumhash, colors=json.dumps(colors))
+    if entry is None:
+        LibDataTable.insert_one(
+            {"itemhash": albumhash, "color": colors[0], "itemtype": "album"}
+        )
+    else:
+        LibDataTable.update_one(albumhash, {"color": colors[0]})
 
     return colors
 
@@ -152,38 +160,42 @@ def add_track(filepath: str) -> None:
     TrackStore.remove_track_by_filepath(filepath)
 
     config = UserConfig()
-    tags = get_tags(filepath, artist_separators=config.artistSeparators)
+    tags = get_tags(filepath, config)
 
     # if the track is somehow invalid, return
     if tags is None or tags["bitrate"] == 0 or tags["duration"] == 0:
         return
 
-    colors = None
-
-    with SQLiteManager() as cur:
-        db.insert_one_track(tags, cur)
-        extracted = extract_thumb(filepath, tags["albumhash"] + ".webp")
-
-        if not extracted:
-            return
-
-        colors = handle_colors(cur, tags["albumhash"])
+    TrackTable.insert_one(tags)
+    extract_thumb(filepath, tags["albumhash"] + ".webp", overwrite=True)
 
+    colors = handle_color(tags["albumhash"])
     track = Track(**tags)
     TrackStore.add_track(track)
 
-    if not AlbumStore.album_exists(track.albumhash):
-        album = AlbumStore.create_album(track)
-        album.set_colors(colors)
-        AlbumStore.add_album(album)
+    # SECTION: Index album
+    albumentry = AlbumStore.albummap.get(track.albumhash)
 
-    artists: list[Artist] = track.artists + track.albumartists  # type: ignore
+    if albumentry is None:
+        album, trackhashes = create_albums([track.trackhash])[0]
+        AlbumStore.index_new_album(album, trackhashes)
+    else:
+        trackhash_exists = track.trackhash in albumentry.trackhashes
+
+        if not trackhash_exists:
+            albumentry.trackhashes.add(track.trackhash)
+            albumentry.album.trackcount += 1
+            albumentry.set_color(colors[0]) if colors else None
+
+    # SECTION: Index artist
+    artists = create_artists(track.artisthashes)
 
     for artist in artists:
-        if not ArtistStore.artist_exists(artist.artisthash):
-            ArtistStore.add_artist(Artist(artist.name))
-
-    extract_thumb(filepath, track.image, overwrite=True)
+        ArtistStore.artistmap[artist[0].artisthash] = ArtistMapEntry(
+            artist=artist[0],
+            albumhashes=artist[1],
+            trackhashes=artist[2],
+        )
 
 
 def remove_track(filepath: str) -> None:
@@ -287,16 +299,33 @@ class Handler(PatternMatchingEventHandler):
         NOT FIRED IN WINDOWS
         """
         try:
-            self.files_to_process.remove(event.src_path)
-            if os.path.getsize(event.src_path) > 0:
+            # Get initial file size
+            initial_size = os.path.getsize(event.src_path)
+
+            # Wait for 10 seconds
+            time.sleep(10)
+
+            # Check if file size has changed
+            current_size = os.path.getsize(event.src_path)
+
+            if current_size > 0 and current_size == initial_size:
                 path = self.get_abs_path(event.src_path)
                 add_track(path)
+                # Remove from processing list only after successful processing
+                self.files_to_process.remove(event.src_path)
+            else:
+                # File is still being modified or has been deleted
+                log.info(
+                    f"File {event.src_path} is still being modified. Skipping processing for now."
+                )
         except FileNotFoundError:
             # file was closed and deleted.
-            pass
+            log.info(f"File {event.src_path} was closed and deleted before processing.")
         except ValueError:
-            # file was removed from the list by another event handler.
-            pass
+            # file was already removed from the list by another event handler.
+            log.info(
+                f"File {event.src_path} was already removed from the processing list."
+            )
 
     def on_modified(self, event):
         # this event handler is triggered twice on windows
@@ -317,7 +346,7 @@ class Handler(PatternMatchingEventHandler):
 
         if current_size == previous_size:
             # Wait for a short duration to ensure the file write operation is complete
-            time.sleep(5)
+            time.sleep(10)
 
             # Check the file size again
             try:
diff --git a/app/models/track.py b/app/models/track.py
index e9f7710f..4f493f8c 100644
--- a/app/models/track.py
+++ b/app/models/track.py
@@ -57,8 +57,10 @@ class Track:
 
     def toggle_favorite_user(self, userid: int):
         """
-        Adds or removes the given user from the list of users
-        who have favorited the track.
+        Toggles the favorite status of the track for a given user.
+
+        Args:
+            userid (int): The ID of the user toggling the favorite status.
         """
         if userid in self.fav_userids:
             self.fav_userids.remove(userid)
@@ -66,6 +68,11 @@ class Track:
             self.fav_userids.append(userid)
 
     def __post_init__(self):
+        """
+        Performs post-initialization processing on the track object.
+        This includes setting original values, processing artists and genres,
+        and removing duplicate artists.
+        """
         self.og_title = self.title
         self.og_album = self.album
 
@@ -97,11 +104,13 @@ class Track:
             and not seen_albumartists.add(tuple(d.items()))
         ]
 
+        self.recreate_trackhash()
         self.config = None
 
     def split_artists(self):
         """
-        Splits the artists and albumartists based on the given separators, and updates the artisthashes.
+        Splits the artists and albumartists based on the given separators,
+        and updates the artisthashes.
         """
 
         def split(artists: str):
@@ -115,6 +124,10 @@ class Track:
         self.artisthashes = [a["artisthash"] for a in self.artists]
 
     def map_with_config(self):
+        """
+        Applies various transformations to the track's title and album
+        based on the user's configuration settings.
+        """
         new_title = self.title
 
         # Extract featured artists
@@ -156,6 +169,9 @@ class Track:
             )
 
     def process_genres(self):
+        """
+        Processes and standardizes the genre information for the track.
+        """
         if self.genres:
             src_genres: str = self.genres
 
@@ -181,3 +197,11 @@ class Track:
                 for g in genres_list
             ]
             self.genrehashes = [g["genrehash"] for g in self.genres]
+
+    def recreate_trackhash(self):
+        """
+        Recreates the trackhash based on the current title, album, and artist information.
+        """
+        self.trackhash = create_hash(
+            self.title, self.album, *(artist["name"] for artist in self.artists)
+        )
diff --git a/app/store/albums.py b/app/store/albums.py
index 7cbe8b8f..36c60921 100644
--- a/app/store/albums.py
+++ b/app/store/albums.py
@@ -44,19 +44,19 @@ class AlbumMapEntry:
 
 
 class AlbumStore:
-    albums: list[Album] = CustomList()
+    # albums: list[Album] = CustomList()
     albummap: dict[str, AlbumMapEntry] = {}
 
-    @staticmethod
-    def create_album(track: Track):
-        """
-        Creates album object from a track
-        """
-        return Album(
-            albumhash=track.albumhash,
-            albumartists=track.albumartists,  # type: ignore
-            title=track.og_album,
-        )
+    # @staticmethod
+    # def create_album(track: Track):
+    #     """
+    #     Creates album object from a track
+    #     """
+    #     return Album(
+    #         albumhash=track.albumhash,
+    #         albumartists=track.albumartists,  # type: ignore
+    #         title=track.og_album,
+    #     )
 
     @classmethod
     def load_albums(cls, instance_key: str):
@@ -74,6 +74,12 @@ class AlbumStore:
         }
         print("Done!")
 
+    @classmethod
+    def index_new_album(cls, album: Album, trackhashes: set[str]):
+        cls.albummap[album.albumhash] = AlbumMapEntry(
+            album=album, trackhashes=trackhashes
+        )
+
     @classmethod
     def get_flat_list(cls):
         """
@@ -141,12 +147,12 @@ class AlbumStore:
         master_string = "-".join(a.albumartists_hashes for a in cls.albums)
         return master_string.count(artisthash)
 
-    @classmethod
-    def album_exists(cls, albumhash: str) -> bool:
-        """
-        Checks if an album exists.
-        """
-        return albumhash in "-".join([a.albumhash for a in cls.albums])
+    # @classmethod
+    # def album_exists(cls, albumhash: str) -> bool:
+    #     """
+    #     Checks if an album exists.
+    #     """
+    #     return albumhash in "-".join([a.albumhash for a in cls.albums])
 
     @classmethod
     def remove_album(cls, album: Album):
diff --git a/app/store/artists.py b/app/store/artists.py
index f88dcb83..079be398 100644
--- a/app/store/artists.py
+++ b/app/store/artists.py
@@ -34,11 +34,10 @@ class ArtistMapEntry:
 
 
 class ArtistStore:
-    artists: list[Artist] = CustomList()
     artistmap: dict[str, ArtistMapEntry] = {}
 
     @classmethod
-    def load_artists(cls, instance_key: str):
+    def load_artists(cls, instance_key: str, _trackhashes: list[str] = []):
         """
         Loads all artists from the database into the store.
         """
@@ -52,7 +51,7 @@ class ArtistStore:
             artist.artisthash: ArtistMapEntry(
                 artist=artist, albumhashes=albumhashes, trackhashes=trackhashes
             )
-            for artist, trackhashes, albumhashes in create_artists()
+            for artist, trackhashes, albumhashes in create_artists(_trackhashes)
         }
 
         # for track in TrackStore.get_flat_list():
@@ -77,35 +76,35 @@ class ArtistStore:
         """
         return [a.artist for a in cls.artistmap.values()]
 
-    @classmethod
-    def map_artist_color(cls, artist_tuple: tuple):
-        """
-        Maps a color to the corresponding artist.
-        """
+    # @classmethod
+    # def map_artist_color(cls, artist_tuple: tuple):
+    #     """
+    #     Maps a color to the corresponding artist.
+    #     """
 
-        artisthash = artist_tuple[1]
-        color = json.loads(artist_tuple[2])
+    #     artisthash = artist_tuple[1]
+    #     color = json.loads(artist_tuple[2])
 
-        for artist in cls.artists:
-            if artist.artisthash == artisthash:
-                artist.set_colors(color)
-                break
+    #     for artist in cls.artists:
+    #         if artist.artisthash == artisthash:
+    #             artist.set_colors(color)
+    #             break
 
-    @classmethod
-    def add_artist(cls, artist: Artist):
-        """
-        Adds an artist to the store.
-        """
-        cls.artists.append(artist)
+    # @classmethod
+    # def add_artist(cls, artist: Artist):
+    #     """
+    #     Adds an artist to the store.
+    #     """
+    #     cls.artists.append(artist)
 
-    @classmethod
-    def add_artists(cls, artists: list[Artist]):
-        """
-        Adds multiple artists to the store.
-        """
-        for artist in artists:
-            if artist not in cls.artists:
-                cls.artists.append(artist)
+    # @classmethod
+    # def add_artists(cls, artists: list[Artist]):
+    #     """
+    #     Adds multiple artists to the store.
+    #     """
+    #     for artist in artists:
+    #         if artist not in cls.artists:
+    #             cls.artists.append(artist)
 
     @classmethod
     def get_artist_by_hash(cls, artisthash: str):
@@ -124,34 +123,34 @@ class ArtistStore:
         artists = [cls.get_artist_by_hash(hash) for hash in artisthashes]
         return [a for a in artists if a is not None]
 
-    @classmethod
-    def artist_exists(cls, artisthash: str) -> bool:
-        """
-        Checks if an artist exists.
-        """
-        return artisthash in "-".join([a.artisthash for a in cls.artists])
+    # @classmethod
+    # def artist_exists(cls, artisthash: str) -> bool:
+    #     """
+    #     Checks if an artist exists.
+    #     """
+    #     return artisthash in "-".join([a.artisthash for a in cls.artists])
 
-    @classmethod
-    def artist_has_tracks(cls, artisthash: str) -> bool:
-        """
-        Checks if an artist has tracks.
-        """
-        artists: set[str] = set()
+    # @classmethod
+    # def artist_has_tracks(cls, artisthash: str) -> bool:
+    #     """
+    #     Checks if an artist has tracks.
+    #     """
+    #     artists: set[str] = set()
 
-        for track in TrackStore.tracks:
-            artists.update(track.artist_hashes)
-            album_artists: list[str] = [a.artisthash for a in track.albumartists]
-            artists.update(album_artists)
+    #     for track in TrackStore.tracks:
+    #         artists.update(track.artist_hashes)
+    #         album_artists: list[str] = [a.artisthash for a in track.albumartists]
+    #         artists.update(album_artists)
 
-        master_hash = "-".join(artists)
-        return artisthash in master_hash
+    #     master_hash = "-".join(artists)
+    #     return artisthash in master_hash
 
-    @classmethod
-    def remove_artist_by_hash(cls, artisthash: str):
-        """
-        Removes an artist from the store.
-        """
-        cls.artists = CustomList(a for a in cls.artists if a.artisthash != artisthash)
+    # @classmethod
+    # def remove_artist_by_hash(cls, artisthash: str):
+    #     """
+    #     Removes an artist from the store.
+    #     """
+    #     cls.artists = CustomList(a for a in cls.artists if a.artisthash != artisthash)
 
     @classmethod
     def get_artist_tracks(cls, artisthash: str):
diff --git a/app/store/folder.py b/app/store/folder.py
index 34555ccb..b262fe42 100644
--- a/app/store/folder.py
+++ b/app/store/folder.py
@@ -39,10 +39,14 @@ class FolderStore:
             trackhash = cls.map.get(filepath)
 
             if trackhash:
-                track = TrackStore.trackhashmap.get(trackhash)
+                trackgroup = TrackStore.trackhashmap.get(trackhash)
 
-                if track:
-                    yield track.tracks[0]
+                if trackgroup is None:
+                    continue
+
+                for track in trackgroup.tracks:
+                    if track.filepath == filepath:
+                        yield track
 
     @classmethod
     def count_tracks_containing_paths(cls, paths: list[str]):
diff --git a/app/utils/__init__.py b/app/utils/__init__.py
index c432160c..ca253dec 100644
--- a/app/utils/__init__.py
+++ b/app/utils/__init__.py
@@ -14,7 +14,8 @@ def format_number(number: float) -> str:
     return locale.format_string("%d", number, grouping=True)
 
 
-
-
 def flatten(list_: Iterable[list[T]]) -> list[T]:
+    """
+    Flattens a list of lists into a single list.
+    """
     return [item for sublist in list_ for item in sublist]
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 00000000..e69de29b
diff --git a/docs/streaming.md b/docs/streaming.md
new file mode 100644
index 00000000..87b11ac5
--- /dev/null
+++ b/docs/streaming.md
@@ -0,0 +1,3 @@
+## Streaming
+
+## Transcoding
diff --git a/docs/watchdog.md b/docs/watchdog.md
new file mode 100644
index 00000000..e69de29b
diff --git a/poetry.lock b/poetry.lock
index 06d77703..0af17fe4 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -491,6 +491,23 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "ffmpeg-python"
+version = "0.2.0"
+description = "Python bindings for FFmpeg - with complex filtering support"
+optional = false
+python-versions = "*"
+files = [
+    {file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"},
+    {file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"},
+]
+
+[package.dependencies]
+future = "*"
+
+[package.extras]
+dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
+
 [[package]]
 name = "flask"
 version = "2.3.3"
@@ -597,6 +614,17 @@ dotenv = ["python-dotenv"]
 email = ["email-validator"]
 yaml = ["pyyaml"]
 
+[[package]]
+name = "future"
+version = "1.0.0"
+description = "Clean single-source support for Python 3 and 2"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216"},
+    {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"},
+]
+
 [[package]]
 name = "gevent"
 version = "23.9.1"
@@ -2733,4 +2761,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.12"
-content-hash = "184c1c56051131473212b74de5719e2629c630555b13cd4cd2de371b3a2fb195"
+content-hash = "43972b6ffadd14e5047f067a0258f2428ebe351df8bd032dc0bf05df379678a6"
diff --git a/pyproject.toml b/pyproject.toml
index 8275feb6..87d5b809 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,6 +31,7 @@ sqlalchemy = "^2.0.31"
 memory-profiler = "^0.61.0"
 sortedcontainers = "^2.4.0"
 xxhash = "^3.4.1"
+ffmpeg-python = "^0.2.0"
 
 [tool.poetry.dev-dependencies]
 pylint = "^2.15.5"