From 27a277cc5bc07b9b07432b18e2300f420d675e8c Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Tue, 30 Jan 2024 15:49:06 +0100 Subject: [PATCH 01/10] Organization and usefulness for the final user --- build.sh | 18 ++++++++++++++---- dev_build.sh | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 dev_build.sh diff --git a/build.sh b/build.sh index fc397746..ba065e02 100755 --- a/build.sh +++ b/build.sh @@ -1,9 +1,19 @@ -#!/bin/zsh +#!/bin/bash +# REVIEW Above: bash is way more compatible than other shells # builds the latest version of the client and server -cd ../swingmusic-client -yarn build --outDir ../swingmusic/client +# REVIEW These are not useful if you dont have the source code +#cd ../swingmusic-client +#yarn build --outDir ../swingmusic/client +#../swingmusic -cd ../swingmusic +# REVIEW Cleaning up +rm -rf build dist + +# REVIEW Install poetry & requirements +pip install poetry +poetry install --no-root + +# Build the app poetry run python manage.py --build \ No newline at end of file diff --git a/dev_build.sh b/dev_build.sh new file mode 100644 index 00000000..09adbb8e --- /dev/null +++ b/dev_build.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# REVIEW Above: bash is way more compatible than other shells + +# builds the latest version of the client and server + +cd ../swingmusic-client || exit # REVIEW Failsafe exit + +yarn build --outDir ../swingmusic/client +cd ../swingmusic || exit # REVIEW Failsafe exit + +# REVIEW Optional cleaning up +# rm -rf build dist + +# REVIEW Install poetry & requirements +# Build the app +poetry run python manage.py --build \ No newline at end of file From d411f04e981a4d00a23f4df739fc6f6b8b82295e Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Tue, 30 Jan 2024 15:49:47 +0100 Subject: [PATCH 02/10] Added Trunk configuration for those using trunk (suggested: makes the code way cleaner and easier to read) --- .trunk/configs/.hadolint.yaml | 4 +++ .trunk/configs/.isort.cfg | 2 ++ .trunk/configs/.markdownlint.yaml | 10 +++++++ .trunk/configs/.shellcheckrc | 7 +++++ .trunk/configs/.yamllint.yaml | 10 +++++++ .trunk/configs/ruff.toml | 5 ++++ .trunk/configs/svgo.config.js | 14 ++++++++++ .trunk/trunk.yaml | 46 +++++++++++++++++++++++++++++++ 8 files changed, 98 insertions(+) create mode 100644 .trunk/configs/.hadolint.yaml create mode 100644 .trunk/configs/.isort.cfg create mode 100644 .trunk/configs/.markdownlint.yaml create mode 100644 .trunk/configs/.shellcheckrc create mode 100644 .trunk/configs/.yamllint.yaml create mode 100644 .trunk/configs/ruff.toml create mode 100644 .trunk/configs/svgo.config.js create mode 100644 .trunk/trunk.yaml diff --git a/.trunk/configs/.hadolint.yaml b/.trunk/configs/.hadolint.yaml new file mode 100644 index 00000000..98bf0cd2 --- /dev/null +++ b/.trunk/configs/.hadolint.yaml @@ -0,0 +1,4 @@ +# Following source doesn't work in most setups +ignored: + - SC1090 + - SC1091 diff --git a/.trunk/configs/.isort.cfg b/.trunk/configs/.isort.cfg new file mode 100644 index 00000000..b9fb3f3e --- /dev/null +++ b/.trunk/configs/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile=black diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml new file mode 100644 index 00000000..fb940393 --- /dev/null +++ b/.trunk/configs/.markdownlint.yaml @@ -0,0 +1,10 @@ +# Autoformatter friendly markdownlint config (all formatting rules disabled) +default: true +blank_lines: false +bullet: false +html: false +indentation: false +line_length: false +spaces: false +url: false +whitespace: false diff --git a/.trunk/configs/.shellcheckrc b/.trunk/configs/.shellcheckrc new file mode 100644 index 00000000..8c7b1ada --- /dev/null +++ b/.trunk/configs/.shellcheckrc @@ -0,0 +1,7 @@ +enable=all +source-path=SCRIPTDIR +disable=SC2154 + +# If you're having issues with shellcheck following source, disable the errors via: +# disable=SC1090 +# disable=SC1091 diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 00000000..4d444662 --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,10 @@ +rules: + quoted-strings: + required: only-when-needed + extra-allowed: ["{|}"] + empty-values: + forbid-in-block-mappings: true + forbid-in-flow-mappings: true + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/configs/ruff.toml b/.trunk/configs/ruff.toml new file mode 100644 index 00000000..f5a235cf --- /dev/null +++ b/.trunk/configs/ruff.toml @@ -0,0 +1,5 @@ +# Generic, formatter-friendly config. +select = ["B", "D3", "E", "F"] + +# Never enforce `E501` (line length violations). This should be handled by formatters. +ignore = ["E501"] diff --git a/.trunk/configs/svgo.config.js b/.trunk/configs/svgo.config.js new file mode 100644 index 00000000..b257d134 --- /dev/null +++ b/.trunk/configs/svgo.config.js @@ -0,0 +1,14 @@ +module.exports = { + plugins: [ + { + name: "preset-default", + params: { + overrides: { + removeViewBox: false, // https://github.com/svg/svgo/issues/1128 + sortAttrs: true, + removeOffCanvasPaths: true, + }, + }, + }, + ], +}; diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml new file mode 100644 index 00000000..f742b1d5 --- /dev/null +++ b/.trunk/trunk.yaml @@ -0,0 +1,46 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml +version: 0.1 +cli: + version: 1.19.0 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) +plugins: + sources: + - id: trunk + ref: v1.4.2 + uri: https://github.com/trunk-io/plugins +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) +runtimes: + enabled: + - go@1.21.0 + - node@18.12.1 + - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +lint: + enabled: + - actionlint@1.6.26 + - bandit@1.7.7 + - black@24.1.1 + - checkov@3.2.1 + - git-diff-check + - hadolint@2.12.0 + - isort@5.13.2 + - markdownlint@0.39.0 + - osv-scanner@1.6.1 + - prettier@3.2.4 + - ruff@0.1.15 + - shellcheck@0.9.0 + - shfmt@3.6.0 + - svgo@3.2.0 + - taplo@0.8.1 + - terrascan@1.18.11 + - trivy@0.48.3 + - trufflehog@3.66.1 + - yamllint@1.33.0 +actions: + disabled: + - trunk-announce + - trunk-check-pre-push + - trunk-fmt-pre-commit + enabled: + - trunk-upgrade-available From e0581259a68d96f15f834d092a4441c883f918bb Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Tue, 30 Jan 2024 15:51:05 +0100 Subject: [PATCH 03/10] Just some formatting change and utilities for the next step --- app/lib/populate.py | 4 +++- app/settings.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/lib/populate.py b/app/lib/populate.py index d5bb4fce..012340be 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -104,8 +104,9 @@ class Populate: log.error( "Internet connection lost. Downloading artist images stopped." ) + log.error(e) # REVIEW More informations = good else: - log.warning(f"No internet connection. Downloading artist images stopped!") + log.warning("No internet connection. Downloading artist images stopped!") # Re-process the new artist images. if tried_to_download_new_images: @@ -135,6 +136,7 @@ class Populate: unmodified_paths.add(track.filepath) continue except (FileNotFoundError, OSError) as e: + log.warning(e) # REVIEW More informations = good TrackStore.remove_track_obj(track) remove_tracks_by_filepaths(track.filepath) diff --git a/app/settings.py b/app/settings.py index 13cf5724..bfe62a47 100644 --- a/app/settings.py +++ b/app/settings.py @@ -242,6 +242,9 @@ class Keys: @classmethod def load(cls): + # TODO Remove this. Just an handy flag to test the app without the API key + # IS_BUILD = True + if IS_BUILD: cls.LASTFM_API_KEY = configs.LASTFM_API_KEY cls.PLUGIN_LYRICS_AUTHORITY = configs.PLUGIN_LYRICS_AUTHORITY @@ -253,7 +256,8 @@ class Keys: @classmethod def verify_keys(cls): if not cls.LASTFM_API_KEY: - print("ERROR: LASTFM_API_KEY not set in environment") + # REVIEW Ideally, this shouldn't be fatal + print("WARNING: LASTFM_API_KEY not set in environment. Experimental API calls will be implemented") sys.exit(0) @classmethod From 632447bdba96e0096cf03b90f88844e3fcf0bd2e Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Tue, 30 Jan 2024 15:51:46 +0100 Subject: [PATCH 04/10] Experimental Kerve API use --- app/requests/artists.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/requests/artists.py b/app/requests/artists.py index 28737dc1..e543f0ec 100644 --- a/app/requests/artists.py +++ b/app/requests/artists.py @@ -6,7 +6,7 @@ import urllib.parse import requests from requests import ConnectionError, HTTPError, ReadTimeout -from app import settings +#from app import settings from app.utils.hashing import create_hash @@ -14,8 +14,11 @@ def fetch_similar_artists(name: str): """ Fetches similar artists from Last.fm """ - url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={urllib.parse.quote_plus(name, safe='')}&api_key={settings.Keys.LASTFM_API_KEY}&format=json&limit=250" + # REVIEW This is the old way of doing it. The new way is to use the Kerve API. + #url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={urllib.parse.quote_plus(name, safe='')}&api_key={settings.Keys.LASTFM_API_KEY}&format=json&limit=250" + # TODO Cannot be tested due to PR message + url = f"https://kerve.last.fm/kerve/similarartists?artist={urllib.parse.quote_plus(name, safe='')}&autocorrect=1&tracks=1&image_size=large&limit=250&format=json" try: response = requests.get(url, timeout=10) response.raise_for_status() @@ -25,7 +28,8 @@ def fetch_similar_artists(name: str): data = response.json() try: - artists = data["similarartists"]["artist"] + #artists = data["similarartists"]["artist"] + artists = data["results"]["artist"] except KeyError: return [] From c15324791886e58340e76dc50d1ed086d592a784 Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Wed, 31 Jan 2024 18:25:01 +0100 Subject: [PATCH 05/10] API switch to unrestricted ones --- app/api/settings.py | 5 +++-- app/arg_handler.py | 4 ++-- app/lib/populate.py | 3 +++ app/settings.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/api/settings.py b/app/api/settings.py index 92ccdf2c..6b8950ef 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -54,7 +54,8 @@ def rebuild_store(db_dirs: list[str]): try: populate.Populate(instance_key=instance_key) - except populate.PopulateCancelledError: + except populate.PopulateCancelledError as e: + print(e) reload_everything(instance_key) return @@ -63,7 +64,7 @@ def rebuild_store(db_dirs: list[str]): log.info("Rebuilding library... ✅") -# I freaking don't know what this function does anymore +# I freaking don't know what this function does anymore def finalize(new_: list[str], removed_: list[str], db_dirs_: list[str]): """ Params: diff --git a/app/arg_handler.py b/app/arg_handler.py index 82172fdd..64b2de54 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -56,8 +56,8 @@ class HandleArgs: value = settings.Keys.get(key) if not value: - log.error(f"ERROR: {key} not set in environment") - sys.exit(0) + log.error(f"WARNING: {key} not set in environment") + #sys.exit(0) lines.append(f'{key} = "{value}"\n') diff --git a/app/lib/populate.py b/app/lib/populate.py index 012340be..7b3ae9fa 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -114,6 +114,7 @@ class Populate: if has_connection(): try: + print("Attempting to download similar artists...") FetchSimilarArtistsLastFM(instance_key) except PopulateCancelledError as e: log.warn(e) @@ -288,6 +289,7 @@ def save_similar_artists(_map: tuple[str, Artist]): instance_key, artist = _map if POPULATE_KEY != instance_key: + print("Warning: Populate key changed") raise PopulateCancelledError( "'FetchSimilarArtistsLastFM': Populate key changed" ) @@ -323,6 +325,7 @@ class FetchSimilarArtistsLastFM: with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: try: + print("Processing similar artists") results = list( tqdm( executor.map(save_similar_artists, key_artist_map), diff --git a/app/settings.py b/app/settings.py index bfe62a47..7ab3b7e8 100644 --- a/app/settings.py +++ b/app/settings.py @@ -258,7 +258,7 @@ class Keys: if not cls.LASTFM_API_KEY: # REVIEW Ideally, this shouldn't be fatal print("WARNING: LASTFM_API_KEY not set in environment. Experimental API calls will be implemented") - sys.exit(0) + #sys.exit(0) @classmethod def get(cls, key: str): From bc51150e4756b8e2c70d5ba9c66f922fdbf0ca72 Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Wed, 31 Jan 2024 18:25:20 +0100 Subject: [PATCH 06/10] failproof online check (fixes vpns and dns tunnels) --- app/utils/network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/utils/network.py b/app/utils/network.py index 1a1c89c9..1480c521 100644 --- a/app/utils/network.py +++ b/app/utils/network.py @@ -1,12 +1,14 @@ import socket as Socket - -def has_connection(host="8.8.8.8", port=53, timeout=3): + +def has_connection(host="google.it", port=80, timeout=3): """ + # REVIEW Was: Host: 8.8.8.8 (google-public-dns-a.google.com) OpenPort: 53/tcp Service: domain (DNS/TCP) """ + try: Socket.setdefaulttimeout(timeout) Socket.socket(Socket.AF_INET, Socket.SOCK_STREAM).connect((host, port)) From 5a918d078e75c7cc59e0719cb866568896e2e73f Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Wed, 31 Jan 2024 19:59:36 +0100 Subject: [PATCH 07/10] added some handy launchers --- .gitignore | 3 ++- build.sh | 2 +- dev_build.sh | 2 +- run | 3 +++ 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100755 run diff --git a/.gitignore b/.gitignore index 0e5621e6..24ff742d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ logs.txt TODO.md testdata.py -test.py \ No newline at end of file +test.py +nohup.out diff --git a/build.sh b/build.sh index ba065e02..ba5c7edb 100755 --- a/build.sh +++ b/build.sh @@ -12,7 +12,7 @@ rm -rf build dist # REVIEW Install poetry & requirements -pip install poetry +poetry || pip install poetry poetry install --no-root # Build the app diff --git a/dev_build.sh b/dev_build.sh index 09adbb8e..edc8e958 100644 --- a/dev_build.sh +++ b/dev_build.sh @@ -3,8 +3,8 @@ # builds the latest version of the client and server +# NOTE Changes directory to the webclient directory and builds it cd ../swingmusic-client || exit # REVIEW Failsafe exit - yarn build --outDir ../swingmusic/client cd ../swingmusic || exit # REVIEW Failsafe exit diff --git a/run b/run new file mode 100755 index 00000000..4795296f --- /dev/null +++ b/run @@ -0,0 +1,3 @@ +#!/bin/bash + +poetry run python manage.py From 077661b73c1a63f70f298f3e98fb1fc88ef0f696 Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Thu, 1 Feb 2024 20:24:42 +0100 Subject: [PATCH 08/10] lighter md5 method to be nicer on older cpus --- app/utils/hashing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 0dfebf3c..d4a25eb9 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -32,5 +32,6 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = unidecode(str_) str_ = str_.encode("utf-8") - str_ = hashlib.sha256(str_).hexdigest() + str_ = hashlib.md5(str_).hexdigest() + # REVIEW Switched to md5 hashlib.sha256(str_).hexdigest() return str_[-limit:] From f0a8aa02a905f7fbe25bc5979a66b9b958b646dd Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Thu, 1 Feb 2024 20:32:31 +0100 Subject: [PATCH 09/10] opted for sha1 --- app/utils/hashing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/utils/hashing.py b/app/utils/hashing.py index d4a25eb9..5631f480 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -32,6 +32,6 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = unidecode(str_) str_ = str_.encode("utf-8") - str_ = hashlib.md5(str_).hexdigest() - # REVIEW Switched to md5 hashlib.sha256(str_).hexdigest() + str_ = hashlib.sha1(str_).hexdigest() + # REVIEW Switched to sha1 hashlib.sha256(str_).hexdigest() return str_[-limit:] From f66bca67ac79c6d158a566ea169552194c64bd66 Mon Sep 17 00:00:00 2001 From: thecookingsenpai Date: Fri, 2 Feb 2024 16:31:18 +0100 Subject: [PATCH 10/10] Testing with SHA1 gives positive results, also changed the truncation rule --- app/utils/hashing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/utils/hashing.py b/app/utils/hashing.py index 5631f480..e9ed147d 100644 --- a/app/utils/hashing.py +++ b/app/utils/hashing.py @@ -34,4 +34,9 @@ def create_hash(*args: str, decode=False, limit=10) -> str: str_ = str_.encode("utf-8") str_ = hashlib.sha1(str_).hexdigest() # REVIEW Switched to sha1 hashlib.sha256(str_).hexdigest() - return str_[-limit:] + + # REVIEW Take the first limit/2 and last limit/2 characters + # This is to avoid collisions + return str_[:limit // 2] + str_[-limit // 2:] if limit % 2 == 0 else str_[:limit // 2] + str_[-limit // 2 - 1:] + + # return str_[-limit:]