From 8f361e03d185479865d2b045451d40c5d9d6f4e5 Mon Sep 17 00:00:00 2001 From: cahthuranag Date: Fri, 6 Nov 2026 22:56:59 +0000 Subject: [PATCH 01/16] feat(automatic-exchange-rates): add central bank tables as a provider --- app/apps/currencies/exchange_rates/fetcher.py | 1 + .../currencies/exchange_rates/providers.py | 132 +++++++++++ ..._alter_exchangerateservice_service_type.py | 35 +++ app/apps/currencies/models.py | 4 + .../tests/test_allratestoday_provider.py | 208 ++++++++++++++++++ 5 files changed, 380 insertions(+) create mode 100644 app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py create mode 100644 app/apps/currencies/tests/test_allratestoday_provider.py diff --git a/app/apps/currencies/exchange_rates/fetcher.py b/app/apps/currencies/exchange_rates/fetcher.py index d70f088b..6b486304 100644 --- a/app/apps/currencies/exchange_rates/fetcher.py +++ b/app/apps/currencies/exchange_rates/fetcher.py @@ -19,6 +19,7 @@ PROVIDER_MAPPING = { "twelvedatamarkets": providers.TwelveDataMarketsProvider, "yfinance": providers.YFinanceMarketsProvider, "fxmacrodata": providers.FXMacroDataProvider, + "allratestoday": providers.AllRatesTodayProvider, } diff --git a/app/apps/currencies/exchange_rates/providers.py b/app/apps/currencies/exchange_rates/providers.py index 4d79d08c..eacb580b 100644 --- a/app/apps/currencies/exchange_rates/providers.py +++ b/app/apps/currencies/exchange_rates/providers.py @@ -1,4 +1,5 @@ import logging +import re import time import requests @@ -681,3 +682,134 @@ class FXMacroDataProvider(ExchangeRateProvider): ) return results + + +class AllRatesTodayProvider(ExchangeRateProvider): + """ + Implementation for the central bank tables served by allratestoday.com. + + Each service reads the latest published table of one central bank. The API + Key field holds the bank code (e.g. "bcb" or "rbi"), as this endpoint needs + no key. An empty field reads the ECB table. + """ + + BASE_URL = "https://allratestoday.com/api/open/central-bank" + DEFAULT_BANK = "ecb" + rates_inverted = False # A base/quote row says how many quote one base buys + + # A pair can have several rows. Prefer a published mid, else the mean of a buy + # and sell quote. Period averages are left out, they are not a current rate + MID_TYPES = ( + "reference", + "middle", + "official", + "indicative", + "close", + "spot", + "informational", + ) + BUY_SELL_TYPES = (("buy", "sell"), ("tt_buy", "tt_sell")) + + def __init__(self, api_key: str = None): + super().__init__(api_key) + self.bank = (api_key or self.DEFAULT_BANK).strip().lower() + self.session = requests.Session() + + @classmethod + def requires_api_key(cls) -> bool: + return False + + def _parse_table(self, rows: list) -> Dict[Tuple[str, str], Decimal]: + """Reduce the table to one rate per pair, in both directions""" + grouped = {} + for row in rows: + if not isinstance(row, dict) or not row.get("value"): + continue + pair = (row.get("base"), row.get("quote")) + grouped.setdefault(pair, {})[row.get("type")] = Decimal(str(row["value"])) + + rates = {} + for (base, quote), values in grouped.items(): + rate = next((values[t] for t in self.MID_TYPES if t in values), None) + if rate is None: + rate = next( + ( + (values[buy] + values[sell]) / 2 + for buy, sell in self.BUY_SELL_TYPES + if buy in values and sell in values + ), + None, + ) + if rate is None or rate <= 0: + continue + rates[(base, quote)] = rate + rates.setdefault((quote, base), self.invert_rate(rate)) + + return rates + + @staticmethod + def _find_rate( + rates: Dict[Tuple[str, str], Decimal], base: str, quote: str + ) -> Optional[Decimal]: + """Look up a pair, crossing it through a shared currency if needed""" + if (base, quote) in rates: + return rates[(base, quote)] + + for (from_code, via), rate in rates.items(): + if from_code == base and (via, quote) in rates: + return rate * rates[(via, quote)] + + return None + + def get_rates( + self, target_currencies: QuerySet, exchange_currencies: set + ) -> List[Tuple[Currency, Currency, Decimal]]: + results = [] + + if not re.fullmatch(r"[a-z0-9_-]{2,20}", self.bank): + logger.error("Invalid AllRatesToday bank code, it should look like 'bcb'") + return results + + try: + response = self.session.get(f"{self.BASE_URL}/{self.bank}", timeout=30) + response.raise_for_status() + data = response.json() + + if data.get("stale"): + logger.warning( + f"AllRatesToday table for {self.bank} is stale, " + f"last published on {data.get('rate_date')}" + ) + + rates = self._parse_table(data["rates"]) + except requests.RequestException as e: + logger.error(f"Error fetching AllRatesToday table for {self.bank}: {e}") + return results + except (KeyError, TypeError, ValueError, AttributeError, ArithmeticError) as e: + logger.error( + f"Unexpected response structure from AllRatesToday " + f"for {self.bank}: {e}" + ) + return results + + for target_currency in target_currencies: + if target_currency.exchange_currency not in exchange_currencies: + continue + + base_currency = target_currency.exchange_currency + + if base_currency.code == target_currency.code: + results.append((base_currency, target_currency, Decimal("1"))) + continue + + rate = self._find_rate(rates, base_currency.code, target_currency.code) + if rate is None: + logger.error( + f"No rate for {base_currency.code}/{target_currency.code} " + f"in the AllRatesToday table for {self.bank}" + ) + continue + + results.append((base_currency, target_currency, rate)) + + return results diff --git a/app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py b/app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py new file mode 100644 index 00000000..ffe19948 --- /dev/null +++ b/app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.17 on 2026-11-06 22:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("currencies", "0025_alter_exchangerateservice_service_type"), + ] + + operations = [ + migrations.AlterField( + model_name="exchangerateservice", + name="service_type", + field=models.CharField( + choices=[ + ("coingecko_free", "CoinGecko (Demo/Free)"), + ("coingecko_pro", "CoinGecko (Pro)"), + ("transitive", "Transitive (Calculated from Existing Rates)"), + ("frankfurter", "Frankfurter"), + ("twelvedata", "TwelveData"), + ("twelvedatamarkets", "TwelveData Markets"), + ("yfinance", "Yahoo Finance"), + ("fxmacrodata", "FXMacroData"), + ( + "allratestoday", + "AllRatesToday Central Banks (bank code as API key)", + ), + ], + max_length=255, + verbose_name="Service Type", + ), + ), + ] diff --git a/app/apps/currencies/models.py b/app/apps/currencies/models.py index 89e4c583..36287000 100644 --- a/app/apps/currencies/models.py +++ b/app/apps/currencies/models.py @@ -107,6 +107,10 @@ class ExchangeRateService(models.Model): TWELVEDATA_MARKETS = "twelvedatamarkets", "TwelveData Markets" YFINANCE = "yfinance", "Yahoo Finance" FXMACRODATA = "fxmacrodata", "FXMacroData" + ALLRATESTODAY = ( + "allratestoday", + "AllRatesToday Central Banks (bank code as API key)", + ) class IntervalType(models.TextChoices): ON = "on", _("On") diff --git a/app/apps/currencies/tests/test_allratestoday_provider.py b/app/apps/currencies/tests/test_allratestoday_provider.py new file mode 100644 index 00000000..b73d05d7 --- /dev/null +++ b/app/apps/currencies/tests/test_allratestoday_provider.py @@ -0,0 +1,208 @@ +from decimal import Decimal +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import requests + +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING +from apps.currencies.exchange_rates.providers import AllRatesTodayProvider +from apps.currencies.models import ExchangeRateService + + +class _FakeCurrency: + def __init__(self, code, exchange_currency=None): + self.code = code + self.exchange_currency = exchange_currency + + +def _row(base, quote, value, type_="reference"): + return {"base": base, "quote": quote, "type": type_, "value": value} + + +def _response(payload): + response = MagicMock() + response.status_code = 200 + response.json.return_value = payload + return response + + +class AllRatesTodayProviderTests(TestCase): + def setUp(self): + self.brl = _FakeCurrency("BRL") + self.usd = _FakeCurrency("USD") + self.eur = _FakeCurrency("EUR") + self.provider = AllRatesTodayProvider("bcb") + + def _get_rates(self, rows, target, base): + target.exchange_currency = base + with patch.object( + self.provider.session, "get", return_value=_response({"rates": rows}) + ): + return self.provider.get_rates([target], {base}) + + def test_requests_the_configured_bank(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + self.provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_called_once_with( + "https://allratestoday.com/api/open/central-bank/bcb", timeout=30 + ) + + def test_fetches_one_table_for_all_pairs(self): + self.usd.exchange_currency = self.brl + self.eur.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5.4), _row("EUR", "BRL", 6.2)]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ) as mock_get: + rates = self.provider.get_rates([self.usd, self.eur], {self.brl}) + + mock_get.assert_called_once() + self.assertEqual(len(rates), 2) + + def test_empty_field_reads_the_ecb_table(self): + self.assertEqual(AllRatesTodayProvider(None).bank, "ecb") + self.assertEqual(AllRatesTodayProvider("").bank, "ecb") + + def test_normalizes_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider(" BCB ").bank, "bcb") + + def test_rejects_malformed_bank_code_without_a_request(self): + provider = AllRatesTodayProvider("../latest/USD") + self.usd.exchange_currency = self.brl + + with patch.object(provider.session, "get") as mock_get: + rates = provider.get_rates([self.usd], {self.brl}) + + mock_get.assert_not_called() + self.assertEqual(rates, []) + + def test_uses_a_row_in_the_requested_direction(self): + rates = self._get_rates([_row("EUR", "USD", 1.16)], self.usd, self.eur) + + self.assertEqual(rates, [(self.eur, self.usd, Decimal("1.16"))]) + + def test_inverts_a_row_in_the_opposite_direction(self): + # bcb quotes foreign currencies in BRL, so BRL -> USD has to be inverted + rates = self._get_rates([_row("USD", "BRL", 5)], self.brl, self.usd) + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.brl) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_crosses_pairs_through_the_table_home_currency(self): + rows = [_row("USD", "BRL", 5), _row("EUR", "BRL", 6)] + + rates = self._get_rates(rows, self.eur, self.usd) + + self.assertEqual(len(rates), 1) + self.assertAlmostEqual(rates[0][2], Decimal("0.8333333"), places=6) + + def test_same_currency_is_one(self): + rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.usd) + + self.assertEqual(rates, [(self.usd, self.usd, Decimal("1"))]) + + def test_prefers_the_published_mid_over_buy_and_sell(self): + rows = [ + _row("USD", "BRL", 5.3, "buy"), + _row("USD", "BRL", 5.4, "middle"), + _row("USD", "BRL", 5.7, "sell"), + ] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5.4"))]) + + def test_averages_buy_and_sell_without_a_mid(self): + rows = [_row("USD", "BRL", 4, "buy"), _row("USD", "BRL", 6, "sell")] + + rates = self._get_rates(rows, self.brl, self.usd) + + self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))]) + + def test_averages_tt_buy_and_sell_without_a_mid(self): + rows = [ + _row("USD", "MUR", 44.0, "tt_buy"), + _row("USD", "MUR", 46.0, "tt_sell"), + ] + + result = self._get_rates(rows, self.usd, _FakeCurrency("MUR")) + + self.assertEqual(result[0][2], Decimal("1") / Decimal("45.0")) + + def test_ignores_rows_of_other_types(self): + rows = [_row("USD", "BRL", 5, "monthly_average")] + + self.assertEqual(self._get_rates(rows, self.brl, self.usd), []) + + def test_skips_pair_missing_from_the_table(self): + rows = [_row("USD", "BRL", 5)] + + self.assertEqual(self._get_rates(rows, self.eur, self.usd), []) + + def test_ignores_zero_and_malformed_rows(self): + rows = ["nope", _row("USD", "BRL", 0), _row("EUR", "BRL", 6)] + + rates = self._get_rates(rows, self.brl, self.eur) + + self.assertEqual(rates, [(self.eur, self.brl, Decimal("6"))]) + + def test_logs_a_stale_table_and_still_uses_it(self): + self.usd.exchange_currency = self.brl + payload = { + "stale": True, + "rate_date": "2026-10-01", + "rates": [_row("BRL", "USD", 0.2)], + } + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("2026-10-01", logs.output[0]) + self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))]) + + def test_returns_nothing_on_request_error(self): + self.usd.exchange_currency = self.brl + + with patch.object( + self.provider.session, + "get", + side_effect=requests.RequestException("404 Not Found"), + ): + rates = self.provider.get_rates([self.usd], {self.brl}) + + self.assertEqual(rates, []) + + def test_returns_nothing_on_malformed_body(self): + self.usd.exchange_currency = self.brl + + for payload in ({}, [], {"rates": None}): + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_returns_nothing_on_non_json_body(self): + self.usd.exchange_currency = self.brl + response = _response({}) + response.json.side_effect = ValueError("not json") + + with patch.object(self.provider.session, "get", return_value=response): + self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), []) + + def test_is_registered_without_an_api_key(self): + self.assertFalse(AllRatesTodayProvider.requires_api_key()) + self.assertIs(PROVIDER_MAPPING["allratestoday"], AllRatesTodayProvider) + self.assertEqual(ExchangeRateService.ServiceType.ALLRATESTODAY, "allratestoday") From 6cf8cbf786c33888845918d1c98dffd9db537cf8 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 7 Nov 2026 14:56:55 +0530 Subject: [PATCH 02/16] fix(import): stop hardcoding the Docker image's paths Import uploads were always written to /usr/src/app/temp, presets were read from /usr/src/app/import_presets and the caches lived in /var/tmp, so WYGIWYH only worked from the Docker image's layout. Running it as a systemd service with a strict sandbox, or on an immutable distro like NixOS, needs these folders to follow the app or be configurable. - Presets are read from IMPORT_PRESETS_ROOT, the import_presets folder next to the app. - Uploads wait in IMPORT_TEMP_ROOT, the temp folder next to the app by default, and movable with the IMPORT_TEMP_ROOT env var. The web views, the API and the import service all use it. - The file-based caches live under CACHE_ROOT, /var/tmp by default, and movable with the CACHE_ROOT env var. In the Docker image the app sits in /usr/src/app, so every default resolves to the same folder as before. The import service checked that a file was inside the temp folder with a plain string prefix, which also accepted sibling folders such as /usr/src/app/temp-other. It now compares paths. CI no longer needs to create /usr/src/app before running the tests. Fixes #472 --- .github/workflows/tests.yml | 6 - .gitignore | 1 + README.md | 2 + app/WYGIWYH/settings.py | 9 +- app/apps/api/views/imports.py | 3 +- app/apps/import_app/services/presets.py | 6 +- app/apps/import_app/services/v1.py | 12 +- app/apps/import_app/tests/test_paths.py | 151 +++++++++++++++++++ app/apps/import_app/tests/test_qif_import.py | 10 +- app/apps/import_app/views.py | 3 +- 10 files changed, 180 insertions(+), 23 deletions(-) create mode 100644 app/apps/import_app/tests/test_paths.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a704fa5..038881a1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,12 +60,6 @@ jobs: - name: Install dependencies run: uv sync --frozen - # Import code reads/writes the Docker image's paths directly - - name: Create container paths used by the app - run: | - sudo mkdir -p /usr/src/app/temp /usr/src/app/import_presets - sudo chown -R "$(id -u):$(id -g)" /usr/src/app - - name: Check for missing migrations working-directory: app run: uv run python manage.py makemigrations --check --dry-run diff --git a/.gitignore b/.gitignore index 93ba54a4..7c4160ef 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,4 @@ postgres_data/ # Private local uploads app/attachments/ +app/temp/ diff --git a/README.md b/README.md index 10bde2d3..f757b874 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,8 @@ To create the first user, open the container's console using Unraid's UI, by cli | DJANGO_VITE_DEV_MODE | true\|false | false | Enables Vite dev server mode for frontend development. When true, assets are served from Vite's dev server instead of the build manifest. For development only! | | DJANGO_VITE_DEV_SERVER_PORT | int | 5173 | The port where Vite's dev server is running. Only used when DJANGO_VITE_DEV_MODE is true. For development only! | | DJANGO_VITE_DEV_SERVER_HOST | string | localhost | The host where Vite's dev server is running. Only used when DJANGO_VITE_DEV_MODE is true. For development only! | +| IMPORT_TEMP_ROOT | string | temp, in the app folder | Where uploaded import files wait to be processed. Must be writable and shared between the web and task processes. | +| CACHE_ROOT | string | /var/tmp | The writable folder the file-based caches are stored in. | ## OIDC Configuration diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index f7756b73..44f2e9e5 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -313,6 +313,9 @@ LOCALE_PATHS = [BASE_DIR / "locale"] STATIC_URL = "static/" STATIC_ROOT = BASE_DIR / "static_files" ATTACHMENT_MEDIA_ROOT = BASE_DIR / "attachments" +IMPORT_PRESETS_ROOT = BASE_DIR / "import_presets" +# Web and task workers must see the same folder: uploads wait here to be processed +IMPORT_TEMP_ROOT = Path(os.getenv("IMPORT_TEMP_ROOT") or BASE_DIR / "temp").resolve() STATICFILES_DIRS = [ ROOT_DIR / "frontend" / "build", @@ -324,15 +327,17 @@ STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] +CACHE_ROOT = Path(os.getenv("CACHE_ROOT") or "/var/tmp").resolve() + CACHES = { "default": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/var/tmp/django_cache", + "LOCATION": CACHE_ROOT / "django_cache", }, # Kept apart from "default": cachalot fills it and FileBasedCache culls random entries past MAX_ENTRIES "update_check": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/var/tmp/django_cache_update_check", + "LOCATION": CACHE_ROOT / "django_cache_update_check", }, } diff --git a/app/apps/api/views/imports.py b/app/apps/api/views/imports.py index f8e6edb7..9abb945d 100644 --- a/app/apps/api/views/imports.py +++ b/app/apps/api/views/imports.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.core.files.storage import FileSystemStorage from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view, inline_serializer @@ -130,7 +131,7 @@ class ImportViewSet(viewsets.ViewSet): uploaded_file = serializer.validated_data["file"] # Save file to temp location - fs = FileSystemStorage(location="/usr/src/app/temp") + fs = FileSystemStorage(location=settings.IMPORT_TEMP_ROOT) filename = fs.save(uploaded_file.name, uploaded_file) file_path = fs.path(filename) diff --git a/app/apps/import_app/services/presets.py b/app/apps/import_app/services/presets.py index 824a2468..e4cef311 100644 --- a/app/apps/import_app/services/presets.py +++ b/app/apps/import_app/services/presets.py @@ -1,17 +1,17 @@ import json from pathlib import Path +from django.conf import settings + from apps.import_app.models import ImportProfile class PresetService: - PRESET_PATH = "/usr/src/app/import_presets" - @classmethod def get_all_presets(cls): presets = [] - for folder in Path(cls.PRESET_PATH).iterdir(): + for folder in settings.IMPORT_PRESETS_ROOT.iterdir(): if folder.is_dir(): manifest_path = folder / "manifest.json" config_path = folder / "config.yml" diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index bed860ac..5aad0b9e 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -7,12 +7,14 @@ import zipfile from django.db import transaction from datetime import datetime, date from decimal import Decimal, InvalidOperation +from pathlib import Path from typing import Dict, Any, Literal, Union import openpyxl import xlrd import yaml from cachalot.api import cachalot_disabled +from django.conf import settings from django.core.exceptions import FieldDoesNotExist from django.utils import timezone from openpyxl.utils.exceptions import InvalidFileException @@ -39,8 +41,6 @@ logger = logging.getLogger(__name__) class ImportService: - TEMP_DIR = "/usr/src/app/temp" - def __init__(self, import_run: ImportRun): self.import_run: ImportRun = import_run self.profile: ImportProfile = import_run.profile @@ -54,7 +54,7 @@ class ImportService: self.mapping: Dict[str, version_1.ColumnMapping] = self.config.mapping # Ensure temp directory exists - os.makedirs(self.TEMP_DIR, exist_ok=True) + os.makedirs(settings.IMPORT_TEMP_ROOT, exist_ok=True) def _load_config(self) -> version_1.ImportProfileSchema: yaml_data = yaml.safe_load(self.profile.yaml_config) @@ -1132,8 +1132,10 @@ class ImportService: Returns the absolute path. """ abs_path = os.path.abspath(file_path) - if not abs_path.startswith(self.TEMP_DIR): - raise ValueError(f"Invalid file path. File must be in {self.TEMP_DIR}") + if not Path(abs_path).is_relative_to(settings.IMPORT_TEMP_ROOT): + raise ValueError( + f"Invalid file path. File must be in {settings.IMPORT_TEMP_ROOT}" + ) return abs_path def process_file(self, file_path: str): diff --git a/app/apps/import_app/tests/test_paths.py b/app/apps/import_app/tests/test_paths.py new file mode 100644 index 00000000..33c6d32d --- /dev/null +++ b/app/apps/import_app/tests/test_paths.py @@ -0,0 +1,151 @@ +"""Tests for the folders import code reads from and writes to.""" + +import json +import os +import runpy +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +from django.conf import settings +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import SimpleTestCase, override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient + +from apps.import_app.models import ImportRun +from apps.import_app.services import PresetService +from apps.import_app.services.v1 import ImportService +from apps.import_app.tests.test_view_permissions import ( + HTMX, + ImportObjectPermissionTestCase, +) + + +def load_settings(**env): + """Evaluates the settings module from scratch under the given env vars.""" + with patch.dict(os.environ, env): + return runpy.run_path(str(Path(settings.BASE_DIR) / "WYGIWYH" / "settings.py")) + + +class PathSettingsTests(SimpleTestCase): + def test_defaults_match_the_docker_image_layout(self): + loaded = load_settings(IMPORT_TEMP_ROOT="", CACHE_ROOT="") + + base_dir = loaded["BASE_DIR"] + self.assertEqual(loaded["IMPORT_TEMP_ROOT"], base_dir / "temp") + self.assertEqual(loaded["IMPORT_PRESETS_ROOT"], base_dir / "import_presets") + self.assertEqual( + loaded["CACHES"]["default"]["LOCATION"], + Path("/var/tmp/django_cache").resolve(), + ) + self.assertEqual( + loaded["CACHES"]["update_check"]["LOCATION"], + Path("/var/tmp/django_cache_update_check").resolve(), + ) + + def test_env_vars_move_temp_and_cache_folders(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root).resolve() + loaded = load_settings( + IMPORT_TEMP_ROOT=str(root / "uploads"), CACHE_ROOT=str(root / "cache") + ) + + self.assertEqual(loaded["IMPORT_TEMP_ROOT"], root / "uploads") + self.assertEqual( + loaded["CACHES"]["default"]["LOCATION"], root / "cache" / "django_cache" + ) + self.assertEqual( + loaded["CACHES"]["update_check"]["LOCATION"], + root / "cache" / "django_cache_update_check", + ) + + +class PresetServiceTests(SimpleTestCase): + def test_lists_the_presets_shipped_with_the_app(self): + folder_names = { + preset["folder_name"] for preset in PresetService.get_all_presets() + } + + self.assertIn("qif_standard", folder_names) + + def test_reads_presets_from_configured_folder(self): + with tempfile.TemporaryDirectory() as root: + preset = Path(root) / "my_bank" + preset.mkdir() + (preset / "manifest.json").write_text(json.dumps({"name": "My Bank"})) + (preset / "config.yml").write_text("settings: {}") + + with override_settings(IMPORT_PRESETS_ROOT=Path(root)): + presets = PresetService.get_all_presets() + + self.assertEqual([p["name"] for p in presets], ["My Bank"]) + + +class ImportTempRootTests(ImportObjectPermissionTestCase): + def setUp(self): + super().setUp() + self.temp_root = Path(tempfile.mkdtemp()).resolve() + self.addCleanup(shutil.rmtree, self.temp_root, ignore_errors=True) + temp_root_override = override_settings(IMPORT_TEMP_ROOT=self.temp_root) + temp_root_override.enable() + self.addCleanup(temp_root_override.disable) + + @patch("apps.import_app.views.process_import.defer") + def test_web_upload_is_stored_in_temp_root(self, mock_defer): + self.login(self.owner) + response = self.client.post( + reverse("import_run_add", kwargs={"profile_id": self.private_profile.id}), + {"file": SimpleUploadedFile("web.csv", b"date,amount\n")}, + **HTMX, + ) + + self.assertEqual(response.status_code, 204) + self.assertEqual( + mock_defer.call_args.kwargs["file_path"], str(self.temp_root / "web.csv") + ) + self.assertTrue((self.temp_root / "web.csv").is_file()) + + @patch("apps.api.views.imports.process_import.defer") + def test_api_upload_is_stored_in_temp_root(self, mock_defer): + client = APIClient() + client.force_authenticate(user=self.owner) + response = client.post( + "/api/import/import/", + { + "profile_id": self.private_profile.id, + "file": SimpleUploadedFile("api.csv", b"date,amount\n"), + }, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual( + mock_defer.call_args.kwargs["file_path"], str(self.temp_root / "api.csv") + ) + self.assertTrue((self.temp_root / "api.csv").is_file()) + + def _service(self): + run = ImportRun.objects.create( + profile=self.private_profile, file_name="file.csv", owner=self.owner + ) + return ImportService(run) + + def test_service_accepts_files_inside_temp_root(self): + path = str(self.temp_root / "file.csv") + + self.assertEqual(self._service()._validate_file_path(path), path) + + def test_service_rejects_files_outside_temp_root(self): + service = self._service() + + for path in ( + "/etc/passwd", + self.temp_root / ".." / "file.csv", + # Shares the temp root as a string prefix, but is a different folder + f"{self.temp_root}-other/file.csv", + ): + with self.subTest(path=path), self.assertRaises(ValueError): + service._validate_file_path(str(path)) diff --git a/app/apps/import_app/tests/test_qif_import.py b/app/apps/import_app/tests/test_qif_import.py index a79662b3..d2b7e9db 100644 --- a/app/apps/import_app/tests/test_qif_import.py +++ b/app/apps/import_app/tests/test_qif_import.py @@ -1,7 +1,7 @@ from decimal import Decimal import os import shutil -from django.test import TestCase +from django.test import TestCase, override_settings from django.contrib.auth import get_user_model from apps.accounts.models import Account, AccountGroup from apps.currencies.models import Currency @@ -15,10 +15,11 @@ from apps.transactions.models import ( class QIFImportTests(TestCase): def setUp(self): - # Patch TEMP_DIR for testing - self.original_temp_dir = ImportService.TEMP_DIR + # Point the import temp folder somewhere disposable for testing self.test_dir = os.path.abspath("temp_test_import") - ImportService.TEMP_DIR = self.test_dir + temp_root_override = override_settings(IMPORT_TEMP_ROOT=self.test_dir) + temp_root_override.enable() + self.addCleanup(temp_root_override.disable) os.makedirs(self.test_dir, exist_ok=True) # Create user and set context @@ -41,7 +42,6 @@ class QIFImportTests(TestCase): def tearDown(self): delete_current_user() - ImportService.TEMP_DIR = self.original_temp_dir if os.path.exists(self.test_dir): shutil.rmtree(self.test_dir) diff --git a/app/apps/import_app/views.py b/app/apps/import_app/views.py index adef7679..45dc61ab 100644 --- a/app/apps/import_app/views.py +++ b/app/apps/import_app/views.py @@ -1,5 +1,6 @@ import shutil +from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied @@ -204,7 +205,7 @@ def import_run_add(request, profile_id): if form.is_valid(): uploaded_file = request.FILES["file"] - fs = FileSystemStorage(location="/usr/src/app/temp") + fs = FileSystemStorage(location=settings.IMPORT_TEMP_ROOT) filename = fs.save(uploaded_file.name, uploaded_file) file_path = fs.path(filename) From 43145be24daaaa620bcdde3bd06e13b6c3814a3b Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 02:57:07 -0300 Subject: [PATCH 03/16] feat(automatic-exchange-rates): add provider options field Add an Options field to exchange rate services. Providers get the raw text, can validate it on save through clean_options, and describe it with options_help. The service form now shows API Key only for providers that require one, and Options only for providers that read it, with that provider's help. AllRatesToday reads its bank code from Options instead of API Key. --- app/apps/currencies/exchange_rates/base.py | 10 +++- .../currencies/exchange_rates/providers.py | 30 +++++++--- app/apps/currencies/forms.py | 42 +++++++++++++ ...py => 0026_exchangerateservice_options.py} | 18 ++++-- app/apps/currencies/models.py | 15 +++-- .../tests/test_allratestoday_provider.py | 60 +++++++++++++++++-- 6 files changed, 151 insertions(+), 24 deletions(-) rename app/apps/currencies/migrations/{0026_alter_exchangerateservice_service_type.py => 0026_exchangerateservice_options.py} (69%) diff --git a/app/apps/currencies/exchange_rates/base.py b/app/apps/currencies/exchange_rates/base.py index 636d49fd..098ce5bd 100644 --- a/app/apps/currencies/exchange_rates/base.py +++ b/app/apps/currencies/exchange_rates/base.py @@ -8,9 +8,12 @@ from apps.currencies.models import Currency class ExchangeRateProvider(ABC): rates_inverted = False + # Help shown under the service's Options field. None hides the field + options_help = None - def __init__(self, api_key: Optional[str] = None): + def __init__(self, api_key: Optional[str] = None, options: str = ""): self.api_key = api_key + self.options = options or "" @abstractmethod def get_rates( @@ -24,6 +27,11 @@ class ExchangeRateProvider(ABC): """Return True if the service requires an API key""" return True + @classmethod + def clean_options(cls, options: str) -> str: + """Validate the raw Options text on save. Raise ValidationError if invalid""" + return options + @staticmethod def invert_rate(rate: Decimal) -> Decimal: """Invert the given rate.""" diff --git a/app/apps/currencies/exchange_rates/providers.py b/app/apps/currencies/exchange_rates/providers.py index eacb580b..a1ef9211 100644 --- a/app/apps/currencies/exchange_rates/providers.py +++ b/app/apps/currencies/exchange_rates/providers.py @@ -6,7 +6,9 @@ import requests from decimal import Decimal from typing import Tuple, List, Optional, Dict +from django.core.exceptions import ValidationError from django.db.models import QuerySet +from django.utils.translation import gettext_lazy as _ from apps.currencies.models import Currency, ExchangeRate from apps.currencies.exchange_rates.base import ExchangeRateProvider @@ -688,14 +690,18 @@ class AllRatesTodayProvider(ExchangeRateProvider): """ Implementation for the central bank tables served by allratestoday.com. - Each service reads the latest published table of one central bank. The API - Key field holds the bank code (e.g. "bcb" or "rbi"), as this endpoint needs - no key. An empty field reads the ECB table. + Each service reads the latest published table of one central bank, picked by + the bank code in the Options field (e.g. "bcb" or "rbi"). An empty field + reads the ECB table. The endpoint needs no API key. """ BASE_URL = "https://allratestoday.com/api/open/central-bank" DEFAULT_BANK = "ecb" + BANK_CODE_RE = re.compile(r"[a-z0-9_-]{2,20}") rates_inverted = False # A base/quote row says how many quote one base buys + options_help = _( + "Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table." + ) # A pair can have several rows. Prefer a published mid, else the mean of a buy # and sell quote. Period averages are left out, they are not a current rate @@ -710,15 +716,25 @@ class AllRatesTodayProvider(ExchangeRateProvider): ) BUY_SELL_TYPES = (("buy", "sell"), ("tt_buy", "tt_sell")) - def __init__(self, api_key: str = None): - super().__init__(api_key) - self.bank = (api_key or self.DEFAULT_BANK).strip().lower() + def __init__(self, api_key: str = None, options: str = ""): + super().__init__(api_key, options) self.session = requests.Session() + @property + def bank(self) -> str: + return self.options.strip().lower() or self.DEFAULT_BANK + @classmethod def requires_api_key(cls) -> bool: return False + @classmethod + def clean_options(cls, options: str) -> str: + bank = options.strip().lower() + if bank and not cls.BANK_CODE_RE.fullmatch(bank): + raise ValidationError(_("Invalid bank code, it should look like 'bcb'.")) + return bank + def _parse_table(self, rows: list) -> Dict[Tuple[str, str], Decimal]: """Reduce the table to one rate per pair, in both directions""" grouped = {} @@ -766,7 +782,7 @@ class AllRatesTodayProvider(ExchangeRateProvider): ) -> List[Tuple[Currency, Currency, Decimal]]: results = [] - if not re.fullmatch(r"[a-z0-9_-]{2,20}", self.bank): + if not self.BANK_CODE_RE.fullmatch(self.bank): logger.error("Invalid AllRatesToday bank code, it should look like 'bcb'") return results diff --git a/app/apps/currencies/forms.py b/app/apps/currencies/forms.py index 3c9ff64f..3090e8d5 100644 --- a/app/apps/currencies/forms.py +++ b/app/apps/currencies/forms.py @@ -1,13 +1,17 @@ +import json + from apps.common.widgets.crispy.daisyui import Switch from apps.common.widgets.crispy.submit import NoClassSubmit from apps.common.widgets.datepicker import AirDateTimePickerInput from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput from apps.common.widgets.tom_select import TomSelect +from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService from crispy_forms.bootstrap import FormActions from crispy_forms.helper import FormHelper from crispy_forms.layout import Column, Layout, Row from django import forms +from django.core.exceptions import ValidationError from django.forms import CharField from django.utils.translation import gettext_lazy as _ @@ -103,16 +107,41 @@ class ExchangeRateServiceForm(forms.ModelForm): "service_type", "is_active", "api_key", + "options", "interval_type", "fetch_interval", "target_currencies", "target_accounts", "singleton", ] + widgets = {"options": forms.Textarea(attrs={"rows": 2})} + + # Shows API Key and Options only for providers that use them + SERVICE_TYPE_SCRIPT = """ + init send change to me end + on change js(me) + const p = JSON.parse(me.dataset.providers)[me.value] || {}; + document.getElementById('div_id_api_key').classList.toggle('hidden', !p.api_key); + document.getElementById('div_id_options').classList.toggle('hidden', !p.options); + const help = document.getElementById('id_options_helptext'); + if (help) help.textContent = p.options || ''; + end + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + providers = { + key: { + "api_key": provider.requires_api_key(), + "options": str(provider.options_help or ""), + } + for key, provider in PROVIDER_MAPPING.items() + } + self.fields["service_type"].widget.attrs.update( + {"data-providers": json.dumps(providers), "_": self.SERVICE_TYPE_SCRIPT} + ) + self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" @@ -122,6 +151,7 @@ class ExchangeRateServiceForm(forms.ModelForm): Switch("is_active"), Switch("singleton"), "api_key", + "options", Row( Column("interval_type"), Column("fetch_interval"), @@ -142,3 +172,15 @@ class ExchangeRateServiceForm(forms.ModelForm): NoClassSubmit("submit", _("Add"), css_class="btn btn-primary"), ), ) + + def clean(self): + cleaned_data = super().clean() + provider = PROVIDER_MAPPING.get(cleaned_data.get("service_type")) + if provider: + try: + cleaned_data["options"] = provider.clean_options( + cleaned_data.get("options") or "" + ) + except ValidationError as e: + self.add_error("options", e) + return cleaned_data diff --git a/app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py b/app/apps/currencies/migrations/0026_exchangerateservice_options.py similarity index 69% rename from app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py rename to app/apps/currencies/migrations/0026_exchangerateservice_options.py index ffe19948..57681750 100644 --- a/app/apps/currencies/migrations/0026_alter_exchangerateservice_service_type.py +++ b/app/apps/currencies/migrations/0026_exchangerateservice_options.py @@ -1,15 +1,24 @@ -# Generated by Django 5.2.17 on 2026-11-06 22:51 +# Generated by Django 5.2.17 on 2026-11-08 05:52 from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ("currencies", "0025_alter_exchangerateservice_service_type"), ] operations = [ + migrations.AddField( + model_name="exchangerateservice", + name="options", + field=models.TextField( + blank=True, + default="", + help_text="Provider specific settings", + verbose_name="Options", + ), + ), migrations.AlterField( model_name="exchangerateservice", name="service_type", @@ -23,10 +32,7 @@ class Migration(migrations.Migration): ("twelvedatamarkets", "TwelveData Markets"), ("yfinance", "Yahoo Finance"), ("fxmacrodata", "FXMacroData"), - ( - "allratestoday", - "AllRatesToday Central Banks (bank code as API key)", - ), + ("allratestoday", "AllRatesToday Central Banks"), ], max_length=255, verbose_name="Service Type", diff --git a/app/apps/currencies/models.py b/app/apps/currencies/models.py index 36287000..cd01d36b 100644 --- a/app/apps/currencies/models.py +++ b/app/apps/currencies/models.py @@ -107,10 +107,7 @@ class ExchangeRateService(models.Model): TWELVEDATA_MARKETS = "twelvedatamarkets", "TwelveData Markets" YFINANCE = "yfinance", "Yahoo Finance" FXMACRODATA = "fxmacrodata", "FXMacroData" - ALLRATESTODAY = ( - "allratestoday", - "AllRatesToday Central Banks (bank code as API key)", - ) + ALLRATESTODAY = "allratestoday", "AllRatesToday Central Banks" class IntervalType(models.TextChoices): ON = "on", _("On") @@ -129,6 +126,12 @@ class ExchangeRateService(models.Model): verbose_name=_("API Key"), help_text=_("API key for the service (if required)"), ) + options = models.TextField( + blank=True, + default="", + verbose_name=_("Options"), + help_text=_("Provider specific settings"), + ) interval_type = models.CharField( max_length=255, choices=IntervalType.choices, @@ -184,7 +187,9 @@ class ExchangeRateService(models.Model): from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING provider_class = PROVIDER_MAPPING[self.service_type] - return provider_class(self.api_key) + provider = provider_class(self.api_key) + provider.options = self.options or "" + return provider @staticmethod def _parse_hour_ranges(interval_str: str) -> Set[int]: diff --git a/app/apps/currencies/tests/test_allratestoday_provider.py b/app/apps/currencies/tests/test_allratestoday_provider.py index b73d05d7..424dd9bb 100644 --- a/app/apps/currencies/tests/test_allratestoday_provider.py +++ b/app/apps/currencies/tests/test_allratestoday_provider.py @@ -1,11 +1,15 @@ +import json from decimal import Decimal from unittest import TestCase from unittest.mock import MagicMock, patch import requests +from django.core.exceptions import ValidationError +from django.test import TestCase as DjangoTestCase from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING from apps.currencies.exchange_rates.providers import AllRatesTodayProvider +from apps.currencies.forms import ExchangeRateServiceForm from apps.currencies.models import ExchangeRateService @@ -31,7 +35,7 @@ class AllRatesTodayProviderTests(TestCase): self.brl = _FakeCurrency("BRL") self.usd = _FakeCurrency("USD") self.eur = _FakeCurrency("EUR") - self.provider = AllRatesTodayProvider("bcb") + self.provider = AllRatesTodayProvider(options="bcb") def _get_rates(self, rows, target, base): target.exchange_currency = base @@ -67,14 +71,23 @@ class AllRatesTodayProviderTests(TestCase): self.assertEqual(len(rates), 2) def test_empty_field_reads_the_ecb_table(self): - self.assertEqual(AllRatesTodayProvider(None).bank, "ecb") - self.assertEqual(AllRatesTodayProvider("").bank, "ecb") + for options in (None, "", " "): + self.assertEqual(AllRatesTodayProvider(options=options).bank, "ecb") def test_normalizes_the_bank_code(self): - self.assertEqual(AllRatesTodayProvider(" BCB ").bank, "bcb") + self.assertEqual(AllRatesTodayProvider(options=" BCB ").bank, "bcb") + + def test_ignores_the_api_key(self): + self.assertEqual(AllRatesTodayProvider("bcb").bank, "ecb") + + def test_clean_options_validates_the_bank_code(self): + self.assertEqual(AllRatesTodayProvider.clean_options(" BCB "), "bcb") + self.assertEqual(AllRatesTodayProvider.clean_options(""), "") + with self.assertRaises(ValidationError): + AllRatesTodayProvider.clean_options("../latest/USD") def test_rejects_malformed_bank_code_without_a_request(self): - provider = AllRatesTodayProvider("../latest/USD") + provider = AllRatesTodayProvider(options="../latest/USD") self.usd.exchange_currency = self.brl with patch.object(provider.session, "get") as mock_get: @@ -206,3 +219,40 @@ class AllRatesTodayProviderTests(TestCase): self.assertFalse(AllRatesTodayProvider.requires_api_key()) self.assertIs(PROVIDER_MAPPING["allratestoday"], AllRatesTodayProvider) self.assertEqual(ExchangeRateService.ServiceType.ALLRATESTODAY, "allratestoday") + + def test_service_passes_its_options_to_the_provider(self): + service = ExchangeRateService(service_type="allratestoday", options="rbi") + self.assertEqual(service.get_provider().bank, "rbi") + + +class ExchangeRateServiceFormOptionsTests(DjangoTestCase): + def _form(self, **data): + return ExchangeRateServiceForm( + data={ + "name": "Rates", + "service_type": "allratestoday", + "interval_type": "every", + "fetch_interval": "24", + **data, + } + ) + + def test_saves_cleaned_options(self): + form = self._form(options=" BCB ") + + self.assertTrue(form.is_valid(), form.errors) + self.assertEqual(form.save().options, "bcb") + + def test_rejects_invalid_options_on_save(self): + form = self._form(options="../latest/USD") + + self.assertFalse(form.is_valid()) + self.assertIn("options", form.errors) + + def test_tells_the_page_which_fields_each_provider_uses(self): + attrs = self._form().fields["service_type"].widget.attrs + providers = json.loads(attrs["data-providers"]) + + self.assertEqual(providers["allratestoday"]["api_key"], False) + self.assertIn("bank code", providers["allratestoday"]["options"]) + self.assertEqual(providers["twelvedata"], {"api_key": True, "options": ""}) From 6a335b1a635d3cb4eb120914e36773c4f1a45bbb Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 02:57:07 -0300 Subject: [PATCH 04/16] fix(automatic-exchange-rates): skip bad AllRatesToday rows individually A non-numeric, NaN or infinite value raised while parsing and threw away the whole table. Each bad row is now skipped on its own. Log a warning when a table has rows but none of a usable rate type, so a bank with unknown types does not fail silently. --- .../currencies/exchange_rates/providers.py | 24 ++++++++++++++++--- .../tests/test_allratestoday_provider.py | 24 ++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/app/apps/currencies/exchange_rates/providers.py b/app/apps/currencies/exchange_rates/providers.py index a1ef9211..77c86e13 100644 --- a/app/apps/currencies/exchange_rates/providers.py +++ b/app/apps/currencies/exchange_rates/providers.py @@ -739,10 +739,17 @@ class AllRatesTodayProvider(ExchangeRateProvider): """Reduce the table to one rate per pair, in both directions""" grouped = {} for row in rows: - if not isinstance(row, dict) or not row.get("value"): + if not isinstance(row, dict): + continue + try: + value = Decimal(str(row.get("value"))) + except ArithmeticError: + continue + # A bad cell skips its own row, not the whole table + if not value.is_finite() or value <= 0: continue pair = (row.get("base"), row.get("quote")) - grouped.setdefault(pair, {})[row.get("type")] = Decimal(str(row["value"])) + grouped.setdefault(pair, {})[row.get("type")] = value rates = {} for (base, quote), values in grouped.items(): @@ -756,11 +763,22 @@ class AllRatesTodayProvider(ExchangeRateProvider): ), None, ) - if rate is None or rate <= 0: + if rate is None: + logger.debug( + f"AllRatesToday {self.bank}: no usable rate type for " + f"{base}/{quote}, got {sorted(map(str, values))}" + ) continue rates[(base, quote)] = rate rates.setdefault((quote, base), self.invert_rate(rate)) + if grouped and not rates: + types = sorted({str(t) for values in grouped.values() for t in values}) + logger.warning( + f"AllRatesToday table for {self.bank} has no usable rate types, " + f"got {types}" + ) + return rates @staticmethod diff --git a/app/apps/currencies/tests/test_allratestoday_provider.py b/app/apps/currencies/tests/test_allratestoday_provider.py index 424dd9bb..151ef94b 100644 --- a/app/apps/currencies/tests/test_allratestoday_provider.py +++ b/app/apps/currencies/tests/test_allratestoday_provider.py @@ -155,13 +155,35 @@ class AllRatesTodayProviderTests(TestCase): self.assertEqual(self._get_rates(rows, self.brl, self.usd), []) + def test_logs_a_table_without_usable_types(self): + self.usd.exchange_currency = self.brl + payload = {"rates": [_row("USD", "BRL", 5, "monthly_average")]} + + with patch.object( + self.provider.session, "get", return_value=_response(payload) + ): + with self.assertLogs( + "apps.currencies.exchange_rates.providers", level="WARNING" + ) as logs: + self.provider.get_rates([self.usd], {self.brl}) + + self.assertIn("monthly_average", logs.output[0]) + def test_skips_pair_missing_from_the_table(self): rows = [_row("USD", "BRL", 5)] self.assertEqual(self._get_rates(rows, self.eur, self.usd), []) def test_ignores_zero_and_malformed_rows(self): - rows = ["nope", _row("USD", "BRL", 0), _row("EUR", "BRL", 6)] + rows = [ + "nope", + _row("USD", "BRL", 0), + _row("USD", "BRL", "5,4"), + _row("USD", "BRL", None), + _row("USD", "BRL", "Infinity"), + _row("USD", "BRL", "NaN"), + _row("EUR", "BRL", 6), + ] rates = self._get_rates(rows, self.brl, self.eur) From a42265ba492300830dc433f96ccf7f7c0d006ac6 Mon Sep 17 00:00:00 2001 From: eitchtee <10510126+eitchtee@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:01:38 +0000 Subject: [PATCH 05/16] chore(locale): update translation files [skip ci] Automatically generated by Django makemessages workflow --- app/locale/de/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/en/LC_MESSAGES/django.po | 81 ++++++++++++++---------- app/locale/es/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/fr/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/hu/LC_MESSAGES/django.po | 79 ++++++++++++++--------- app/locale/id/LC_MESSAGES/django.po | 81 ++++++++++++++---------- app/locale/it/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/nl/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/pl/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/pt_BR/LC_MESSAGES/django.po | 81 +++++++++++++++--------- app/locale/ru/LC_MESSAGES/django.po | 79 ++++++++++++++--------- app/locale/sv/LC_MESSAGES/django.po | 81 ++++++++++++++---------- app/locale/sw/LC_MESSAGES/django.po | 81 ++++++++++++++---------- app/locale/uk/LC_MESSAGES/django.po | 79 ++++++++++++++--------- app/locale/zh_Hant/LC_MESSAGES/django.po | 81 +++++++++++++++--------- 15 files changed, 740 insertions(+), 469 deletions(-) diff --git a/app/locale/de/LC_MESSAGES/django.po b/app/locale/de/LC_MESSAGES/django.po index 782caca2..7371dcab 100644 --- a/app/locale/de/LC_MESSAGES/django.po +++ b/app/locale/de/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-02 19:49+0000\n" +"POT-Creation-Date: 2026-11-08 06:01+0000\n" "PO-Revision-Date: 2026-08-10 12:57+0000\n" "Last-Translator: GitEbu \n" "Language-Team: German \n" "Language-Team: LANGUAGE \n" @@ -23,8 +23,8 @@ msgid "Group name" msgstr "" #: apps/accounts/forms.py:43 apps/accounts/forms.py:109 -#: apps/currencies/forms.py:53 apps/currencies/forms.py:87 -#: apps/currencies/forms.py:136 apps/dca/forms.py:46 apps/dca/forms.py:205 +#: apps/currencies/forms.py:57 apps/currencies/forms.py:91 +#: apps/currencies/forms.py:166 apps/dca/forms.py:46 apps/dca/forms.py:205 #: apps/import_app/forms.py:32 apps/rules/forms.py:60 apps/rules/forms.py:100 #: apps/rules/forms.py:385 apps/transactions/forms.py:219 #: apps/transactions/forms.py:452 apps/transactions/forms.py:571 @@ -38,8 +38,8 @@ msgid "Update" msgstr "" #: apps/accounts/forms.py:49 apps/accounts/forms.py:115 -#: apps/common/widgets/tom_select.py:13 apps/currencies/forms.py:59 -#: apps/currencies/forms.py:93 apps/currencies/forms.py:142 +#: apps/common/widgets/tom_select.py:13 apps/currencies/forms.py:63 +#: apps/currencies/forms.py:97 apps/currencies/forms.py:172 #: apps/dca/forms.py:52 apps/dca/forms.py:211 apps/import_app/forms.py:38 #: apps/rules/forms.py:66 apps/rules/forms.py:106 apps/rules/forms.py:391 #: apps/transactions/forms.py:206 apps/transactions/forms.py:226 @@ -109,7 +109,7 @@ msgstr "" msgid "Balance against" msgstr "" -#: apps/accounts/forms.py:189 apps/currencies/forms.py:66 +#: apps/accounts/forms.py:189 apps/currencies/forms.py:70 #: apps/dca/models.py:158 apps/rules/forms.py:176 apps/rules/forms.py:190 #: apps/rules/models.py:38 apps/rules/models.py:279 #: apps/transactions/forms.py:86 apps/transactions/forms.py:482 @@ -496,11 +496,20 @@ msgstr "" msgid "No results..." msgstr "" -#: apps/currencies/forms.py:16 apps/currencies/models.py:22 +#: apps/currencies/exchange_rates/providers.py:703 +msgid "" +"Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table." +msgstr "" + +#: apps/currencies/exchange_rates/providers.py:735 +msgid "Invalid bank code, it should look like 'bcb'." +msgstr "" + +#: apps/currencies/forms.py:20 apps/currencies/models.py:22 msgid "Prefix" msgstr "" -#: apps/currencies/forms.py:17 apps/currencies/models.py:23 +#: apps/currencies/forms.py:21 apps/currencies/models.py:23 msgid "Suffix" msgstr "" @@ -562,27 +571,27 @@ msgstr "" msgid "From and To currencies cannot be the same." msgstr "" -#: apps/currencies/models.py:112 +#: apps/currencies/models.py:113 msgid "On" msgstr "" -#: apps/currencies/models.py:113 +#: apps/currencies/models.py:114 msgid "Every X hours" msgstr "" -#: apps/currencies/models.py:114 +#: apps/currencies/models.py:115 msgid "Not on" msgstr "" -#: apps/currencies/models.py:116 +#: apps/currencies/models.py:117 msgid "Service Name" msgstr "" -#: apps/currencies/models.py:118 +#: apps/currencies/models.py:119 msgid "Service Type" msgstr "" -#: apps/currencies/models.py:120 apps/transactions/filters.py:27 +#: apps/currencies/models.py:121 apps/transactions/filters.py:27 #: apps/transactions/models.py:253 apps/transactions/models.py:277 #: apps/transactions/models.py:301 templates/categories/fragments/list.html:16 #: templates/entities/fragments/list.html:16 @@ -594,77 +603,85 @@ msgstr "" msgid "Active" msgstr "" -#: apps/currencies/models.py:125 +#: apps/currencies/models.py:126 msgid "API Key" msgstr "" -#: apps/currencies/models.py:126 +#: apps/currencies/models.py:127 msgid "API key for the service (if required)" msgstr "" -#: apps/currencies/models.py:131 -msgid "Interval Type" +#: apps/currencies/models.py:132 +msgid "Options" msgstr "" -#: apps/currencies/models.py:135 -msgid "Interval" +#: apps/currencies/models.py:133 +msgid "Provider specific settings" msgstr "" #: apps/currencies/models.py:138 -msgid "Last Successful Fetch" +msgid "Interval Type" +msgstr "" + +#: apps/currencies/models.py:142 +msgid "Interval" msgstr "" #: apps/currencies/models.py:145 +msgid "Last Successful Fetch" +msgstr "" + +#: apps/currencies/models.py:152 msgid "Target Currencies" msgstr "" -#: apps/currencies/models.py:147 +#: apps/currencies/models.py:154 msgid "" "Select currencies to fetch exchange rates for. Rates will be fetched for " "each currency against their set exchange currency." msgstr "" -#: apps/currencies/models.py:155 +#: apps/currencies/models.py:162 msgid "Target Accounts" msgstr "" -#: apps/currencies/models.py:157 +#: apps/currencies/models.py:164 msgid "" "Select accounts to fetch exchange rates for. Rates will be fetched for each " "account's currency against their set exchange currency." msgstr "" -#: apps/currencies/models.py:164 +#: apps/currencies/models.py:171 msgid "Single exchange rate" msgstr "" -#: apps/currencies/models.py:167 +#: apps/currencies/models.py:174 msgid "Create one exchange rate and keep updating it. Avoids database clutter." msgstr "" -#: apps/currencies/models.py:172 +#: apps/currencies/models.py:179 msgid "Exchange Rate Service" msgstr "" -#: apps/currencies/models.py:173 +#: apps/currencies/models.py:180 msgid "Exchange Rate Services" msgstr "" -#: apps/currencies/models.py:225 +#: apps/currencies/models.py:234 msgid "'Every X hours' interval type requires a positive integer." msgstr "" -#: apps/currencies/models.py:234 +#: apps/currencies/models.py:243 msgid "'Every X hours' interval must be between 1 and 24." msgstr "" -#: apps/currencies/models.py:248 +#: apps/currencies/models.py:257 msgid "" "Invalid hour format. Use comma-separated hours (0-23) and/or ranges (e.g., " "'1-5,8,10-12')." msgstr "" -#: apps/currencies/models.py:259 +#: apps/currencies/models.py:268 msgid "" "Invalid format. Please check the requirements for your selected interval " "type." diff --git a/app/locale/es/LC_MESSAGES/django.po b/app/locale/es/LC_MESSAGES/django.po index 88a6a9a9..cc3089e0 100644 --- a/app/locale/es/LC_MESSAGES/django.po +++ b/app/locale/es/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-02 19:49+0000\n" +"POT-Creation-Date: 2026-11-08 06:01+0000\n" "PO-Revision-Date: 2026-07-10 04:57+0000\n" "Last-Translator: Juan David Afanador \n" "Language-Team: Spanish \n" "Language-Team: French \n" "Language-Team: Hungarian \n" "Language-Team: Italian \n" "Language-Team: Dutch \n" "Language-Team: Polish \n" "Language-Team: Portuguese (Brazil) \n" "Language-Team: Russian \n" "Language-Team: Swedish \n" "Language-Team: Swahili \n" "Language-Team: Ukrainian \n" "Language-Team: Chinese (Traditional Han script) Date: Sun, 8 Nov 2026 03:09:04 -0300 Subject: [PATCH 06/16] fix(import): resolve symlinks when checking the upload path The temp root is resolve()d in settings but the file path was only abspath()ed, so a symlink inside the temp folder could point outside it and still pass the check. Resolve both sides the same way. Also drop the unused /temp folder from the prod image. --- app/apps/import_app/services/v1.py | 3 ++- app/apps/import_app/tests/test_paths.py | 10 ++++++++++ docker/prod/django/Dockerfile | 4 +--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 5aad0b9e..2bc16981 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -1131,7 +1131,8 @@ class ImportService: Validates that the file path is within the allowed temporary directory. Returns the absolute path. """ - abs_path = os.path.abspath(file_path) + # resolve() like IMPORT_TEMP_ROOT, so symlinks can't point outside it + abs_path = str(Path(file_path).resolve()) if not Path(abs_path).is_relative_to(settings.IMPORT_TEMP_ROOT): raise ValueError( f"Invalid file path. File must be in {settings.IMPORT_TEMP_ROOT}" diff --git a/app/apps/import_app/tests/test_paths.py b/app/apps/import_app/tests/test_paths.py index 33c6d32d..a9eb7807 100644 --- a/app/apps/import_app/tests/test_paths.py +++ b/app/apps/import_app/tests/test_paths.py @@ -149,3 +149,13 @@ class ImportTempRootTests(ImportObjectPermissionTestCase): ): with self.subTest(path=path), self.assertRaises(ValueError): service._validate_file_path(str(path)) + + def test_service_rejects_symlinks_pointing_outside_temp_root(self): + link = self.temp_root / "link.csv" + try: + link.symlink_to("/etc/passwd") + except OSError: + self.skipTest("symlinks not available") + + with self.assertRaises(ValueError): + self._service()._validate_file_path(str(link)) diff --git a/docker/prod/django/Dockerfile b/docker/prod/django/Dockerfile index 9ad00cb2..fd83829f 100644 --- a/docker/prod/django/Dockerfile +++ b/docker/prod/django/Dockerfile @@ -69,9 +69,7 @@ RUN sed -i 's/\r$//g' /start && \ COPY --chown=app:app ./app . RUN chown -R app:app /usr/src/app && \ - chown -R app:app /usr/src/frontend && \ - mkdir /temp && \ - chown -R app:app /temp + chown -R app:app /usr/src/frontend USER app From de1df9d13c333e1292eb56f5676dadbcc38f352a Mon Sep 17 00:00:00 2001 From: eitchtee <10510126+eitchtee@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:13:26 +0000 Subject: [PATCH 07/16] chore(locale): update translation files [skip ci] Automatically generated by Django makemessages workflow --- app/locale/de/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/en/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/es/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/fr/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/hu/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/id/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/it/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/nl/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/pl/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/pt_BR/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/ru/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/sv/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/sw/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/uk/LC_MESSAGES/django.po | 18 +++++++++--------- app/locale/zh_Hant/LC_MESSAGES/django.po | 18 +++++++++--------- 15 files changed, 135 insertions(+), 135 deletions(-) diff --git a/app/locale/de/LC_MESSAGES/django.po b/app/locale/de/LC_MESSAGES/django.po index 7371dcab..cf8024de 100644 --- a/app/locale/de/LC_MESSAGES/django.po +++ b/app/locale/de/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-08 06:01+0000\n" +"POT-Creation-Date: 2026-11-08 06:13+0000\n" "PO-Revision-Date: 2026-08-10 12:57+0000\n" "Last-Translator: GitEbu \n" "Language-Team: German \n" "Language-Team: LANGUAGE \n" @@ -234,7 +234,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:109 #: apps/accounts/views/accounts.py:137 apps/dca/views.py:106 -#: apps/import_app/views.py:147 apps/rules/views.py:191 +#: apps/import_app/views.py:148 apps/rules/views.py:191 #: apps/transactions/views/categories.py:163 #: apps/transactions/views/entities.py:135 apps/transactions/views/tags.py:131 msgid "Item no longer shared with you" @@ -242,7 +242,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:132 #: apps/accounts/views/accounts.py:182 apps/dca/views.py:131 -#: apps/import_app/views.py:273 apps/rules/views.py:217 +#: apps/import_app/views.py:274 apps/rules/views.py:217 #: apps/transactions/views/categories.py:188 #: apps/transactions/views/entities.py:160 apps/transactions/views/tags.py:154 msgid "Ownership taken successfully" @@ -250,7 +250,7 @@ msgstr "" #: apps/accounts/views/account_groups.py:152 #: apps/accounts/views/accounts.py:106 apps/dca/views.py:151 -#: apps/import_app/views.py:294 apps/rules/views.py:238 +#: apps/import_app/views.py:295 apps/rules/views.py:238 #: apps/transactions/views/categories.py:130 #: apps/transactions/views/entities.py:180 apps/transactions/views/tags.py:174 msgid "Configuration saved successfully" @@ -1020,23 +1020,23 @@ msgstr "" msgid "File name" msgstr "" -#: apps/import_app/views.py:75 +#: apps/import_app/views.py:76 msgid "Import Profile added successfully" msgstr "" -#: apps/import_app/views.py:113 +#: apps/import_app/views.py:114 msgid "Import Profile update successfully" msgstr "" -#: apps/import_app/views.py:142 +#: apps/import_app/views.py:143 msgid "Import Profile deleted successfully" msgstr "" -#: apps/import_app/views.py:222 +#: apps/import_app/views.py:223 msgid "Import Run queued successfully" msgstr "" -#: apps/import_app/views.py:249 +#: apps/import_app/views.py:250 msgid "Run deleted successfully" msgstr "" diff --git a/app/locale/es/LC_MESSAGES/django.po b/app/locale/es/LC_MESSAGES/django.po index cc3089e0..e57d0adf 100644 --- a/app/locale/es/LC_MESSAGES/django.po +++ b/app/locale/es/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-11-08 06:01+0000\n" +"POT-Creation-Date: 2026-11-08 06:13+0000\n" "PO-Revision-Date: 2026-07-10 04:57+0000\n" "Last-Translator: Juan David Afanador \n" "Language-Team: Spanish \n" "Language-Team: French \n" "Language-Team: Hungarian \n" "Language-Team: Italian \n" "Language-Team: Dutch \n" "Language-Team: Polish \n" "Language-Team: Portuguese (Brazil) \n" "Language-Team: Russian \n" "Language-Team: Swedish \n" "Language-Team: Swahili \n" "Language-Team: Ukrainian \n" "Language-Team: Chinese (Traditional Han script) Date: Sun, 8 Nov 2026 03:17:33 -0300 Subject: [PATCH 08/16] docs(readme): give IMPORT_TEMP_ROOT a literal default and move path vars up List the Docker path as the default like the other rows do, and move both rows out from under the dev-only Vite settings. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d35ea5a..ba019170 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ To create the first user, open the container's console using Unraid's UI, by cli | ENABLE_SOFT_DELETE | true\|false | false | Whether to enable transactions soft delete, if enabled, deleted transactions will remain in the database. Useful for imports and avoiding duplicate entries. | | KEEP_DELETED_TRANSACTIONS_FOR | int | 365 | Time in days to keep soft deleted transactions for. If 0, will keep all transactions indefinitely. Only works if ENABLE_SOFT_DELETE is true. | | TASK_WORKERS | int | 1 | How many workers to have for async tasks. One should be enough for most use cases | +| IMPORT_TEMP_ROOT | string | /usr/src/app/temp | Where uploaded import files wait to be processed. Defaults to the `temp` folder inside the app folder. Must be writable and shared between the web and task processes. | +| CACHE_ROOT | string | /var/tmp | The writable folder the file-based caches are stored in. | | DEMO | true\|false | false | If demo mode is enabled. | | ADMIN_EMAIL | string | None | Automatically creates an admin account with this email. Must have `ADMIN_PASSWORD` also set. | | ADMIN_PASSWORD | string | None | Automatically creates an admin account with this password. Must have `ADMIN_EMAIL` also set. | @@ -199,8 +201,6 @@ To create the first user, open the container's console using Unraid's UI, by cli | DJANGO_VITE_DEV_MODE | true\|false | false | Enables Vite dev server mode for frontend development. When true, assets are served from Vite's dev server instead of the build manifest. For development only! | | DJANGO_VITE_DEV_SERVER_PORT | int | 5173 | The port where Vite's dev server is running. Only used when DJANGO_VITE_DEV_MODE is true. For development only! | | DJANGO_VITE_DEV_SERVER_HOST | string | localhost | The host where Vite's dev server is running. Only used when DJANGO_VITE_DEV_MODE is true. For development only! | -| IMPORT_TEMP_ROOT | string | temp, in the app folder | Where uploaded import files wait to be processed. Must be writable and shared between the web and task processes. | -| CACHE_ROOT | string | /var/tmp | The writable folder the file-based caches are stored in. | ## OIDC Configuration From 03f2b0aeb4248de3797ba911ee2e3732c462b25a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sun, 8 Nov 2026 03:26:11 -0300 Subject: [PATCH 09/16] fix(toasts): space the close button from the text and edges Lay the alert out as an icon | text | button grid so the close button gets a consistent gap from the message and the same visual inset as the icon. Long messages now wrap instead of pushing the button, and the aria-label is quoted. --- app/templates/common/fragments/toasts.html | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/app/templates/common/fragments/toasts.html b/app/templates/common/fragments/toasts.html index c3fbd18a..a6da99f3 100644 --- a/app/templates/common/fragments/toasts.html +++ b/app/templates/common/fragments/toasts.html @@ -2,25 +2,22 @@ {% load toast_bg %} {% if messages %} {% for message in messages %} -