Merge pull request #642 from cahthuranag/feat/allratestoday-provider

feat(automatic-exchange-rates): add central bank tables as a provider
This commit is contained in:
Herculino Trotta authored and GitHub committed 2026-11-08 03:01:13 -03:00
commit 03900ab4d1
7 files changed
+549 -2

No files matched your search

+9 -1
View File
@@ -8,9 +8,12 @@ from apps.currencies.models import Currency
class ExchangeRateProvider(ABC):
rates_inverted = False
# Help shown under the service's Options field. None hides the field
options_help = None
def __init__(self, api_key: Optional[str] = None):
def __init__(self, api_key: Optional[str] = None, options: str = ""):
self.api_key = api_key
self.options = options or ""
@abstractmethod
def get_rates(
@@ -24,6 +27,11 @@ class ExchangeRateProvider(ABC):
"""Return True if the service requires an API key"""
return True
@classmethod
def clean_options(cls, options: str) -> str:
"""Validate the raw Options text on save. Raise ValidationError if invalid"""
return options
@staticmethod
def invert_rate(rate: Decimal) -> Decimal:
"""Invert the given rate."""
@@ -19,6 +19,7 @@ PROVIDER_MAPPING = {
"twelvedatamarkets": providers.TwelveDataMarketsProvider,
"yfinance": providers.YFinanceMarketsProvider,
"fxmacrodata": providers.FXMacroDataProvider,
"allratestoday": providers.AllRatesTodayProvider,
}
@@ -1,11 +1,14 @@
import logging
import re
import time
import requests
from decimal import Decimal
from typing import Tuple, List, Optional, Dict
from django.core.exceptions import ValidationError
from django.db.models import QuerySet
from django.utils.translation import gettext_lazy as _
from apps.currencies.models import Currency, ExchangeRate
from apps.currencies.exchange_rates.base import ExchangeRateProvider
@@ -681,3 +684,166 @@ class FXMacroDataProvider(ExchangeRateProvider):
)
return results
class AllRatesTodayProvider(ExchangeRateProvider):
"""
Implementation for the central bank tables served by allratestoday.com.
Each service reads the latest published table of one central bank, picked by
the bank code in the Options field (e.g. "bcb" or "rbi"). An empty field
reads the ECB table. The endpoint needs no API key.
"""
BASE_URL = "https://allratestoday.com/api/open/central-bank"
DEFAULT_BANK = "ecb"
BANK_CODE_RE = re.compile(r"[a-z0-9_-]{2,20}")
rates_inverted = False # A base/quote row says how many quote one base buys
options_help = _(
"Central bank code, e.g. 'bcb' or 'rbi'. Leave empty to use the ECB table."
)
# A pair can have several rows. Prefer a published mid, else the mean of a buy
# and sell quote. Period averages are left out, they are not a current rate
MID_TYPES = (
"reference",
"middle",
"official",
"indicative",
"close",
"spot",
"informational",
)
BUY_SELL_TYPES = (("buy", "sell"), ("tt_buy", "tt_sell"))
def __init__(self, api_key: str = None, options: str = ""):
super().__init__(api_key, options)
self.session = requests.Session()
@property
def bank(self) -> str:
return self.options.strip().lower() or self.DEFAULT_BANK
@classmethod
def requires_api_key(cls) -> bool:
return False
@classmethod
def clean_options(cls, options: str) -> str:
bank = options.strip().lower()
if bank and not cls.BANK_CODE_RE.fullmatch(bank):
raise ValidationError(_("Invalid bank code, it should look like 'bcb'."))
return bank
def _parse_table(self, rows: list) -> Dict[Tuple[str, str], Decimal]:
"""Reduce the table to one rate per pair, in both directions"""
grouped = {}
for row in rows:
if not isinstance(row, dict):
continue
try:
value = Decimal(str(row.get("value")))
except ArithmeticError:
continue
# A bad cell skips its own row, not the whole table
if not value.is_finite() or value <= 0:
continue
pair = (row.get("base"), row.get("quote"))
grouped.setdefault(pair, {})[row.get("type")] = value
rates = {}
for (base, quote), values in grouped.items():
rate = next((values[t] for t in self.MID_TYPES if t in values), None)
if rate is None:
rate = next(
(
(values[buy] + values[sell]) / 2
for buy, sell in self.BUY_SELL_TYPES
if buy in values and sell in values
),
None,
)
if rate is None:
logger.debug(
f"AllRatesToday {self.bank}: no usable rate type for "
f"{base}/{quote}, got {sorted(map(str, values))}"
)
continue
rates[(base, quote)] = rate
rates.setdefault((quote, base), self.invert_rate(rate))
if grouped and not rates:
types = sorted({str(t) for values in grouped.values() for t in values})
logger.warning(
f"AllRatesToday table for {self.bank} has no usable rate types, "
f"got {types}"
)
return rates
@staticmethod
def _find_rate(
rates: Dict[Tuple[str, str], Decimal], base: str, quote: str
) -> Optional[Decimal]:
"""Look up a pair, crossing it through a shared currency if needed"""
if (base, quote) in rates:
return rates[(base, quote)]
for (from_code, via), rate in rates.items():
if from_code == base and (via, quote) in rates:
return rate * rates[(via, quote)]
return None
def get_rates(
self, target_currencies: QuerySet, exchange_currencies: set
) -> List[Tuple[Currency, Currency, Decimal]]:
results = []
if not self.BANK_CODE_RE.fullmatch(self.bank):
logger.error("Invalid AllRatesToday bank code, it should look like 'bcb'")
return results
try:
response = self.session.get(f"{self.BASE_URL}/{self.bank}", timeout=30)
response.raise_for_status()
data = response.json()
if data.get("stale"):
logger.warning(
f"AllRatesToday table for {self.bank} is stale, "
f"last published on {data.get('rate_date')}"
)
rates = self._parse_table(data["rates"])
except requests.RequestException as e:
logger.error(f"Error fetching AllRatesToday table for {self.bank}: {e}")
return results
except (KeyError, TypeError, ValueError, AttributeError, ArithmeticError) as e:
logger.error(
f"Unexpected response structure from AllRatesToday "
f"for {self.bank}: {e}"
)
return results
for target_currency in target_currencies:
if target_currency.exchange_currency not in exchange_currencies:
continue
base_currency = target_currency.exchange_currency
if base_currency.code == target_currency.code:
results.append((base_currency, target_currency, Decimal("1")))
continue
rate = self._find_rate(rates, base_currency.code, target_currency.code)
if rate is None:
logger.error(
f"No rate for {base_currency.code}/{target_currency.code} "
f"in the AllRatesToday table for {self.bank}"
)
continue
results.append((base_currency, target_currency, rate))
return results
+42
View File
@@ -1,13 +1,17 @@
import json
from apps.common.widgets.crispy.daisyui import Switch
from apps.common.widgets.crispy.submit import NoClassSubmit
from apps.common.widgets.datepicker import AirDateTimePickerInput
from apps.common.widgets.decimal import ArbitraryDecimalDisplayNumberInput
from apps.common.widgets.tom_select import TomSelect
from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING
from apps.currencies.models import Currency, ExchangeRate, ExchangeRateService
from crispy_forms.bootstrap import FormActions
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Column, Layout, Row
from django import forms
from django.core.exceptions import ValidationError
from django.forms import CharField
from django.utils.translation import gettext_lazy as _
@@ -103,16 +107,41 @@ class ExchangeRateServiceForm(forms.ModelForm):
"service_type",
"is_active",
"api_key",
"options",
"interval_type",
"fetch_interval",
"target_currencies",
"target_accounts",
"singleton",
]
widgets = {"options": forms.Textarea(attrs={"rows": 2})}
# Shows API Key and Options only for providers that use them
SERVICE_TYPE_SCRIPT = """
init send change to me end
on change js(me)
const p = JSON.parse(me.dataset.providers)[me.value] || {};
document.getElementById('div_id_api_key').classList.toggle('hidden', !p.api_key);
document.getElementById('div_id_options').classList.toggle('hidden', !p.options);
const help = document.getElementById('id_options_helptext');
if (help) help.textContent = p.options || '';
end
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
providers = {
key: {
"api_key": provider.requires_api_key(),
"options": str(provider.options_help or ""),
}
for key, provider in PROVIDER_MAPPING.items()
}
self.fields["service_type"].widget.attrs.update(
{"data-providers": json.dumps(providers), "_": self.SERVICE_TYPE_SCRIPT}
)
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.form_method = "post"
@@ -122,6 +151,7 @@ class ExchangeRateServiceForm(forms.ModelForm):
Switch("is_active"),
Switch("singleton"),
"api_key",
"options",
Row(
Column("interval_type"),
Column("fetch_interval"),
@@ -142,3 +172,15 @@ class ExchangeRateServiceForm(forms.ModelForm):
NoClassSubmit("submit", _("Add"), css_class="btn btn-primary"),
),
)
def clean(self):
cleaned_data = super().clean()
provider = PROVIDER_MAPPING.get(cleaned_data.get("service_type"))
if provider:
try:
cleaned_data["options"] = provider.clean_options(
cleaned_data.get("options") or ""
)
except ValidationError as e:
self.add_error("options", e)
return cleaned_data
@@ -0,0 +1,41 @@
# Generated by Django 5.2.17 on 2026-11-08 05:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("currencies", "0025_alter_exchangerateservice_service_type"),
]
operations = [
migrations.AddField(
model_name="exchangerateservice",
name="options",
field=models.TextField(
blank=True,
default="",
help_text="Provider specific settings",
verbose_name="Options",
),
),
migrations.AlterField(
model_name="exchangerateservice",
name="service_type",
field=models.CharField(
choices=[
("coingecko_free", "CoinGecko (Demo/Free)"),
("coingecko_pro", "CoinGecko (Pro)"),
("transitive", "Transitive (Calculated from Existing Rates)"),
("frankfurter", "Frankfurter"),
("twelvedata", "TwelveData"),
("twelvedatamarkets", "TwelveData Markets"),
("yfinance", "Yahoo Finance"),
("fxmacrodata", "FXMacroData"),
("allratestoday", "AllRatesToday Central Banks"),
],
max_length=255,
verbose_name="Service Type",
),
),
]
+10 -1
View File
@@ -107,6 +107,7 @@ class ExchangeRateService(models.Model):
TWELVEDATA_MARKETS = "twelvedatamarkets", "TwelveData Markets"
YFINANCE = "yfinance", "Yahoo Finance"
FXMACRODATA = "fxmacrodata", "FXMacroData"
ALLRATESTODAY = "allratestoday", "AllRatesToday Central Banks"
class IntervalType(models.TextChoices):
ON = "on", _("On")
@@ -125,6 +126,12 @@ class ExchangeRateService(models.Model):
verbose_name=_("API Key"),
help_text=_("API key for the service (if required)"),
)
options = models.TextField(
blank=True,
default="",
verbose_name=_("Options"),
help_text=_("Provider specific settings"),
)
interval_type = models.CharField(
max_length=255,
choices=IntervalType.choices,
@@ -180,7 +187,9 @@ class ExchangeRateService(models.Model):
from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING
provider_class = PROVIDER_MAPPING[self.service_type]
return provider_class(self.api_key)
provider = provider_class(self.api_key)
provider.options = self.options or ""
return provider
@staticmethod
def _parse_hour_ranges(interval_str: str) -> Set[int]:
@@ -0,0 +1,280 @@
import json
from decimal import Decimal
from unittest import TestCase
from unittest.mock import MagicMock, patch
import requests
from django.core.exceptions import ValidationError
from django.test import TestCase as DjangoTestCase
from apps.currencies.exchange_rates.fetcher import PROVIDER_MAPPING
from apps.currencies.exchange_rates.providers import AllRatesTodayProvider
from apps.currencies.forms import ExchangeRateServiceForm
from apps.currencies.models import ExchangeRateService
class _FakeCurrency:
def __init__(self, code, exchange_currency=None):
self.code = code
self.exchange_currency = exchange_currency
def _row(base, quote, value, type_="reference"):
return {"base": base, "quote": quote, "type": type_, "value": value}
def _response(payload):
response = MagicMock()
response.status_code = 200
response.json.return_value = payload
return response
class AllRatesTodayProviderTests(TestCase):
def setUp(self):
self.brl = _FakeCurrency("BRL")
self.usd = _FakeCurrency("USD")
self.eur = _FakeCurrency("EUR")
self.provider = AllRatesTodayProvider(options="bcb")
def _get_rates(self, rows, target, base):
target.exchange_currency = base
with patch.object(
self.provider.session, "get", return_value=_response({"rates": rows})
):
return self.provider.get_rates([target], {base})
def test_requests_the_configured_bank(self):
self.usd.exchange_currency = self.brl
payload = {"rates": [_row("USD", "BRL", 5.4)]}
with patch.object(
self.provider.session, "get", return_value=_response(payload)
) as mock_get:
self.provider.get_rates([self.usd], {self.brl})
mock_get.assert_called_once_with(
"https://allratestoday.com/api/open/central-bank/bcb", timeout=30
)
def test_fetches_one_table_for_all_pairs(self):
self.usd.exchange_currency = self.brl
self.eur.exchange_currency = self.brl
payload = {"rates": [_row("USD", "BRL", 5.4), _row("EUR", "BRL", 6.2)]}
with patch.object(
self.provider.session, "get", return_value=_response(payload)
) as mock_get:
rates = self.provider.get_rates([self.usd, self.eur], {self.brl})
mock_get.assert_called_once()
self.assertEqual(len(rates), 2)
def test_empty_field_reads_the_ecb_table(self):
for options in (None, "", " "):
self.assertEqual(AllRatesTodayProvider(options=options).bank, "ecb")
def test_normalizes_the_bank_code(self):
self.assertEqual(AllRatesTodayProvider(options=" BCB ").bank, "bcb")
def test_ignores_the_api_key(self):
self.assertEqual(AllRatesTodayProvider("bcb").bank, "ecb")
def test_clean_options_validates_the_bank_code(self):
self.assertEqual(AllRatesTodayProvider.clean_options(" BCB "), "bcb")
self.assertEqual(AllRatesTodayProvider.clean_options(""), "")
with self.assertRaises(ValidationError):
AllRatesTodayProvider.clean_options("../latest/USD")
def test_rejects_malformed_bank_code_without_a_request(self):
provider = AllRatesTodayProvider(options="../latest/USD")
self.usd.exchange_currency = self.brl
with patch.object(provider.session, "get") as mock_get:
rates = provider.get_rates([self.usd], {self.brl})
mock_get.assert_not_called()
self.assertEqual(rates, [])
def test_uses_a_row_in_the_requested_direction(self):
rates = self._get_rates([_row("EUR", "USD", 1.16)], self.usd, self.eur)
self.assertEqual(rates, [(self.eur, self.usd, Decimal("1.16"))])
def test_inverts_a_row_in_the_opposite_direction(self):
# bcb quotes foreign currencies in BRL, so BRL -> USD has to be inverted
rates = self._get_rates([_row("USD", "BRL", 5)], self.brl, self.usd)
self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))])
rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.brl)
self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))])
def test_crosses_pairs_through_the_table_home_currency(self):
rows = [_row("USD", "BRL", 5), _row("EUR", "BRL", 6)]
rates = self._get_rates(rows, self.eur, self.usd)
self.assertEqual(len(rates), 1)
self.assertAlmostEqual(rates[0][2], Decimal("0.8333333"), places=6)
def test_same_currency_is_one(self):
rates = self._get_rates([_row("USD", "BRL", 5)], self.usd, self.usd)
self.assertEqual(rates, [(self.usd, self.usd, Decimal("1"))])
def test_prefers_the_published_mid_over_buy_and_sell(self):
rows = [
_row("USD", "BRL", 5.3, "buy"),
_row("USD", "BRL", 5.4, "middle"),
_row("USD", "BRL", 5.7, "sell"),
]
rates = self._get_rates(rows, self.brl, self.usd)
self.assertEqual(rates, [(self.usd, self.brl, Decimal("5.4"))])
def test_averages_buy_and_sell_without_a_mid(self):
rows = [_row("USD", "BRL", 4, "buy"), _row("USD", "BRL", 6, "sell")]
rates = self._get_rates(rows, self.brl, self.usd)
self.assertEqual(rates, [(self.usd, self.brl, Decimal("5"))])
def test_averages_tt_buy_and_sell_without_a_mid(self):
rows = [
_row("USD", "MUR", 44.0, "tt_buy"),
_row("USD", "MUR", 46.0, "tt_sell"),
]
result = self._get_rates(rows, self.usd, _FakeCurrency("MUR"))
self.assertEqual(result[0][2], Decimal("1") / Decimal("45.0"))
def test_ignores_rows_of_other_types(self):
rows = [_row("USD", "BRL", 5, "monthly_average")]
self.assertEqual(self._get_rates(rows, self.brl, self.usd), [])
def test_logs_a_table_without_usable_types(self):
self.usd.exchange_currency = self.brl
payload = {"rates": [_row("USD", "BRL", 5, "monthly_average")]}
with patch.object(
self.provider.session, "get", return_value=_response(payload)
):
with self.assertLogs(
"apps.currencies.exchange_rates.providers", level="WARNING"
) as logs:
self.provider.get_rates([self.usd], {self.brl})
self.assertIn("monthly_average", logs.output[0])
def test_skips_pair_missing_from_the_table(self):
rows = [_row("USD", "BRL", 5)]
self.assertEqual(self._get_rates(rows, self.eur, self.usd), [])
def test_ignores_zero_and_malformed_rows(self):
rows = [
"nope",
_row("USD", "BRL", 0),
_row("USD", "BRL", "5,4"),
_row("USD", "BRL", None),
_row("USD", "BRL", "Infinity"),
_row("USD", "BRL", "NaN"),
_row("EUR", "BRL", 6),
]
rates = self._get_rates(rows, self.brl, self.eur)
self.assertEqual(rates, [(self.eur, self.brl, Decimal("6"))])
def test_logs_a_stale_table_and_still_uses_it(self):
self.usd.exchange_currency = self.brl
payload = {
"stale": True,
"rate_date": "2026-10-01",
"rates": [_row("BRL", "USD", 0.2)],
}
with patch.object(
self.provider.session, "get", return_value=_response(payload)
):
with self.assertLogs(
"apps.currencies.exchange_rates.providers", level="WARNING"
) as logs:
rates = self.provider.get_rates([self.usd], {self.brl})
self.assertIn("2026-10-01", logs.output[0])
self.assertEqual(rates, [(self.brl, self.usd, Decimal("0.2"))])
def test_returns_nothing_on_request_error(self):
self.usd.exchange_currency = self.brl
with patch.object(
self.provider.session,
"get",
side_effect=requests.RequestException("404 Not Found"),
):
rates = self.provider.get_rates([self.usd], {self.brl})
self.assertEqual(rates, [])
def test_returns_nothing_on_malformed_body(self):
self.usd.exchange_currency = self.brl
for payload in ({}, [], {"rates": None}):
with patch.object(
self.provider.session, "get", return_value=_response(payload)
):
self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), [])
def test_returns_nothing_on_non_json_body(self):
self.usd.exchange_currency = self.brl
response = _response({})
response.json.side_effect = ValueError("not json")
with patch.object(self.provider.session, "get", return_value=response):
self.assertEqual(self.provider.get_rates([self.usd], {self.brl}), [])
def test_is_registered_without_an_api_key(self):
self.assertFalse(AllRatesTodayProvider.requires_api_key())
self.assertIs(PROVIDER_MAPPING["allratestoday"], AllRatesTodayProvider)
self.assertEqual(ExchangeRateService.ServiceType.ALLRATESTODAY, "allratestoday")
def test_service_passes_its_options_to_the_provider(self):
service = ExchangeRateService(service_type="allratestoday", options="rbi")
self.assertEqual(service.get_provider().bank, "rbi")
class ExchangeRateServiceFormOptionsTests(DjangoTestCase):
def _form(self, **data):
return ExchangeRateServiceForm(
data={
"name": "Rates",
"service_type": "allratestoday",
"interval_type": "every",
"fetch_interval": "24",
**data,
}
)
def test_saves_cleaned_options(self):
form = self._form(options=" BCB ")
self.assertTrue(form.is_valid(), form.errors)
self.assertEqual(form.save().options, "bcb")
def test_rejects_invalid_options_on_save(self):
form = self._form(options="../latest/USD")
self.assertFalse(form.is_valid())
self.assertIn("options", form.errors)
def test_tells_the_page_which_fields_each_provider_uses(self):
attrs = self._form().fields["service_type"].widget.attrs
providers = json.loads(attrs["data-providers"])
self.assertEqual(providers["allratestoday"]["api_key"], False)
self.assertIn("bank code", providers["allratestoday"]["options"])
self.assertEqual(providers["twelvedata"], {"api_key": True, "options": ""})