Merge remote-tracking branch 'origin/main'

This commit is contained in:
Weblate committed 2026-11-20 02:44:56 +00:00
commit dddd757bb5
13 files changed
+1362 -7

No files matched your search

+11
View File
@@ -23,6 +23,7 @@ from apps.rules.models import (
TransactionRule,
TransactionRuleAction,
)
from apps.transactions.auto_tagging.service import apply_auto_tagging
from apps.transactions.models import (
SubTransaction,
Transaction,
@@ -849,6 +850,16 @@ def check_for_transaction_rules(
error="Initial trigger didn't match, this rule will be skipped",
)
_log("Initial trigger didn't match, this rule will be skipped")
if not dry_run and signal == "transaction_created":
try:
apply_auto_tagging(instance, user)
except Exception:
logger.error(
"Error while applying auto-tagging for transaction %s",
instance.id,
exc_info=True,
)
except Exception as e:
_log(
"** Error while executing 'check_for_transaction_rules' task",
+8
View File
@@ -1,6 +1,7 @@
from django.contrib import admin
from apps.transactions.models import (
AutoTaggerModel,
Transaction,
TransactionCategory,
TransactionTag,
@@ -79,3 +80,10 @@ class TransactionTagModelAdmin(SharedObjectModelAdmin):
@admin.register(TransactionEntity)
class TransactionEntityModelAdmin(SharedObjectModelAdmin):
pass
@admin.register(AutoTaggerModel)
class AutoTaggerModelAdmin(admin.ModelAdmin):
list_display = ["user", "transaction_count", "version", "trained_at"]
readonly_fields = ["trained_at", "transaction_count", "version"]
exclude = ["data"]
@@ -0,0 +1,21 @@
from apps.transactions.auto_tagging.classifier import (
MODEL_VERSION,
Prediction,
TrainingRecord,
TransactionAutoTagger,
)
from apps.transactions.auto_tagging.service import (
apply_auto_tagging,
ensure_model_fresh_async,
train_for_user,
)
__all__ = [
"MODEL_VERSION",
"Prediction",
"TrainingRecord",
"TransactionAutoTagger",
"apply_auto_tagging",
"ensure_model_fresh_async",
"train_for_user",
]
@@ -0,0 +1,427 @@
from __future__ import annotations
import heapq
import json
import math
import re
import unicodedata
import zlib
from collections import defaultdict
from dataclasses import dataclass, field
from decimal import Decimal
MODEL_VERSION = 1
MAX_TRAINING_DOCS = 5000
TOP_K = 10
MIN_SIMILARITY = 0.35
NEAR_EXACT_SIMILARITY = 0.85
MIN_SUPPORT_NEIGHBORS = 2
CATEGORY_CONFIDENCE_THRESHOLD = 0.65
MULTI_LABEL_CONFIDENCE_THRESHOLD = 0.60
# Strip dates (2025-01-20, 20/01/2025, 20/01) and installment counters (01/12)
_DATE_OR_INSTALLMENT_RE = re.compile(
r"\b\d{1,4}[-/.]\d{1,2}(?:[-/.]\d{1,4})?\b"
)
_WORD_RE = re.compile(r"[a-z0-9]+")
@dataclass(frozen=True)
class TrainingRecord:
description: str
notes: str
account_id: int
tx_type: str
amount: Decimal | float | int
category_id: int | None
tag_ids: tuple[int, ...] = ()
entity_ids: tuple[int, ...] = ()
@dataclass(frozen=True)
class Prediction:
category_id: int | None = None
tag_ids: tuple[int, ...] = ()
entity_ids: tuple[int, ...] = ()
@property
def is_empty(self) -> bool:
return (
self.category_id is None
and not self.tag_ids
and not self.entity_ids
)
def _normalize_text(text: str) -> str:
if not text:
return ""
decomposed = unicodedata.normalize("NFKD", text)
stripped = "".join(
ch for ch in decomposed if unicodedata.category(ch) != "Mn"
)
lowered = stripped.lower()
return _DATE_OR_INSTALLMENT_RE.sub(" ", lowered)
def _tokenize_text(text: str) -> list[str]:
normalized = _normalize_text(text)
if not normalized:
return []
return [
tok
for tok in _WORD_RE.findall(normalized)
if len(tok) >= 2 and not tok.isdigit()
]
def _add_text_features(
features: dict[str, float],
tokens: list[str],
*,
unigram_weight: float,
bigram_weight: float,
char_ngram_weight: float,
) -> None:
for tok in tokens:
features[tok] = features.get(tok, 0.0) + unigram_weight
if char_ngram_weight > 0.0 and len(tok) >= 6 and tok.isalpha():
for i in range(len(tok) - 3):
ngram = f"c:{tok[i : i + 4]}"
features[ngram] = features.get(ngram, 0.0) + char_ngram_weight
for i in range(len(tokens) - 1):
bigram = f"b:{tokens[i]}_{tokens[i + 1]}"
features[bigram] = features.get(bigram, 0.0) + bigram_weight
def _amount_bucket(amount: Decimal | float | int | None) -> int:
if amount is None:
return 0
val = abs(float(amount))
if val < 1.0:
return 0
return int(math.floor(math.log10(val))) + 1
def _extract_feature_parts(
description: str,
notes: str,
account_id: int,
tx_type: str,
amount: Decimal | float | int | None,
) -> tuple[dict[str, float], set[str], bool]:
"""Extract combined features plus the set of tokens from description.
Returns ``(features, desc_feature_keys, has_description)``.
"""
desc_tokens = _tokenize_text(description)
notes_tokens = _tokenize_text(notes)
if not desc_tokens and not notes_tokens:
return {}, set(), False
desc_features: dict[str, float] = {}
if desc_tokens:
_add_text_features(
desc_features,
desc_tokens,
unigram_weight=2.0,
bigram_weight=1.5,
char_ngram_weight=0.6,
)
features = dict(desc_features)
if notes_tokens:
_add_text_features(
features,
notes_tokens,
unigram_weight=1.0,
bigram_weight=0.75,
char_ngram_weight=0.0,
)
features[f"__type_{tx_type}"] = 0.5
features[f"__acct_{account_id}"] = 0.6
features[f"__amt_{_amount_bucket(amount)}"] = 0.4
return features, set(desc_features.keys()), bool(desc_tokens)
def extract_features(
description: str,
notes: str,
account_id: int,
tx_type: str,
amount: Decimal | float | int | None,
) -> dict[str, float]:
"""Extract weighted text + structured features for a transaction.
Returns an empty dict when neither description nor notes yields any
meaningful text tokens, preventing blind matches on account/type alone.
"""
features, _, _ = _extract_feature_parts(
description=description,
notes=notes,
account_id=account_id,
tx_type=tx_type,
amount=amount,
)
return features
@dataclass
class TransactionAutoTagger:
"""Zero-dependency TF-IDF + Cosine Similarity k-NN classifier."""
idf: dict[str, float] = field(default_factory=dict)
# Flat posting lists: [doc_idx_0, weight_0, doc_idx_1, weight_1, ...]
inverted_index: dict[str, list[float | int]] = field(default_factory=dict)
doc_categories: list[int | None] = field(default_factory=list)
doc_tags: list[list[int]] = field(default_factory=list)
doc_entities: list[list[int]] = field(default_factory=list)
doc_counts: list[int] = field(default_factory=list)
total_records: int = 0
version: int = MODEL_VERSION
@property
def num_docs(self) -> int:
return self.total_records or len(self.doc_categories)
@classmethod
def fit(cls, records: list[TrainingRecord]) -> TransactionAutoTagger:
# Collapse identical (features, labels) into weighted prototypes
prototypes: dict[tuple, int] = {}
doc_features: list[dict[str, float]] = []
doc_categories: list[int | None] = []
doc_tags: list[list[int]] = []
doc_entities: list[list[int]] = []
doc_counts: list[int] = []
df: dict[str, int] = defaultdict(int)
total_records = 0
for rec in records[:MAX_TRAINING_DOCS]:
feats = extract_features(
description=rec.description,
notes=rec.notes,
account_id=rec.account_id,
tx_type=rec.tx_type,
amount=rec.amount,
)
if not feats:
continue
total_records += 1
for tok in feats:
df[tok] += 1
sorted_tags = tuple(sorted(rec.tag_ids))
sorted_entities = tuple(sorted(rec.entity_ids))
sig = (
tuple(sorted(feats.items())),
rec.category_id,
sorted_tags,
sorted_entities,
)
existing_idx = prototypes.get(sig)
if existing_idx is not None:
doc_counts[existing_idx] += 1
else:
prototypes[sig] = len(doc_features)
doc_features.append(feats)
doc_categories.append(rec.category_id)
doc_tags.append(list(sorted_tags))
doc_entities.append(list(sorted_entities))
doc_counts.append(1)
if total_records == 0:
return cls()
idf = {
tok: round(math.log((1.0 + total_records) / (1.0 + count)) + 1.0, 4)
for tok, count in df.items()
}
inverted_index: dict[str, list[float | int]] = defaultdict(list)
for doc_idx, feats in enumerate(doc_features):
weights = {
tok: (1.0 + math.log(tf)) * idf[tok]
for tok, tf in feats.items()
}
norm = math.sqrt(sum(w * w for w in weights.values()))
if norm <= 0.0:
continue
for tok, w in weights.items():
postings = inverted_index[tok]
postings.append(doc_idx)
postings.append(round(w / norm, 4))
return cls(
idf=idf,
inverted_index=dict(inverted_index),
doc_categories=doc_categories,
doc_tags=doc_tags,
doc_entities=doc_entities,
doc_counts=doc_counts,
total_records=total_records,
version=MODEL_VERSION,
)
def to_bytes(self) -> bytes:
payload = {
"v": self.version,
"n": self.total_records,
"idf": self.idf,
"idx": self.inverted_index,
"cat": self.doc_categories,
"tag": self.doc_tags,
"ent": self.doc_entities,
"cnt": self.doc_counts,
}
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
return zlib.compress(raw, level=6)
@classmethod
def from_bytes(cls, blob: bytes | memoryview) -> TransactionAutoTagger | None:
if not blob:
return None
try:
payload = json.loads(zlib.decompress(bytes(blob)).decode("utf-8"))
except (zlib.error, json.JSONDecodeError, UnicodeDecodeError):
return None
if payload.get("v") != MODEL_VERSION or "cnt" not in payload:
return None
return cls(
idf=payload["idf"],
inverted_index=payload["idx"],
doc_categories=payload["cat"],
doc_tags=payload["tag"],
doc_entities=payload["ent"],
doc_counts=payload["cnt"],
total_records=payload.get("n", len(payload["cat"])),
version=payload["v"],
)
def predict(
self,
*,
description: str,
notes: str,
account_id: int,
tx_type: str,
amount: Decimal | float | int | None,
allowed_category_ids: set[int] | None = None,
allowed_tag_ids: set[int] | None = None,
allowed_entity_ids: set[int] | None = None,
) -> Prediction:
if self.num_docs == 0:
return Prediction()
feats, desc_feature_keys, has_description = _extract_feature_parts(
description=description,
notes=notes,
account_id=account_id,
tx_type=tx_type,
amount=amount,
)
if not feats:
return Prediction()
query_weights: dict[str, float] = {}
matched_text_tokens = 0
unknown_idf = math.log(1.0 + self.num_docs) + 1.0
norm_sq = 0.0
for tok, tf in feats.items():
tok_idf = self.idf.get(tok)
if tok_idf is not None:
w = (1.0 + math.log(tf)) * tok_idf
query_weights[tok] = w
norm_sq += w * w
if not tok.startswith("__"):
matched_text_tokens += 1
elif not tok.startswith("__"):
# Penalize unknown description tokens so unrelated descriptions
# abstain, but don't let unique notes (e.g. itemized details)
# discard a matching description.
if not has_description or tok in desc_feature_keys:
w = (1.0 + math.log(tf)) * unknown_idf
norm_sq += w * w
if matched_text_tokens == 0 or norm_sq <= 0.0:
return Prediction()
norm = math.sqrt(norm_sq)
scores: dict[int, float] = defaultdict(float)
for tok, w in query_weights.items():
q_w = w / norm
postings = self.inverted_index.get(tok)
if postings:
for i in range(0, len(postings), 2):
scores[postings[i]] += q_w * postings[i + 1]
candidates = [
(sim, doc_idx)
for doc_idx, sim in scores.items()
if sim >= MIN_SIMILARITY
]
if not candidates:
return Prediction()
top_neighbors = heapq.nlargest(TOP_K, candidates, key=lambda x: x[0])
total_support = sum(self.doc_counts[doc_idx] for _, doc_idx in top_neighbors)
if total_support < MIN_SUPPORT_NEIGHBORS:
if top_neighbors[0][0] < NEAR_EXACT_SIMILARITY:
return Prediction()
total_weight = 0.0
cat_weights: dict[int, float] = defaultdict(float)
cat_present_weight = 0.0
tag_weights: dict[int, float] = defaultdict(float)
entity_weights: dict[int, float] = defaultdict(float)
for sim, doc_idx in top_neighbors:
count = self.doc_counts[doc_idx]
w = (sim * sim) * (1.0 + math.log(count))
total_weight += w
cat_id = self.doc_categories[doc_idx]
if cat_id is not None:
cat_present_weight += w
cat_weights[cat_id] += w
for tid in self.doc_tags[doc_idx]:
tag_weights[tid] += w
for eid in self.doc_entities[doc_idx]:
entity_weights[eid] += w
predicted_category: int | None = None
if cat_weights and (cat_present_weight / total_weight) >= 0.5:
best_cat, best_w = max(cat_weights.items(), key=lambda item: item[1])
if (best_w / cat_present_weight) >= CATEGORY_CONFIDENCE_THRESHOLD:
if allowed_category_ids is None or best_cat in allowed_category_ids:
predicted_category = best_cat
predicted_tags = tuple(
sorted(
tid
for tid, w in tag_weights.items()
if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD
and (allowed_tag_ids is None or tid in allowed_tag_ids)
)
)
predicted_entities = tuple(
sorted(
eid
for eid, w in entity_weights.items()
if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD
and (allowed_entity_ids is None or eid in allowed_entity_ids)
)
)
return Prediction(
category_id=predicted_category,
tag_ids=predicted_tags,
entity_ids=predicted_entities,
)
@@ -0,0 +1,238 @@
from __future__ import annotations
import logging
from collections import OrderedDict, defaultdict
from datetime import timedelta
from django.db.models import Q
from django.utils import timezone
from apps.common.middleware.thread_local import (
delete_current_user,
get_current_user,
write_current_user,
)
from apps.transactions.auto_tagging.classifier import (
MAX_TRAINING_DOCS,
MODEL_VERSION,
Prediction,
TrainingRecord,
TransactionAutoTagger,
)
from apps.transactions.models import (
AutoTaggerModel,
Transaction,
TransactionCategory,
TransactionEntity,
TransactionTag,
)
logger = logging.getLogger(__name__)
STALE_MODEL_AGE = timedelta(hours=1)
_TAGGER_CACHE_MAXSIZE = 4
_tagger_cache: OrderedDict[tuple[int, str, int], TransactionAutoTagger] = OrderedDict()
def _get_cached_tagger(model_obj: AutoTaggerModel) -> TransactionAutoTagger | None:
cache_key = (model_obj.user_id, model_obj.trained_at.isoformat(), model_obj.version)
cached = _tagger_cache.get(cache_key)
if cached is not None:
_tagger_cache.move_to_end(cache_key)
return cached
tagger = TransactionAutoTagger.from_bytes(model_obj.data)
if tagger is not None:
_tagger_cache[cache_key] = tagger
while len(_tagger_cache) > _TAGGER_CACHE_MAXSIZE:
_tagger_cache.popitem(last=False)
return tagger
def _with_user_context(user):
"""Context manager that temporarily sets the thread-local current user."""
class _UserContext:
def __init__(self, target_user):
self.target_user = target_user
self.previous_user = None
def __enter__(self):
self.previous_user = get_current_user()
write_current_user(self.target_user)
return self.target_user
def __exit__(self, exc_type, exc_val, exc_tb):
if self.previous_user is not None:
write_current_user(self.previous_user)
elif get_current_user() is not None:
delete_current_user()
return _UserContext(user)
def train_for_user(user) -> AutoTaggerModel:
"""Train and persist the auto-tagging model for a single user."""
with _with_user_context(user):
qs = (
Transaction.objects.filter(linked_transaction__isnull=True)
.exclude(Q(description="") & Q(notes=""))
.filter(
Q(category__isnull=False)
| Q(tags__isnull=False)
| Q(entities__isnull=False)
)
.distinct()
.order_by("-date", "-id")
.values_list(
"id",
"description",
"notes",
"account_id",
"type",
"amount",
"category_id",
)[:MAX_TRAINING_DOCS]
)
rows = list(qs)
tx_ids = [row[0] for row in rows]
tags_by_tx: dict[int, list[int]] = defaultdict(list)
entities_by_tx: dict[int, list[int]] = defaultdict(list)
if tx_ids:
for tx_id, tag_id in Transaction.tags.through.objects.filter(
transaction_id__in=tx_ids
).values_list("transaction_id", "transactiontag_id"):
tags_by_tx[tx_id].append(tag_id)
for tx_id, entity_id in Transaction.entities.through.objects.filter(
transaction_id__in=tx_ids
).values_list("transaction_id", "transactionentity_id"):
entities_by_tx[tx_id].append(entity_id)
records = [
TrainingRecord(
description=description or "",
notes=notes or "",
account_id=account_id,
tx_type=tx_type,
amount=amount,
category_id=category_id,
tag_ids=tuple(tags_by_tx.get(tx_id, ())),
entity_ids=tuple(entities_by_tx.get(tx_id, ())),
)
for tx_id, description, notes, account_id, tx_type, amount, category_id in rows
]
tagger = TransactionAutoTagger.fit(records)
model_obj, _ = AutoTaggerModel.objects.update_or_create(
user=user,
defaults={
"data": tagger.to_bytes(),
"transaction_count": tagger.num_docs,
"version": MODEL_VERSION,
},
)
return model_obj
def ensure_model_fresh_async(user_id: int) -> None:
"""Defer a background retrain if Procrastinate is available."""
from apps.transactions.tasks import train_user_auto_tagger
try:
train_user_auto_tagger.configure(
queueing_lock=f"train_auto_tagger_{user_id}"
).defer(user_id=user_id)
except Exception:
logger.debug(
"Could not defer auto-tagger training for user %s",
user_id,
exc_info=True,
)
def _load_or_train_tagger(user) -> TransactionAutoTagger | None:
model_obj = AutoTaggerModel.objects.filter(user=user).first()
if model_obj is None or model_obj.version != MODEL_VERSION:
model_obj = train_for_user(user)
elif timezone.now() - model_obj.trained_at > STALE_MODEL_AGE:
ensure_model_fresh_async(user.id)
return _get_cached_tagger(model_obj)
def apply_auto_tagging(instance: Transaction, user) -> bool:
"""Auto-assign category, tags, and entities to an unclassified transaction.
Only runs when:
- The user has auto_tagging enabled in UserSettings
- The transaction is not deleted and not part of a transfer
- The transaction currently has NO category, NO tags, and NO entities
"""
user_settings = getattr(user, "settings", None)
if user_settings is None or not user_settings.auto_tagging:
return False
if instance.deleted or instance.linked_transaction_id is not None:
return False
if (
instance.category_id is not None
or Transaction.tags.through.objects.filter(transaction_id=instance.pk).exists()
or Transaction.entities.through.objects.filter(
transaction_id=instance.pk
).exists()
):
return False
with _with_user_context(user):
tagger = _load_or_train_tagger(user)
if tagger is None or tagger.num_docs == 0:
return False
allowed_category_ids = set(
TransactionCategory.objects.filter(
active=True, auto_match=True
).values_list("id", flat=True)
)
allowed_tag_ids = set(
TransactionTag.objects.filter(
active=True, auto_match=True
).values_list("id", flat=True)
)
allowed_entity_ids = set(
TransactionEntity.objects.filter(
active=True, auto_match=True
).values_list("id", flat=True)
)
prediction: Prediction = tagger.predict(
description=instance.description,
notes=instance.notes,
account_id=instance.account_id,
tx_type=instance.type,
amount=instance.amount,
allowed_category_ids=allowed_category_ids,
allowed_tag_ids=allowed_tag_ids,
allowed_entity_ids=allowed_entity_ids,
)
if prediction.is_empty:
return False
if prediction.category_id is not None:
Transaction.all_objects.filter(pk=instance.pk).update(
emit_signal=False,
category_id=prediction.category_id,
)
instance.category_id = prediction.category_id
if prediction.tag_ids:
instance.tags.add(*prediction.tag_ids)
if prediction.entity_ids:
instance.entities.add(*prediction.entity_ids)
return True
+19 -6
View File
@@ -1091,7 +1091,7 @@ class InstallmentPlanForm(SubTransactionsFormMixin, forms.ModelForm):
class TransactionTagForm(forms.ModelForm):
class Meta:
model = TransactionTag
fields = ["name", "active"]
fields = ["name", "active", "auto_match"]
labels = {"name": _("Tag name")}
def __init__(self, *args, **kwargs):
@@ -1100,7 +1100,11 @@ class TransactionTagForm(forms.ModelForm):
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.form_method = "post"
self.helper.layout = Layout(Field("name", css_class="mb-3"), Switch("active"))
self.helper.layout = Layout(
Field("name", css_class="mb-3"),
Switch("active"),
Switch("auto_match"),
)
if self.instance and self.instance.pk:
self.helper.layout.append(
@@ -1119,7 +1123,7 @@ class TransactionTagForm(forms.ModelForm):
class TransactionEntityForm(forms.ModelForm):
class Meta:
model = TransactionEntity
fields = ["name", "active"]
fields = ["name", "active", "auto_match"]
labels = {"name": _("Entity name")}
def __init__(self, *args, **kwargs):
@@ -1128,7 +1132,11 @@ class TransactionEntityForm(forms.ModelForm):
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.form_method = "post"
self.helper.layout = Layout(Field("name"), Switch("active"))
self.helper.layout = Layout(
Field("name"),
Switch("active"),
Switch("auto_match"),
)
if self.instance and self.instance.pk:
self.helper.layout.append(
@@ -1147,7 +1155,7 @@ class TransactionEntityForm(forms.ModelForm):
class TransactionCategoryForm(forms.ModelForm):
class Meta:
model = TransactionCategory
fields = ["name", "mute", "active"]
fields = ["name", "mute", "active", "auto_match"]
labels = {"name": _("Category name")}
help_texts = {
"mute": _("Muted categories won't be displayed on monthly summaries")
@@ -1159,7 +1167,12 @@ class TransactionCategoryForm(forms.ModelForm):
self.helper = FormHelper()
self.helper.form_tag = False
self.helper.form_method = "post"
self.helper.layout = Layout(Field("name"), Switch("mute"), Switch("active"))
self.helper.layout = Layout(
Field("name"),
Switch("mute"),
Switch("active"),
Switch("auto_match"),
)
if self.instance and self.instance.pk:
self.helper.layout.append(
@@ -0,0 +1,47 @@
# Generated by Django 5.2.17 on 2026-11-19 21:22
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('transactions', '0054_filterpreset_relative_to'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AddField(
model_name='transactioncategory',
name='auto_match',
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this category based on past transactions', verbose_name='Auto-match'),
),
migrations.AddField(
model_name='transactionentity',
name='auto_match',
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this entity based on past transactions', verbose_name='Auto-match'),
),
migrations.AddField(
model_name='transactiontag',
name='auto_match',
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this tag based on past transactions', verbose_name='Auto-match'),
),
migrations.CreateModel(
name='AutoTaggerModel',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('data', models.BinaryField(default=b'')),
('transaction_count', models.PositiveIntegerField(default=0)),
('version', models.PositiveSmallIntegerField(default=1)),
('trained_at', models.DateTimeField(auto_now=True)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='auto_tagger_model', to=settings.AUTH_USER_MODEL)),
],
options={
'verbose_name': 'Auto-Tagger Model',
'verbose_name_plural': 'Auto-Tagger Models',
'db_table': 'auto_tagger_models',
},
),
]
+41
View File
@@ -261,6 +261,13 @@ class TransactionCategory(SharedObject):
"Deactivated categories won't be able to be selected when creating new transactions"
),
)
auto_match = models.BooleanField(
default=True,
verbose_name=_("Auto-match"),
help_text=_(
"Allow auto-tagging to automatically assign this category based on past transactions"
),
)
objects = SharedObjectManager()
all_objects = models.Manager() # Unfiltered manager
@@ -285,6 +292,13 @@ class TransactionTag(SharedObject):
"Deactivated tags won't be able to be selected when creating new transactions"
),
)
auto_match = models.BooleanField(
default=True,
verbose_name=_("Auto-match"),
help_text=_(
"Allow auto-tagging to automatically assign this tag based on past transactions"
),
)
objects = SharedObjectManager()
all_objects = models.Manager() # Unfiltered manager
@@ -309,6 +323,13 @@ class TransactionEntity(SharedObject):
"Deactivated entities won't be able to be selected when creating new transactions"
),
)
auto_match = models.BooleanField(
default=True,
verbose_name=_("Auto-match"),
help_text=_(
"Allow auto-tagging to automatically assign this entity based on past transactions"
),
)
objects = SharedObjectManager()
all_objects = models.Manager() # Unfiltered manager
@@ -1567,3 +1588,23 @@ class QuickTransaction(TransferTemplate, OwnedObject):
def __str__(self):
return self.name
class AutoTaggerModel(models.Model):
user = models.OneToOneField(
settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
related_name="auto_tagger_model",
)
data = models.BinaryField(default=b"")
transaction_count = models.PositiveIntegerField(default=0)
version = models.PositiveSmallIntegerField(default=1)
trained_at = models.DateTimeField(auto_now=True)
class Meta:
verbose_name = _("Auto-Tagger Model")
verbose_name_plural = _("Auto-Tagger Models")
db_table = "auto_tagger_models"
def __str__(self):
return f"AutoTaggerModel({self.user_id}, n={self.transaction_count})"
+49 -1
View File
@@ -2,12 +2,15 @@ import logging
from datetime import timedelta
from cachalot.api import cachalot_disabled, invalidate
from django.contrib.auth import get_user_model
from django.db.models import Max
from django.utils import timezone
from django.conf import settings
from procrastinate.contrib.django import app
from apps.transactions.models import RecurringTransaction, Transaction
from apps.transactions.auto_tagging.service import _with_user_context, train_for_user
from apps.transactions.models import AutoTaggerModel, RecurringTransaction, Transaction
logger = logging.getLogger(__name__)
@@ -50,3 +53,48 @@ def cleanup_deleted_transactions(timestamp=None):
deleted_count, _ = old_transactions.hard_delete()
return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days."
@app.task(name="train_user_auto_tagger")
def train_user_auto_tagger(user_id: int):
user = get_user_model().objects.filter(id=user_id).first()
if user is None:
return "User not found."
user_settings = getattr(user, "settings", None)
if user_settings is None or not user_settings.auto_tagging:
return "Auto-tagging disabled for user."
with cachalot_disabled():
model_obj = train_for_user(user)
return f"Trained auto-tagger for user {user_id} on {model_obj.transaction_count} transactions."
@app.periodic(cron="30 2 * * *")
@app.task(lock="retrain_auto_taggers", name="retrain_auto_taggers")
def retrain_auto_taggers(timestamp=None):
User = get_user_model()
users = User.objects.filter(settings__auto_tagging=True, is_active=True).select_related(
"settings"
)
existing_models = {
m.user_id: m.trained_at
for m in AutoTaggerModel.objects.filter(user__in=users)
}
retrained = 0
with cachalot_disabled():
for user in users:
trained_at = existing_models.get(user.id)
with _with_user_context(user):
latest_tx = Transaction.objects.aggregate(latest=Max("updated_at"))[
"latest"
]
if trained_at is not None and (
latest_tx is None or latest_tx <= trained_at
):
continue
train_for_user(user)
retrained += 1
return f"Retrained auto-tagger for {retrained} user(s)."
@@ -0,0 +1,474 @@
import datetime
from decimal import Decimal
from unittest.mock import patch
from django.contrib.auth import get_user_model
from django.test import TestCase
from apps.accounts.models import Account, AccountGroup
from apps.common.middleware.thread_local import (
delete_current_user,
get_current_user,
write_current_user,
)
from apps.currencies.models import Currency
from apps.rules.models import TransactionRule, TransactionRuleAction
from apps.rules.tasks import check_for_transaction_rules
from apps.transactions.auto_tagging.classifier import (
TrainingRecord,
TransactionAutoTagger,
)
from apps.transactions.auto_tagging.service import apply_auto_tagging, train_for_user
from apps.transactions.forms import (
TransactionCategoryForm,
TransactionEntityForm,
TransactionTagForm,
)
from apps.transactions.models import (
AutoTaggerModel,
Transaction,
TransactionCategory,
TransactionEntity,
TransactionTag,
)
from apps.transactions.tasks import retrain_auto_taggers, train_user_auto_tagger
from apps.users.forms import UserSettingsForm
User = get_user_model()
class TransactionAutoTaggerClassifierTests(TestCase):
def _sample_records(self) -> list[TrainingRecord]:
return [
TrainingRecord(
description="UBER *TRIP HELP.UBER.COM 12/01",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("24.50"),
category_id=10,
tag_ids=(100,),
entity_ids=(200,),
),
TrainingRecord(
description="UBER *TRIP 15/01/2025",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("18.90"),
category_id=10,
tag_ids=(100,),
entity_ids=(200,),
),
TrainingRecord(
description="UBER *TRIP 2025-01-20",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("31.00"),
category_id=10,
tag_ids=(100,),
entity_ids=(200,),
),
TrainingRecord(
description="PIX PADARIA SÃO JOÃO",
notes="breakfast",
account_id=1,
tx_type="EX",
amount=Decimal("15.00"),
category_id=20,
tag_ids=(101, 102),
entity_ids=(201,),
),
TrainingRecord(
description="PIX PADARIA SAO JOAO",
notes="coffee",
account_id=1,
tx_type="EX",
amount=Decimal("12.00"),
category_id=20,
tag_ids=(101, 102),
entity_ids=(201,),
),
]
def test_predicts_category_tags_and_entities_with_noise_and_accents(self):
tagger = TransactionAutoTagger.fit(self._sample_records())
uber_pred = tagger.predict(
description="UBER *TRIP 03/02/2025",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("22.00"),
)
self.assertEqual(uber_pred.category_id, 10)
self.assertEqual(uber_pred.tag_ids, (100,))
self.assertEqual(uber_pred.entity_ids, (200,))
bakery_pred = tagger.predict(
description="PIX PADARIA SAO JOAO 04/02",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("14.00"),
)
self.assertEqual(bakery_pred.category_id, 20)
self.assertEqual(bakery_pred.tag_ids, (101, 102))
self.assertEqual(bakery_pred.entity_ids, (201,))
def test_serialization_roundtrip_and_invalid_blob(self):
tagger = TransactionAutoTagger.fit(self._sample_records())
blob = tagger.to_bytes()
restored = TransactionAutoTagger.from_bytes(blob)
self.assertIsNotNone(restored)
self.assertEqual(restored.num_docs, tagger.num_docs)
pred = restored.predict(
description="UBER *TRIP",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("20.00"),
)
self.assertEqual(pred.category_id, 10)
self.assertIsNone(TransactionAutoTagger.from_bytes(b""))
self.assertIsNone(TransactionAutoTagger.from_bytes(b"not-valid-zlib"))
def test_abstains_on_empty_text_or_unrelated_description(self):
tagger = TransactionAutoTagger.fit(self._sample_records())
empty_pred = tagger.predict(
description="",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("20.00"),
)
self.assertTrue(empty_pred.is_empty)
unrelated_pred = tagger.predict(
description="COMPLETELY UNKNOWN MERCHANT XYZ",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("20.00"),
)
self.assertTrue(unrelated_pred.is_empty)
# Sharing only 1 weak token among mostly unknown words must still abstain
weak_overlap_pred = tagger.predict(
description="COMPLETELY RANDOM TRIP THING",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("20.00"),
)
self.assertTrue(weak_overlap_pred.is_empty)
def test_unique_notes_do_not_discard_matching_description(self):
tagger = TransactionAutoTagger.fit(self._sample_records())
pred = tagger.predict(
description="UBER *TRIP 05/02",
notes="airport pickup for visiting cousins with heavy suitcases",
account_id=1,
tx_type="EX",
amount=Decimal("25.00"),
)
self.assertEqual(pred.category_id, 10)
self.assertEqual(pred.tag_ids, (100,))
self.assertEqual(pred.entity_ids, (200,))
def test_respects_allowed_label_sets(self):
tagger = TransactionAutoTagger.fit(self._sample_records())
pred = tagger.predict(
description="UBER *TRIP",
notes="",
account_id=1,
tx_type="EX",
amount=Decimal("20.00"),
allowed_category_ids=set(),
allowed_tag_ids={100},
allowed_entity_ids=set(),
)
self.assertIsNone(pred.category_id)
self.assertEqual(pred.tag_ids, (100,))
self.assertEqual(pred.entity_ids, ())
class AutoTaggingServiceAndRulesTests(TestCase):
def setUp(self):
self.user = User.objects.create_user(
email="alice@example.com", password="password123"
)
self.other_user = User.objects.create_user(
email="bob@example.com", password="password123"
)
self.currency = Currency.objects.create(
code="USD", name="US Dollar", decimal_places=2, prefix="$ "
)
self.group = AccountGroup.objects.create(name="Checking", owner=self.user)
self.account = Account.objects.create(
name="Main Card",
group=self.group,
currency=self.currency,
owner=self.user,
visibility="private",
)
self.second_account = Account.objects.create(
name="Savings",
group=self.group,
currency=self.currency,
owner=self.user,
visibility="private",
)
self.cat_transport = TransactionCategory.objects.create(
name="Transport", owner=self.user, visibility="private"
)
self.cat_other = TransactionCategory.objects.create(
name="Other", owner=self.user, visibility="private"
)
self.tag_commute = TransactionTag.objects.create(
name="Commute", owner=self.user, visibility="private"
)
self.tag_rule = TransactionTag.objects.create(
name="RuleApplied", owner=self.user, visibility="private"
)
self.entity_uber = TransactionEntity.objects.create(
name="Uber", owner=self.user, visibility="private"
)
# Seed historical labeled transactions for Alice
for day in (1, 5, 10):
tx = Transaction.objects.create(
owner=self.user,
account=self.account,
type=Transaction.Type.EXPENSE,
date=datetime.date(2025, 1, day),
amount=Decimal("25.00"),
description=f"UBER *TRIP 0{day}/01",
category=self.cat_transport,
)
tx.tags.add(self.tag_commute)
tx.entities.add(self.entity_uber)
def tearDown(self):
if get_current_user() is not None:
delete_current_user()
def _tag_ids(self, tx: Transaction) -> list[int]:
return list(
Transaction.tags.through.objects.filter(transaction_id=tx.pk).values_list(
"transactiontag_id", flat=True
)
)
def _entity_ids(self, tx: Transaction) -> list[int]:
return list(
Transaction.entities.through.objects.filter(
transaction_id=tx.pk
).values_list("transactionentity_id", flat=True)
)
def _create_unclassified_uber_tx(self, **overrides) -> Transaction:
defaults = {
"owner": self.user,
"account": self.account,
"type": Transaction.Type.EXPENSE,
"date": datetime.date(2025, 2, 1),
"amount": Decimal("27.00"),
"description": "UBER *TRIP 01/02",
}
defaults.update(overrides)
return Transaction.objects.create(**defaults)
def test_auto_tags_unclassified_transaction_after_rules(self):
tx = self._create_unclassified_uber_tx()
check_for_transaction_rules(
instance_id=tx.id,
user_id=self.user.id,
signal="transaction_created",
)
tx.refresh_from_db()
self.assertEqual(tx.category_id, self.cat_transport.id)
self.assertEqual(self._tag_ids(tx), [self.tag_commute.id])
self.assertEqual(self._entity_ids(tx), [self.entity_uber.id])
def test_skips_when_user_already_set_category_tag_or_entity(self):
train_for_user(self.user)
tx_with_cat = self._create_unclassified_uber_tx(category=self.cat_other)
self.assertFalse(apply_auto_tagging(tx_with_cat, self.user))
tx_with_cat.refresh_from_db()
self.assertEqual(tx_with_cat.category_id, self.cat_other.id)
self.assertEqual(self._tag_ids(tx_with_cat), [])
self.assertEqual(self._entity_ids(tx_with_cat), [])
tx_with_tag = self._create_unclassified_uber_tx()
tx_with_tag.tags.add(self.tag_rule)
self.assertFalse(apply_auto_tagging(tx_with_tag, self.user))
tx_with_tag.refresh_from_db()
self.assertIsNone(tx_with_tag.category_id)
self.assertEqual(self._entity_ids(tx_with_tag), [])
tx_with_entity = self._create_unclassified_uber_tx()
tx_with_entity.entities.add(self.entity_uber)
self.assertFalse(apply_auto_tagging(tx_with_entity, self.user))
tx_with_entity.refresh_from_db()
self.assertIsNone(tx_with_entity.category_id)
self.assertEqual(self._tag_ids(tx_with_entity), [])
def test_rules_run_before_auto_tagging_and_prevent_auto_tagging_if_rule_classifies(self):
rule = TransactionRule.objects.create(
owner=self.user,
visibility="private",
name="Tag Uber via Rule",
trigger="'UBER' in description",
on_create=True,
active=True,
)
TransactionRuleAction.objects.create(
rule=rule,
field=TransactionRuleAction.Field.tags,
value=f"['{self.tag_rule.name}']",
)
tx = self._create_unclassified_uber_tx()
check_for_transaction_rules(
instance_id=tx.id,
user_id=self.user.id,
signal="transaction_created",
)
tx.refresh_from_db()
# Rule set the tag; auto-tagging should see the transaction is already classified and skip it
self.assertEqual(self._tag_ids(tx), [self.tag_rule.id])
self.assertIsNone(tx.category_id)
self.assertEqual(self._entity_ids(tx), [])
def test_user_settings_disable_auto_tagging(self):
self.user.settings.auto_tagging = False
self.user.settings.save()
tx = self._create_unclassified_uber_tx()
check_for_transaction_rules(
instance_id=tx.id,
user_id=self.user.id,
signal="transaction_created",
)
tx.refresh_from_db()
self.assertIsNone(tx.category_id)
self.assertEqual(self._tag_ids(tx), [])
self.assertEqual(self._entity_ids(tx), [])
def test_per_item_auto_match_opt_out_and_inactive_items(self):
train_for_user(self.user)
self.cat_transport.auto_match = False
self.cat_transport.save()
self.tag_commute.active = False
self.tag_commute.save()
tx = self._create_unclassified_uber_tx()
applied = apply_auto_tagging(tx, self.user)
self.assertTrue(applied)
tx.refresh_from_db()
self.assertIsNone(tx.category_id)
self.assertEqual(self._tag_ids(tx), [])
self.assertEqual(self._entity_ids(tx), [self.entity_uber.id])
def test_skips_transfers(self):
train_for_user(self.user)
tx_from = self._create_unclassified_uber_tx()
tx_from.add_transfer_side(self.second_account, Decimal("27.00"))
tx_from.refresh_from_db()
self.assertFalse(apply_auto_tagging(tx_from, self.user))
self.assertIsNone(tx_from.category_id)
def test_multi_user_isolation(self):
train_for_user(self.user)
bob_group = AccountGroup.objects.create(name="Bob Group", owner=self.other_user)
bob_account = Account.objects.create(
name="Bob Card",
group=bob_group,
currency=self.currency,
owner=self.other_user,
visibility="private",
)
bob_tx = Transaction.objects.create(
owner=self.other_user,
account=bob_account,
type=Transaction.Type.EXPENSE,
date=datetime.date(2025, 2, 1),
amount=Decimal("25.00"),
description="UBER *TRIP 01/02",
)
self.assertFalse(apply_auto_tagging(bob_tx, self.other_user))
bob_tx.refresh_from_db()
self.assertIsNone(bob_tx.category_id)
self.assertFalse(bob_tx.tags.exists())
self.assertFalse(bob_tx.entities.exists())
def test_procrastinate_training_tasks(self):
msg = train_user_auto_tagger(self.user.id)
self.assertIn("Trained auto-tagger", msg)
model_obj = AutoTaggerModel.objects.get(user=self.user)
self.assertEqual(model_obj.transaction_count, 3)
# Periodic task skips already-up-to-date user
with patch(
"apps.transactions.tasks.train_for_user", wraps=train_for_user
) as mock_train:
retrain_auto_taggers()
trained_user_ids = [call.args[0].id for call in mock_train.call_args_list]
self.assertNotIn(self.user.id, trained_user_ids)
def test_retrain_auto_taggers_detects_shared_account_updates(self):
# Train Bob initially (0 transactions visible)
train_for_user(self.other_user)
self.assertEqual(
AutoTaggerModel.objects.get(user=self.other_user).transaction_count, 0
)
# Share Alice's account with Bob and add a new transaction owned by Alice
self.account.shared_with.add(self.other_user)
Transaction.objects.create(
owner=self.user,
account=self.account,
type=Transaction.Type.EXPENSE,
date=datetime.date(2025, 2, 2),
amount=Decimal("30.00"),
description="UBER *TRIP SHARED",
category=self.cat_transport,
)
retrain_auto_taggers()
bob_model = AutoTaggerModel.objects.get(user=self.other_user)
self.assertGreater(bob_model.transaction_count, 0)
def test_forms_expose_auto_tagging_and_auto_match_fields(self):
write_current_user(self.user)
settings_form = UserSettingsForm(instance=self.user.settings)
self.assertIn("auto_tagging", settings_form.fields)
cat_form = TransactionCategoryForm(instance=self.cat_transport)
self.assertIn("auto_match", cat_form.fields)
tag_form = TransactionTagForm(instance=self.tag_commute)
self.assertIn("auto_match", tag_form.fields)
entity_form = TransactionEntityForm(instance=self.entity_uber)
self.assertIn("auto_match", entity_form.fields)
+2
View File
@@ -158,6 +158,7 @@ class UserSettingsForm(forms.ModelForm):
"number_format",
"default_account",
"combine_transfers",
"auto_tagging",
"volume",
]
widgets = {
@@ -185,6 +186,7 @@ class UserSettingsForm(forms.ModelForm):
"start_page",
"default_account",
Switch("combine_transfers"),
Switch("auto_tagging"),
HTML('<hr class="hr my-3" />'),
Range("volume"),
FormActions(
@@ -0,0 +1,18 @@
# Generated by Django 5.2.17 on 2026-11-19 21:22
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('users', '0029_usersettings_combine_transfers'),
]
operations = [
migrations.AddField(
model_name='usersettings',
name='auto_tagging',
field=models.BooleanField(default=True, help_text='Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions', verbose_name='Auto-tagging'),
),
]
+7
View File
@@ -484,6 +484,13 @@ class UserSettings(models.Model):
verbose_name=_("Combine transfers"),
help_text=_("Show both sides of a transfer as a single item"),
)
auto_tagging = models.BooleanField(
default=True,
verbose_name=_("Auto-tagging"),
help_text=_(
"Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions"
),
)
volume = models.PositiveIntegerField(
default=10,
validators=[MinValueValidator(0), MaxValueValidator(10)],