diff --git a/app/apps/rules/tasks.py b/app/apps/rules/tasks.py index ba4573c1..47cfa3ca 100644 --- a/app/apps/rules/tasks.py +++ b/app/apps/rules/tasks.py @@ -23,6 +23,7 @@ from apps.rules.models import ( TransactionRule, TransactionRuleAction, ) +from apps.transactions.auto_tagging.service import apply_auto_tagging from apps.transactions.models import ( SubTransaction, Transaction, @@ -849,6 +850,16 @@ def check_for_transaction_rules( error="Initial trigger didn't match, this rule will be skipped", ) _log("Initial trigger didn't match, this rule will be skipped") + + if not dry_run and signal == "transaction_created": + try: + apply_auto_tagging(instance, user) + except Exception: + logger.error( + "Error while applying auto-tagging for transaction %s", + instance.id, + exc_info=True, + ) except Exception as e: _log( "** Error while executing 'check_for_transaction_rules' task", diff --git a/app/apps/transactions/admin.py b/app/apps/transactions/admin.py index 9fbedb8d..8d1bc56c 100644 --- a/app/apps/transactions/admin.py +++ b/app/apps/transactions/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin from apps.transactions.models import ( + AutoTaggerModel, Transaction, TransactionCategory, TransactionTag, @@ -79,3 +80,10 @@ class TransactionTagModelAdmin(SharedObjectModelAdmin): @admin.register(TransactionEntity) class TransactionEntityModelAdmin(SharedObjectModelAdmin): pass + + +@admin.register(AutoTaggerModel) +class AutoTaggerModelAdmin(admin.ModelAdmin): + list_display = ["user", "transaction_count", "version", "trained_at"] + readonly_fields = ["trained_at", "transaction_count", "version"] + exclude = ["data"] diff --git a/app/apps/transactions/auto_tagging/__init__.py b/app/apps/transactions/auto_tagging/__init__.py new file mode 100644 index 00000000..1f85e000 --- /dev/null +++ b/app/apps/transactions/auto_tagging/__init__.py @@ -0,0 +1,21 @@ +from apps.transactions.auto_tagging.classifier import ( + MODEL_VERSION, + Prediction, + TrainingRecord, + TransactionAutoTagger, +) +from apps.transactions.auto_tagging.service import ( + apply_auto_tagging, + ensure_model_fresh_async, + train_for_user, +) + +__all__ = [ + "MODEL_VERSION", + "Prediction", + "TrainingRecord", + "TransactionAutoTagger", + "apply_auto_tagging", + "ensure_model_fresh_async", + "train_for_user", +] diff --git a/app/apps/transactions/auto_tagging/classifier.py b/app/apps/transactions/auto_tagging/classifier.py new file mode 100644 index 00000000..b77f2c68 --- /dev/null +++ b/app/apps/transactions/auto_tagging/classifier.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import heapq +import json +import math +import re +import unicodedata +import zlib +from collections import defaultdict +from dataclasses import dataclass, field +from decimal import Decimal + +MODEL_VERSION = 1 + +MAX_TRAINING_DOCS = 5000 +TOP_K = 10 +MIN_SIMILARITY = 0.35 +NEAR_EXACT_SIMILARITY = 0.85 +MIN_SUPPORT_NEIGHBORS = 2 +CATEGORY_CONFIDENCE_THRESHOLD = 0.65 +MULTI_LABEL_CONFIDENCE_THRESHOLD = 0.60 + +# Strip dates (2025-01-20, 20/01/2025, 20/01) and installment counters (01/12) +_DATE_OR_INSTALLMENT_RE = re.compile( + r"\b\d{1,4}[-/.]\d{1,2}(?:[-/.]\d{1,4})?\b" +) +_WORD_RE = re.compile(r"[a-z0-9]+") + + +@dataclass(frozen=True) +class TrainingRecord: + description: str + notes: str + account_id: int + tx_type: str + amount: Decimal | float | int + category_id: int | None + tag_ids: tuple[int, ...] = () + entity_ids: tuple[int, ...] = () + + +@dataclass(frozen=True) +class Prediction: + category_id: int | None = None + tag_ids: tuple[int, ...] = () + entity_ids: tuple[int, ...] = () + + @property + def is_empty(self) -> bool: + return ( + self.category_id is None + and not self.tag_ids + and not self.entity_ids + ) + + +def _normalize_text(text: str) -> str: + if not text: + return "" + decomposed = unicodedata.normalize("NFKD", text) + stripped = "".join( + ch for ch in decomposed if unicodedata.category(ch) != "Mn" + ) + lowered = stripped.lower() + return _DATE_OR_INSTALLMENT_RE.sub(" ", lowered) + + +def _tokenize_text(text: str) -> list[str]: + normalized = _normalize_text(text) + if not normalized: + return [] + return [ + tok + for tok in _WORD_RE.findall(normalized) + if len(tok) >= 2 and not tok.isdigit() + ] + + +def _add_text_features( + features: dict[str, float], + tokens: list[str], + *, + unigram_weight: float, + bigram_weight: float, + char_ngram_weight: float, +) -> None: + for tok in tokens: + features[tok] = features.get(tok, 0.0) + unigram_weight + if char_ngram_weight > 0.0 and len(tok) >= 6 and tok.isalpha(): + for i in range(len(tok) - 3): + ngram = f"c:{tok[i : i + 4]}" + features[ngram] = features.get(ngram, 0.0) + char_ngram_weight + + for i in range(len(tokens) - 1): + bigram = f"b:{tokens[i]}_{tokens[i + 1]}" + features[bigram] = features.get(bigram, 0.0) + bigram_weight + + +def _amount_bucket(amount: Decimal | float | int | None) -> int: + if amount is None: + return 0 + val = abs(float(amount)) + if val < 1.0: + return 0 + return int(math.floor(math.log10(val))) + 1 + + +def _extract_feature_parts( + description: str, + notes: str, + account_id: int, + tx_type: str, + amount: Decimal | float | int | None, +) -> tuple[dict[str, float], set[str], bool]: + """Extract combined features plus the set of tokens from description. + + Returns ``(features, desc_feature_keys, has_description)``. + """ + desc_tokens = _tokenize_text(description) + notes_tokens = _tokenize_text(notes) + if not desc_tokens and not notes_tokens: + return {}, set(), False + + desc_features: dict[str, float] = {} + if desc_tokens: + _add_text_features( + desc_features, + desc_tokens, + unigram_weight=2.0, + bigram_weight=1.5, + char_ngram_weight=0.6, + ) + + features = dict(desc_features) + if notes_tokens: + _add_text_features( + features, + notes_tokens, + unigram_weight=1.0, + bigram_weight=0.75, + char_ngram_weight=0.0, + ) + + features[f"__type_{tx_type}"] = 0.5 + features[f"__acct_{account_id}"] = 0.6 + features[f"__amt_{_amount_bucket(amount)}"] = 0.4 + return features, set(desc_features.keys()), bool(desc_tokens) + + +def extract_features( + description: str, + notes: str, + account_id: int, + tx_type: str, + amount: Decimal | float | int | None, +) -> dict[str, float]: + """Extract weighted text + structured features for a transaction. + + Returns an empty dict when neither description nor notes yields any + meaningful text tokens, preventing blind matches on account/type alone. + """ + features, _, _ = _extract_feature_parts( + description=description, + notes=notes, + account_id=account_id, + tx_type=tx_type, + amount=amount, + ) + return features + + +@dataclass +class TransactionAutoTagger: + """Zero-dependency TF-IDF + Cosine Similarity k-NN classifier.""" + + idf: dict[str, float] = field(default_factory=dict) + # Flat posting lists: [doc_idx_0, weight_0, doc_idx_1, weight_1, ...] + inverted_index: dict[str, list[float | int]] = field(default_factory=dict) + doc_categories: list[int | None] = field(default_factory=list) + doc_tags: list[list[int]] = field(default_factory=list) + doc_entities: list[list[int]] = field(default_factory=list) + doc_counts: list[int] = field(default_factory=list) + total_records: int = 0 + version: int = MODEL_VERSION + + @property + def num_docs(self) -> int: + return self.total_records or len(self.doc_categories) + + @classmethod + def fit(cls, records: list[TrainingRecord]) -> TransactionAutoTagger: + # Collapse identical (features, labels) into weighted prototypes + prototypes: dict[tuple, int] = {} + doc_features: list[dict[str, float]] = [] + doc_categories: list[int | None] = [] + doc_tags: list[list[int]] = [] + doc_entities: list[list[int]] = [] + doc_counts: list[int] = [] + df: dict[str, int] = defaultdict(int) + total_records = 0 + + for rec in records[:MAX_TRAINING_DOCS]: + feats = extract_features( + description=rec.description, + notes=rec.notes, + account_id=rec.account_id, + tx_type=rec.tx_type, + amount=rec.amount, + ) + if not feats: + continue + total_records += 1 + for tok in feats: + df[tok] += 1 + + sorted_tags = tuple(sorted(rec.tag_ids)) + sorted_entities = tuple(sorted(rec.entity_ids)) + sig = ( + tuple(sorted(feats.items())), + rec.category_id, + sorted_tags, + sorted_entities, + ) + existing_idx = prototypes.get(sig) + if existing_idx is not None: + doc_counts[existing_idx] += 1 + else: + prototypes[sig] = len(doc_features) + doc_features.append(feats) + doc_categories.append(rec.category_id) + doc_tags.append(list(sorted_tags)) + doc_entities.append(list(sorted_entities)) + doc_counts.append(1) + + if total_records == 0: + return cls() + + idf = { + tok: round(math.log((1.0 + total_records) / (1.0 + count)) + 1.0, 4) + for tok, count in df.items() + } + + inverted_index: dict[str, list[float | int]] = defaultdict(list) + for doc_idx, feats in enumerate(doc_features): + weights = { + tok: (1.0 + math.log(tf)) * idf[tok] + for tok, tf in feats.items() + } + norm = math.sqrt(sum(w * w for w in weights.values())) + if norm <= 0.0: + continue + for tok, w in weights.items(): + postings = inverted_index[tok] + postings.append(doc_idx) + postings.append(round(w / norm, 4)) + + return cls( + idf=idf, + inverted_index=dict(inverted_index), + doc_categories=doc_categories, + doc_tags=doc_tags, + doc_entities=doc_entities, + doc_counts=doc_counts, + total_records=total_records, + version=MODEL_VERSION, + ) + + def to_bytes(self) -> bytes: + payload = { + "v": self.version, + "n": self.total_records, + "idf": self.idf, + "idx": self.inverted_index, + "cat": self.doc_categories, + "tag": self.doc_tags, + "ent": self.doc_entities, + "cnt": self.doc_counts, + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return zlib.compress(raw, level=6) + + @classmethod + def from_bytes(cls, blob: bytes | memoryview) -> TransactionAutoTagger | None: + if not blob: + return None + try: + payload = json.loads(zlib.decompress(bytes(blob)).decode("utf-8")) + except (zlib.error, json.JSONDecodeError, UnicodeDecodeError): + return None + if payload.get("v") != MODEL_VERSION or "cnt" not in payload: + return None + return cls( + idf=payload["idf"], + inverted_index=payload["idx"], + doc_categories=payload["cat"], + doc_tags=payload["tag"], + doc_entities=payload["ent"], + doc_counts=payload["cnt"], + total_records=payload.get("n", len(payload["cat"])), + version=payload["v"], + ) + + def predict( + self, + *, + description: str, + notes: str, + account_id: int, + tx_type: str, + amount: Decimal | float | int | None, + allowed_category_ids: set[int] | None = None, + allowed_tag_ids: set[int] | None = None, + allowed_entity_ids: set[int] | None = None, + ) -> Prediction: + if self.num_docs == 0: + return Prediction() + + feats, desc_feature_keys, has_description = _extract_feature_parts( + description=description, + notes=notes, + account_id=account_id, + tx_type=tx_type, + amount=amount, + ) + if not feats: + return Prediction() + + query_weights: dict[str, float] = {} + matched_text_tokens = 0 + unknown_idf = math.log(1.0 + self.num_docs) + 1.0 + norm_sq = 0.0 + + for tok, tf in feats.items(): + tok_idf = self.idf.get(tok) + if tok_idf is not None: + w = (1.0 + math.log(tf)) * tok_idf + query_weights[tok] = w + norm_sq += w * w + if not tok.startswith("__"): + matched_text_tokens += 1 + elif not tok.startswith("__"): + # Penalize unknown description tokens so unrelated descriptions + # abstain, but don't let unique notes (e.g. itemized details) + # discard a matching description. + if not has_description or tok in desc_feature_keys: + w = (1.0 + math.log(tf)) * unknown_idf + norm_sq += w * w + + if matched_text_tokens == 0 or norm_sq <= 0.0: + return Prediction() + + norm = math.sqrt(norm_sq) + + scores: dict[int, float] = defaultdict(float) + for tok, w in query_weights.items(): + q_w = w / norm + postings = self.inverted_index.get(tok) + if postings: + for i in range(0, len(postings), 2): + scores[postings[i]] += q_w * postings[i + 1] + + candidates = [ + (sim, doc_idx) + for doc_idx, sim in scores.items() + if sim >= MIN_SIMILARITY + ] + if not candidates: + return Prediction() + + top_neighbors = heapq.nlargest(TOP_K, candidates, key=lambda x: x[0]) + total_support = sum(self.doc_counts[doc_idx] for _, doc_idx in top_neighbors) + if total_support < MIN_SUPPORT_NEIGHBORS: + if top_neighbors[0][0] < NEAR_EXACT_SIMILARITY: + return Prediction() + + total_weight = 0.0 + cat_weights: dict[int, float] = defaultdict(float) + cat_present_weight = 0.0 + tag_weights: dict[int, float] = defaultdict(float) + entity_weights: dict[int, float] = defaultdict(float) + + for sim, doc_idx in top_neighbors: + count = self.doc_counts[doc_idx] + w = (sim * sim) * (1.0 + math.log(count)) + total_weight += w + + cat_id = self.doc_categories[doc_idx] + if cat_id is not None: + cat_present_weight += w + cat_weights[cat_id] += w + + for tid in self.doc_tags[doc_idx]: + tag_weights[tid] += w + + for eid in self.doc_entities[doc_idx]: + entity_weights[eid] += w + + predicted_category: int | None = None + if cat_weights and (cat_present_weight / total_weight) >= 0.5: + best_cat, best_w = max(cat_weights.items(), key=lambda item: item[1]) + if (best_w / cat_present_weight) >= CATEGORY_CONFIDENCE_THRESHOLD: + if allowed_category_ids is None or best_cat in allowed_category_ids: + predicted_category = best_cat + + predicted_tags = tuple( + sorted( + tid + for tid, w in tag_weights.items() + if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD + and (allowed_tag_ids is None or tid in allowed_tag_ids) + ) + ) + + predicted_entities = tuple( + sorted( + eid + for eid, w in entity_weights.items() + if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD + and (allowed_entity_ids is None or eid in allowed_entity_ids) + ) + ) + + return Prediction( + category_id=predicted_category, + tag_ids=predicted_tags, + entity_ids=predicted_entities, + ) diff --git a/app/apps/transactions/auto_tagging/service.py b/app/apps/transactions/auto_tagging/service.py new file mode 100644 index 00000000..d98f84af --- /dev/null +++ b/app/apps/transactions/auto_tagging/service.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import logging +from collections import OrderedDict, defaultdict +from datetime import timedelta + +from django.db.models import Q +from django.utils import timezone + +from apps.common.middleware.thread_local import ( + delete_current_user, + get_current_user, + write_current_user, +) +from apps.transactions.auto_tagging.classifier import ( + MAX_TRAINING_DOCS, + MODEL_VERSION, + Prediction, + TrainingRecord, + TransactionAutoTagger, +) +from apps.transactions.models import ( + AutoTaggerModel, + Transaction, + TransactionCategory, + TransactionEntity, + TransactionTag, +) + +logger = logging.getLogger(__name__) + +STALE_MODEL_AGE = timedelta(hours=1) +_TAGGER_CACHE_MAXSIZE = 4 +_tagger_cache: OrderedDict[tuple[int, str, int], TransactionAutoTagger] = OrderedDict() + + +def _get_cached_tagger(model_obj: AutoTaggerModel) -> TransactionAutoTagger | None: + cache_key = (model_obj.user_id, model_obj.trained_at.isoformat(), model_obj.version) + cached = _tagger_cache.get(cache_key) + if cached is not None: + _tagger_cache.move_to_end(cache_key) + return cached + + tagger = TransactionAutoTagger.from_bytes(model_obj.data) + if tagger is not None: + _tagger_cache[cache_key] = tagger + while len(_tagger_cache) > _TAGGER_CACHE_MAXSIZE: + _tagger_cache.popitem(last=False) + return tagger + + +def _with_user_context(user): + """Context manager that temporarily sets the thread-local current user.""" + + class _UserContext: + def __init__(self, target_user): + self.target_user = target_user + self.previous_user = None + + def __enter__(self): + self.previous_user = get_current_user() + write_current_user(self.target_user) + return self.target_user + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.previous_user is not None: + write_current_user(self.previous_user) + elif get_current_user() is not None: + delete_current_user() + + return _UserContext(user) + + +def train_for_user(user) -> AutoTaggerModel: + """Train and persist the auto-tagging model for a single user.""" + with _with_user_context(user): + qs = ( + Transaction.objects.filter(linked_transaction__isnull=True) + .exclude(Q(description="") & Q(notes="")) + .filter( + Q(category__isnull=False) + | Q(tags__isnull=False) + | Q(entities__isnull=False) + ) + .distinct() + .order_by("-date", "-id") + .values_list( + "id", + "description", + "notes", + "account_id", + "type", + "amount", + "category_id", + )[:MAX_TRAINING_DOCS] + ) + rows = list(qs) + + tx_ids = [row[0] for row in rows] + tags_by_tx: dict[int, list[int]] = defaultdict(list) + entities_by_tx: dict[int, list[int]] = defaultdict(list) + + if tx_ids: + for tx_id, tag_id in Transaction.tags.through.objects.filter( + transaction_id__in=tx_ids + ).values_list("transaction_id", "transactiontag_id"): + tags_by_tx[tx_id].append(tag_id) + + for tx_id, entity_id in Transaction.entities.through.objects.filter( + transaction_id__in=tx_ids + ).values_list("transaction_id", "transactionentity_id"): + entities_by_tx[tx_id].append(entity_id) + + records = [ + TrainingRecord( + description=description or "", + notes=notes or "", + account_id=account_id, + tx_type=tx_type, + amount=amount, + category_id=category_id, + tag_ids=tuple(tags_by_tx.get(tx_id, ())), + entity_ids=tuple(entities_by_tx.get(tx_id, ())), + ) + for tx_id, description, notes, account_id, tx_type, amount, category_id in rows + ] + + tagger = TransactionAutoTagger.fit(records) + model_obj, _ = AutoTaggerModel.objects.update_or_create( + user=user, + defaults={ + "data": tagger.to_bytes(), + "transaction_count": tagger.num_docs, + "version": MODEL_VERSION, + }, + ) + return model_obj + + +def ensure_model_fresh_async(user_id: int) -> None: + """Defer a background retrain if Procrastinate is available.""" + from apps.transactions.tasks import train_user_auto_tagger + + try: + train_user_auto_tagger.configure( + queueing_lock=f"train_auto_tagger_{user_id}" + ).defer(user_id=user_id) + except Exception: + logger.debug( + "Could not defer auto-tagger training for user %s", + user_id, + exc_info=True, + ) + + +def _load_or_train_tagger(user) -> TransactionAutoTagger | None: + model_obj = AutoTaggerModel.objects.filter(user=user).first() + if model_obj is None or model_obj.version != MODEL_VERSION: + model_obj = train_for_user(user) + elif timezone.now() - model_obj.trained_at > STALE_MODEL_AGE: + ensure_model_fresh_async(user.id) + + return _get_cached_tagger(model_obj) + + +def apply_auto_tagging(instance: Transaction, user) -> bool: + """Auto-assign category, tags, and entities to an unclassified transaction. + + Only runs when: + - The user has auto_tagging enabled in UserSettings + - The transaction is not deleted and not part of a transfer + - The transaction currently has NO category, NO tags, and NO entities + """ + user_settings = getattr(user, "settings", None) + if user_settings is None or not user_settings.auto_tagging: + return False + + if instance.deleted or instance.linked_transaction_id is not None: + return False + + if ( + instance.category_id is not None + or Transaction.tags.through.objects.filter(transaction_id=instance.pk).exists() + or Transaction.entities.through.objects.filter( + transaction_id=instance.pk + ).exists() + ): + return False + + with _with_user_context(user): + tagger = _load_or_train_tagger(user) + if tagger is None or tagger.num_docs == 0: + return False + + allowed_category_ids = set( + TransactionCategory.objects.filter( + active=True, auto_match=True + ).values_list("id", flat=True) + ) + allowed_tag_ids = set( + TransactionTag.objects.filter( + active=True, auto_match=True + ).values_list("id", flat=True) + ) + allowed_entity_ids = set( + TransactionEntity.objects.filter( + active=True, auto_match=True + ).values_list("id", flat=True) + ) + + prediction: Prediction = tagger.predict( + description=instance.description, + notes=instance.notes, + account_id=instance.account_id, + tx_type=instance.type, + amount=instance.amount, + allowed_category_ids=allowed_category_ids, + allowed_tag_ids=allowed_tag_ids, + allowed_entity_ids=allowed_entity_ids, + ) + + if prediction.is_empty: + return False + + if prediction.category_id is not None: + Transaction.all_objects.filter(pk=instance.pk).update( + emit_signal=False, + category_id=prediction.category_id, + ) + instance.category_id = prediction.category_id + + if prediction.tag_ids: + instance.tags.add(*prediction.tag_ids) + + if prediction.entity_ids: + instance.entities.add(*prediction.entity_ids) + + return True diff --git a/app/apps/transactions/forms.py b/app/apps/transactions/forms.py index 24ebe568..fd7e70f7 100644 --- a/app/apps/transactions/forms.py +++ b/app/apps/transactions/forms.py @@ -1091,7 +1091,7 @@ class InstallmentPlanForm(SubTransactionsFormMixin, forms.ModelForm): class TransactionTagForm(forms.ModelForm): class Meta: model = TransactionTag - fields = ["name", "active"] + fields = ["name", "active", "auto_match"] labels = {"name": _("Tag name")} def __init__(self, *args, **kwargs): @@ -1100,7 +1100,11 @@ class TransactionTagForm(forms.ModelForm): self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" - self.helper.layout = Layout(Field("name", css_class="mb-3"), Switch("active")) + self.helper.layout = Layout( + Field("name", css_class="mb-3"), + Switch("active"), + Switch("auto_match"), + ) if self.instance and self.instance.pk: self.helper.layout.append( @@ -1119,7 +1123,7 @@ class TransactionTagForm(forms.ModelForm): class TransactionEntityForm(forms.ModelForm): class Meta: model = TransactionEntity - fields = ["name", "active"] + fields = ["name", "active", "auto_match"] labels = {"name": _("Entity name")} def __init__(self, *args, **kwargs): @@ -1128,7 +1132,11 @@ class TransactionEntityForm(forms.ModelForm): self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" - self.helper.layout = Layout(Field("name"), Switch("active")) + self.helper.layout = Layout( + Field("name"), + Switch("active"), + Switch("auto_match"), + ) if self.instance and self.instance.pk: self.helper.layout.append( @@ -1147,7 +1155,7 @@ class TransactionEntityForm(forms.ModelForm): class TransactionCategoryForm(forms.ModelForm): class Meta: model = TransactionCategory - fields = ["name", "mute", "active"] + fields = ["name", "mute", "active", "auto_match"] labels = {"name": _("Category name")} help_texts = { "mute": _("Muted categories won't be displayed on monthly summaries") @@ -1159,7 +1167,12 @@ class TransactionCategoryForm(forms.ModelForm): self.helper = FormHelper() self.helper.form_tag = False self.helper.form_method = "post" - self.helper.layout = Layout(Field("name"), Switch("mute"), Switch("active")) + self.helper.layout = Layout( + Field("name"), + Switch("mute"), + Switch("active"), + Switch("auto_match"), + ) if self.instance and self.instance.pk: self.helper.layout.append( diff --git a/app/apps/transactions/migrations/0055_transactioncategory_auto_match_and_more.py b/app/apps/transactions/migrations/0055_transactioncategory_auto_match_and_more.py new file mode 100644 index 00000000..fe37c620 --- /dev/null +++ b/app/apps/transactions/migrations/0055_transactioncategory_auto_match_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 5.2.17 on 2026-11-19 21:22 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0054_filterpreset_relative_to'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='transactioncategory', + name='auto_match', + field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this category based on past transactions', verbose_name='Auto-match'), + ), + migrations.AddField( + model_name='transactionentity', + name='auto_match', + field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this entity based on past transactions', verbose_name='Auto-match'), + ), + migrations.AddField( + model_name='transactiontag', + name='auto_match', + field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this tag based on past transactions', verbose_name='Auto-match'), + ), + migrations.CreateModel( + name='AutoTaggerModel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', models.BinaryField(default=b'')), + ('transaction_count', models.PositiveIntegerField(default=0)), + ('version', models.PositiveSmallIntegerField(default=1)), + ('trained_at', models.DateTimeField(auto_now=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='auto_tagger_model', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Auto-Tagger Model', + 'verbose_name_plural': 'Auto-Tagger Models', + 'db_table': 'auto_tagger_models', + }, + ), + ] diff --git a/app/apps/transactions/models.py b/app/apps/transactions/models.py index 8d6c9282..02ee982a 100644 --- a/app/apps/transactions/models.py +++ b/app/apps/transactions/models.py @@ -261,6 +261,13 @@ class TransactionCategory(SharedObject): "Deactivated categories won't be able to be selected when creating new transactions" ), ) + auto_match = models.BooleanField( + default=True, + verbose_name=_("Auto-match"), + help_text=_( + "Allow auto-tagging to automatically assign this category based on past transactions" + ), + ) objects = SharedObjectManager() all_objects = models.Manager() # Unfiltered manager @@ -285,6 +292,13 @@ class TransactionTag(SharedObject): "Deactivated tags won't be able to be selected when creating new transactions" ), ) + auto_match = models.BooleanField( + default=True, + verbose_name=_("Auto-match"), + help_text=_( + "Allow auto-tagging to automatically assign this tag based on past transactions" + ), + ) objects = SharedObjectManager() all_objects = models.Manager() # Unfiltered manager @@ -309,6 +323,13 @@ class TransactionEntity(SharedObject): "Deactivated entities won't be able to be selected when creating new transactions" ), ) + auto_match = models.BooleanField( + default=True, + verbose_name=_("Auto-match"), + help_text=_( + "Allow auto-tagging to automatically assign this entity based on past transactions" + ), + ) objects = SharedObjectManager() all_objects = models.Manager() # Unfiltered manager @@ -1567,3 +1588,23 @@ class QuickTransaction(TransferTemplate, OwnedObject): def __str__(self): return self.name + + +class AutoTaggerModel(models.Model): + user = models.OneToOneField( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="auto_tagger_model", + ) + data = models.BinaryField(default=b"") + transaction_count = models.PositiveIntegerField(default=0) + version = models.PositiveSmallIntegerField(default=1) + trained_at = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = _("Auto-Tagger Model") + verbose_name_plural = _("Auto-Tagger Models") + db_table = "auto_tagger_models" + + def __str__(self): + return f"AutoTaggerModel({self.user_id}, n={self.transaction_count})" diff --git a/app/apps/transactions/tasks.py b/app/apps/transactions/tasks.py index 271d261a..c65160b8 100644 --- a/app/apps/transactions/tasks.py +++ b/app/apps/transactions/tasks.py @@ -2,12 +2,15 @@ import logging from datetime import timedelta from cachalot.api import cachalot_disabled, invalidate +from django.contrib.auth import get_user_model +from django.db.models import Max from django.utils import timezone from django.conf import settings from procrastinate.contrib.django import app -from apps.transactions.models import RecurringTransaction, Transaction +from apps.transactions.auto_tagging.service import _with_user_context, train_for_user +from apps.transactions.models import AutoTaggerModel, RecurringTransaction, Transaction logger = logging.getLogger(__name__) @@ -50,3 +53,48 @@ def cleanup_deleted_transactions(timestamp=None): deleted_count, _ = old_transactions.hard_delete() return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days." + + +@app.task(name="train_user_auto_tagger") +def train_user_auto_tagger(user_id: int): + user = get_user_model().objects.filter(id=user_id).first() + if user is None: + return "User not found." + + user_settings = getattr(user, "settings", None) + if user_settings is None or not user_settings.auto_tagging: + return "Auto-tagging disabled for user." + + with cachalot_disabled(): + model_obj = train_for_user(user) + return f"Trained auto-tagger for user {user_id} on {model_obj.transaction_count} transactions." + + +@app.periodic(cron="30 2 * * *") +@app.task(lock="retrain_auto_taggers", name="retrain_auto_taggers") +def retrain_auto_taggers(timestamp=None): + User = get_user_model() + users = User.objects.filter(settings__auto_tagging=True, is_active=True).select_related( + "settings" + ) + existing_models = { + m.user_id: m.trained_at + for m in AutoTaggerModel.objects.filter(user__in=users) + } + + retrained = 0 + with cachalot_disabled(): + for user in users: + trained_at = existing_models.get(user.id) + with _with_user_context(user): + latest_tx = Transaction.objects.aggregate(latest=Max("updated_at"))[ + "latest" + ] + if trained_at is not None and ( + latest_tx is None or latest_tx <= trained_at + ): + continue + train_for_user(user) + retrained += 1 + + return f"Retrained auto-tagger for {retrained} user(s)." diff --git a/app/apps/transactions/tests/test_auto_tagging.py b/app/apps/transactions/tests/test_auto_tagging.py new file mode 100644 index 00000000..3a955a96 --- /dev/null +++ b/app/apps/transactions/tests/test_auto_tagging.py @@ -0,0 +1,474 @@ +import datetime +from decimal import Decimal +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase + +from apps.accounts.models import Account, AccountGroup +from apps.common.middleware.thread_local import ( + delete_current_user, + get_current_user, + write_current_user, +) +from apps.currencies.models import Currency +from apps.rules.models import TransactionRule, TransactionRuleAction +from apps.rules.tasks import check_for_transaction_rules +from apps.transactions.auto_tagging.classifier import ( + TrainingRecord, + TransactionAutoTagger, +) +from apps.transactions.auto_tagging.service import apply_auto_tagging, train_for_user +from apps.transactions.forms import ( + TransactionCategoryForm, + TransactionEntityForm, + TransactionTagForm, +) +from apps.transactions.models import ( + AutoTaggerModel, + Transaction, + TransactionCategory, + TransactionEntity, + TransactionTag, +) +from apps.transactions.tasks import retrain_auto_taggers, train_user_auto_tagger +from apps.users.forms import UserSettingsForm + +User = get_user_model() + + +class TransactionAutoTaggerClassifierTests(TestCase): + def _sample_records(self) -> list[TrainingRecord]: + return [ + TrainingRecord( + description="UBER *TRIP HELP.UBER.COM 12/01", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("24.50"), + category_id=10, + tag_ids=(100,), + entity_ids=(200,), + ), + TrainingRecord( + description="UBER *TRIP 15/01/2025", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("18.90"), + category_id=10, + tag_ids=(100,), + entity_ids=(200,), + ), + TrainingRecord( + description="UBER *TRIP 2025-01-20", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("31.00"), + category_id=10, + tag_ids=(100,), + entity_ids=(200,), + ), + TrainingRecord( + description="PIX PADARIA SÃO JOÃO", + notes="breakfast", + account_id=1, + tx_type="EX", + amount=Decimal("15.00"), + category_id=20, + tag_ids=(101, 102), + entity_ids=(201,), + ), + TrainingRecord( + description="PIX PADARIA SAO JOAO", + notes="coffee", + account_id=1, + tx_type="EX", + amount=Decimal("12.00"), + category_id=20, + tag_ids=(101, 102), + entity_ids=(201,), + ), + ] + + def test_predicts_category_tags_and_entities_with_noise_and_accents(self): + tagger = TransactionAutoTagger.fit(self._sample_records()) + + uber_pred = tagger.predict( + description="UBER *TRIP 03/02/2025", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("22.00"), + ) + self.assertEqual(uber_pred.category_id, 10) + self.assertEqual(uber_pred.tag_ids, (100,)) + self.assertEqual(uber_pred.entity_ids, (200,)) + + bakery_pred = tagger.predict( + description="PIX PADARIA SAO JOAO 04/02", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("14.00"), + ) + self.assertEqual(bakery_pred.category_id, 20) + self.assertEqual(bakery_pred.tag_ids, (101, 102)) + self.assertEqual(bakery_pred.entity_ids, (201,)) + + def test_serialization_roundtrip_and_invalid_blob(self): + tagger = TransactionAutoTagger.fit(self._sample_records()) + blob = tagger.to_bytes() + restored = TransactionAutoTagger.from_bytes(blob) + + self.assertIsNotNone(restored) + self.assertEqual(restored.num_docs, tagger.num_docs) + pred = restored.predict( + description="UBER *TRIP", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("20.00"), + ) + self.assertEqual(pred.category_id, 10) + + self.assertIsNone(TransactionAutoTagger.from_bytes(b"")) + self.assertIsNone(TransactionAutoTagger.from_bytes(b"not-valid-zlib")) + + def test_abstains_on_empty_text_or_unrelated_description(self): + tagger = TransactionAutoTagger.fit(self._sample_records()) + + empty_pred = tagger.predict( + description="", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("20.00"), + ) + self.assertTrue(empty_pred.is_empty) + + unrelated_pred = tagger.predict( + description="COMPLETELY UNKNOWN MERCHANT XYZ", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("20.00"), + ) + self.assertTrue(unrelated_pred.is_empty) + + # Sharing only 1 weak token among mostly unknown words must still abstain + weak_overlap_pred = tagger.predict( + description="COMPLETELY RANDOM TRIP THING", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("20.00"), + ) + self.assertTrue(weak_overlap_pred.is_empty) + + def test_unique_notes_do_not_discard_matching_description(self): + tagger = TransactionAutoTagger.fit(self._sample_records()) + + pred = tagger.predict( + description="UBER *TRIP 05/02", + notes="airport pickup for visiting cousins with heavy suitcases", + account_id=1, + tx_type="EX", + amount=Decimal("25.00"), + ) + self.assertEqual(pred.category_id, 10) + self.assertEqual(pred.tag_ids, (100,)) + self.assertEqual(pred.entity_ids, (200,)) + + def test_respects_allowed_label_sets(self): + tagger = TransactionAutoTagger.fit(self._sample_records()) + pred = tagger.predict( + description="UBER *TRIP", + notes="", + account_id=1, + tx_type="EX", + amount=Decimal("20.00"), + allowed_category_ids=set(), + allowed_tag_ids={100}, + allowed_entity_ids=set(), + ) + self.assertIsNone(pred.category_id) + self.assertEqual(pred.tag_ids, (100,)) + self.assertEqual(pred.entity_ids, ()) + + +class AutoTaggingServiceAndRulesTests(TestCase): + def setUp(self): + self.user = User.objects.create_user( + email="alice@example.com", password="password123" + ) + self.other_user = User.objects.create_user( + email="bob@example.com", password="password123" + ) + + self.currency = Currency.objects.create( + code="USD", name="US Dollar", decimal_places=2, prefix="$ " + ) + self.group = AccountGroup.objects.create(name="Checking", owner=self.user) + self.account = Account.objects.create( + name="Main Card", + group=self.group, + currency=self.currency, + owner=self.user, + visibility="private", + ) + self.second_account = Account.objects.create( + name="Savings", + group=self.group, + currency=self.currency, + owner=self.user, + visibility="private", + ) + + self.cat_transport = TransactionCategory.objects.create( + name="Transport", owner=self.user, visibility="private" + ) + self.cat_other = TransactionCategory.objects.create( + name="Other", owner=self.user, visibility="private" + ) + self.tag_commute = TransactionTag.objects.create( + name="Commute", owner=self.user, visibility="private" + ) + self.tag_rule = TransactionTag.objects.create( + name="RuleApplied", owner=self.user, visibility="private" + ) + self.entity_uber = TransactionEntity.objects.create( + name="Uber", owner=self.user, visibility="private" + ) + + # Seed historical labeled transactions for Alice + for day in (1, 5, 10): + tx = Transaction.objects.create( + owner=self.user, + account=self.account, + type=Transaction.Type.EXPENSE, + date=datetime.date(2025, 1, day), + amount=Decimal("25.00"), + description=f"UBER *TRIP 0{day}/01", + category=self.cat_transport, + ) + tx.tags.add(self.tag_commute) + tx.entities.add(self.entity_uber) + + def tearDown(self): + if get_current_user() is not None: + delete_current_user() + + def _tag_ids(self, tx: Transaction) -> list[int]: + return list( + Transaction.tags.through.objects.filter(transaction_id=tx.pk).values_list( + "transactiontag_id", flat=True + ) + ) + + def _entity_ids(self, tx: Transaction) -> list[int]: + return list( + Transaction.entities.through.objects.filter( + transaction_id=tx.pk + ).values_list("transactionentity_id", flat=True) + ) + + def _create_unclassified_uber_tx(self, **overrides) -> Transaction: + defaults = { + "owner": self.user, + "account": self.account, + "type": Transaction.Type.EXPENSE, + "date": datetime.date(2025, 2, 1), + "amount": Decimal("27.00"), + "description": "UBER *TRIP 01/02", + } + defaults.update(overrides) + return Transaction.objects.create(**defaults) + + def test_auto_tags_unclassified_transaction_after_rules(self): + tx = self._create_unclassified_uber_tx() + + check_for_transaction_rules( + instance_id=tx.id, + user_id=self.user.id, + signal="transaction_created", + ) + + tx.refresh_from_db() + self.assertEqual(tx.category_id, self.cat_transport.id) + self.assertEqual(self._tag_ids(tx), [self.tag_commute.id]) + self.assertEqual(self._entity_ids(tx), [self.entity_uber.id]) + + def test_skips_when_user_already_set_category_tag_or_entity(self): + train_for_user(self.user) + + tx_with_cat = self._create_unclassified_uber_tx(category=self.cat_other) + self.assertFalse(apply_auto_tagging(tx_with_cat, self.user)) + tx_with_cat.refresh_from_db() + self.assertEqual(tx_with_cat.category_id, self.cat_other.id) + self.assertEqual(self._tag_ids(tx_with_cat), []) + self.assertEqual(self._entity_ids(tx_with_cat), []) + + tx_with_tag = self._create_unclassified_uber_tx() + tx_with_tag.tags.add(self.tag_rule) + self.assertFalse(apply_auto_tagging(tx_with_tag, self.user)) + tx_with_tag.refresh_from_db() + self.assertIsNone(tx_with_tag.category_id) + self.assertEqual(self._entity_ids(tx_with_tag), []) + + tx_with_entity = self._create_unclassified_uber_tx() + tx_with_entity.entities.add(self.entity_uber) + self.assertFalse(apply_auto_tagging(tx_with_entity, self.user)) + tx_with_entity.refresh_from_db() + self.assertIsNone(tx_with_entity.category_id) + self.assertEqual(self._tag_ids(tx_with_entity), []) + + def test_rules_run_before_auto_tagging_and_prevent_auto_tagging_if_rule_classifies(self): + rule = TransactionRule.objects.create( + owner=self.user, + visibility="private", + name="Tag Uber via Rule", + trigger="'UBER' in description", + on_create=True, + active=True, + ) + TransactionRuleAction.objects.create( + rule=rule, + field=TransactionRuleAction.Field.tags, + value=f"['{self.tag_rule.name}']", + ) + + tx = self._create_unclassified_uber_tx() + check_for_transaction_rules( + instance_id=tx.id, + user_id=self.user.id, + signal="transaction_created", + ) + + tx.refresh_from_db() + # Rule set the tag; auto-tagging should see the transaction is already classified and skip it + self.assertEqual(self._tag_ids(tx), [self.tag_rule.id]) + self.assertIsNone(tx.category_id) + self.assertEqual(self._entity_ids(tx), []) + + def test_user_settings_disable_auto_tagging(self): + self.user.settings.auto_tagging = False + self.user.settings.save() + + tx = self._create_unclassified_uber_tx() + check_for_transaction_rules( + instance_id=tx.id, + user_id=self.user.id, + signal="transaction_created", + ) + + tx.refresh_from_db() + self.assertIsNone(tx.category_id) + self.assertEqual(self._tag_ids(tx), []) + self.assertEqual(self._entity_ids(tx), []) + + def test_per_item_auto_match_opt_out_and_inactive_items(self): + train_for_user(self.user) + + self.cat_transport.auto_match = False + self.cat_transport.save() + self.tag_commute.active = False + self.tag_commute.save() + + tx = self._create_unclassified_uber_tx() + applied = apply_auto_tagging(tx, self.user) + self.assertTrue(applied) + + tx.refresh_from_db() + self.assertIsNone(tx.category_id) + self.assertEqual(self._tag_ids(tx), []) + self.assertEqual(self._entity_ids(tx), [self.entity_uber.id]) + + def test_skips_transfers(self): + train_for_user(self.user) + + tx_from = self._create_unclassified_uber_tx() + tx_from.add_transfer_side(self.second_account, Decimal("27.00")) + tx_from.refresh_from_db() + + self.assertFalse(apply_auto_tagging(tx_from, self.user)) + self.assertIsNone(tx_from.category_id) + + def test_multi_user_isolation(self): + train_for_user(self.user) + + bob_group = AccountGroup.objects.create(name="Bob Group", owner=self.other_user) + bob_account = Account.objects.create( + name="Bob Card", + group=bob_group, + currency=self.currency, + owner=self.other_user, + visibility="private", + ) + bob_tx = Transaction.objects.create( + owner=self.other_user, + account=bob_account, + type=Transaction.Type.EXPENSE, + date=datetime.date(2025, 2, 1), + amount=Decimal("25.00"), + description="UBER *TRIP 01/02", + ) + + self.assertFalse(apply_auto_tagging(bob_tx, self.other_user)) + bob_tx.refresh_from_db() + self.assertIsNone(bob_tx.category_id) + self.assertFalse(bob_tx.tags.exists()) + self.assertFalse(bob_tx.entities.exists()) + + def test_procrastinate_training_tasks(self): + msg = train_user_auto_tagger(self.user.id) + self.assertIn("Trained auto-tagger", msg) + model_obj = AutoTaggerModel.objects.get(user=self.user) + self.assertEqual(model_obj.transaction_count, 3) + + # Periodic task skips already-up-to-date user + with patch( + "apps.transactions.tasks.train_for_user", wraps=train_for_user + ) as mock_train: + retrain_auto_taggers() + trained_user_ids = [call.args[0].id for call in mock_train.call_args_list] + self.assertNotIn(self.user.id, trained_user_ids) + + def test_retrain_auto_taggers_detects_shared_account_updates(self): + # Train Bob initially (0 transactions visible) + train_for_user(self.other_user) + self.assertEqual( + AutoTaggerModel.objects.get(user=self.other_user).transaction_count, 0 + ) + + # Share Alice's account with Bob and add a new transaction owned by Alice + self.account.shared_with.add(self.other_user) + Transaction.objects.create( + owner=self.user, + account=self.account, + type=Transaction.Type.EXPENSE, + date=datetime.date(2025, 2, 2), + amount=Decimal("30.00"), + description="UBER *TRIP SHARED", + category=self.cat_transport, + ) + + retrain_auto_taggers() + + bob_model = AutoTaggerModel.objects.get(user=self.other_user) + self.assertGreater(bob_model.transaction_count, 0) + + def test_forms_expose_auto_tagging_and_auto_match_fields(self): + write_current_user(self.user) + settings_form = UserSettingsForm(instance=self.user.settings) + self.assertIn("auto_tagging", settings_form.fields) + + cat_form = TransactionCategoryForm(instance=self.cat_transport) + self.assertIn("auto_match", cat_form.fields) + + tag_form = TransactionTagForm(instance=self.tag_commute) + self.assertIn("auto_match", tag_form.fields) + + entity_form = TransactionEntityForm(instance=self.entity_uber) + self.assertIn("auto_match", entity_form.fields) diff --git a/app/apps/users/forms.py b/app/apps/users/forms.py index 46639a74..e2e61bd3 100644 --- a/app/apps/users/forms.py +++ b/app/apps/users/forms.py @@ -158,6 +158,7 @@ class UserSettingsForm(forms.ModelForm): "number_format", "default_account", "combine_transfers", + "auto_tagging", "volume", ] widgets = { @@ -185,6 +186,7 @@ class UserSettingsForm(forms.ModelForm): "start_page", "default_account", Switch("combine_transfers"), + Switch("auto_tagging"), HTML('
'), Range("volume"), FormActions( diff --git a/app/apps/users/migrations/0030_usersettings_auto_tagging.py b/app/apps/users/migrations/0030_usersettings_auto_tagging.py new file mode 100644 index 00000000..94389381 --- /dev/null +++ b/app/apps/users/migrations/0030_usersettings_auto_tagging.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-11-19 21:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0029_usersettings_combine_transfers'), + ] + + operations = [ + migrations.AddField( + model_name='usersettings', + name='auto_tagging', + field=models.BooleanField(default=True, help_text='Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions', verbose_name='Auto-tagging'), + ), + ] diff --git a/app/apps/users/models.py b/app/apps/users/models.py index 62594cc1..4f5fd815 100644 --- a/app/apps/users/models.py +++ b/app/apps/users/models.py @@ -484,6 +484,13 @@ class UserSettings(models.Model): verbose_name=_("Combine transfers"), help_text=_("Show both sides of a transfer as a single item"), ) + auto_tagging = models.BooleanField( + default=True, + verbose_name=_("Auto-tagging"), + help_text=_( + "Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions" + ), + ) volume = models.PositiveIntegerField( default=10, validators=[MinValueValidator(0), MaxValueValidator(10)],