mirror of
https://github.com/eitchtee/WYGIWYH.git
synced 2026-11-20 07:31:44 +01:00
Merge remote-tracking branch 'origin/main'
This commit is contained in:
commit
dddd757bb5
13 files changed
+1362
-7
No files matched your search
@@ -23,6 +23,7 @@ from apps.rules.models import (
|
||||
TransactionRule,
|
||||
TransactionRuleAction,
|
||||
)
|
||||
from apps.transactions.auto_tagging.service import apply_auto_tagging
|
||||
from apps.transactions.models import (
|
||||
SubTransaction,
|
||||
Transaction,
|
||||
@@ -849,6 +850,16 @@ def check_for_transaction_rules(
|
||||
error="Initial trigger didn't match, this rule will be skipped",
|
||||
)
|
||||
_log("Initial trigger didn't match, this rule will be skipped")
|
||||
|
||||
if not dry_run and signal == "transaction_created":
|
||||
try:
|
||||
apply_auto_tagging(instance, user)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Error while applying auto-tagging for transaction %s",
|
||||
instance.id,
|
||||
exc_info=True,
|
||||
)
|
||||
except Exception as e:
|
||||
_log(
|
||||
"** Error while executing 'check_for_transaction_rules' task",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from apps.transactions.models import (
|
||||
AutoTaggerModel,
|
||||
Transaction,
|
||||
TransactionCategory,
|
||||
TransactionTag,
|
||||
@@ -79,3 +80,10 @@ class TransactionTagModelAdmin(SharedObjectModelAdmin):
|
||||
@admin.register(TransactionEntity)
|
||||
class TransactionEntityModelAdmin(SharedObjectModelAdmin):
|
||||
pass
|
||||
|
||||
|
||||
@admin.register(AutoTaggerModel)
|
||||
class AutoTaggerModelAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "transaction_count", "version", "trained_at"]
|
||||
readonly_fields = ["trained_at", "transaction_count", "version"]
|
||||
exclude = ["data"]
|
||||
@@ -0,0 +1,21 @@
|
||||
from apps.transactions.auto_tagging.classifier import (
|
||||
MODEL_VERSION,
|
||||
Prediction,
|
||||
TrainingRecord,
|
||||
TransactionAutoTagger,
|
||||
)
|
||||
from apps.transactions.auto_tagging.service import (
|
||||
apply_auto_tagging,
|
||||
ensure_model_fresh_async,
|
||||
train_for_user,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MODEL_VERSION",
|
||||
"Prediction",
|
||||
"TrainingRecord",
|
||||
"TransactionAutoTagger",
|
||||
"apply_auto_tagging",
|
||||
"ensure_model_fresh_async",
|
||||
"train_for_user",
|
||||
]
|
||||
@@ -0,0 +1,427 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import heapq
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import unicodedata
|
||||
import zlib
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
|
||||
MODEL_VERSION = 1
|
||||
|
||||
MAX_TRAINING_DOCS = 5000
|
||||
TOP_K = 10
|
||||
MIN_SIMILARITY = 0.35
|
||||
NEAR_EXACT_SIMILARITY = 0.85
|
||||
MIN_SUPPORT_NEIGHBORS = 2
|
||||
CATEGORY_CONFIDENCE_THRESHOLD = 0.65
|
||||
MULTI_LABEL_CONFIDENCE_THRESHOLD = 0.60
|
||||
|
||||
# Strip dates (2025-01-20, 20/01/2025, 20/01) and installment counters (01/12)
|
||||
_DATE_OR_INSTALLMENT_RE = re.compile(
|
||||
r"\b\d{1,4}[-/.]\d{1,2}(?:[-/.]\d{1,4})?\b"
|
||||
)
|
||||
_WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrainingRecord:
|
||||
description: str
|
||||
notes: str
|
||||
account_id: int
|
||||
tx_type: str
|
||||
amount: Decimal | float | int
|
||||
category_id: int | None
|
||||
tag_ids: tuple[int, ...] = ()
|
||||
entity_ids: tuple[int, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Prediction:
|
||||
category_id: int | None = None
|
||||
tag_ids: tuple[int, ...] = ()
|
||||
entity_ids: tuple[int, ...] = ()
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
self.category_id is None
|
||||
and not self.tag_ids
|
||||
and not self.entity_ids
|
||||
)
|
||||
|
||||
|
||||
def _normalize_text(text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
decomposed = unicodedata.normalize("NFKD", text)
|
||||
stripped = "".join(
|
||||
ch for ch in decomposed if unicodedata.category(ch) != "Mn"
|
||||
)
|
||||
lowered = stripped.lower()
|
||||
return _DATE_OR_INSTALLMENT_RE.sub(" ", lowered)
|
||||
|
||||
|
||||
def _tokenize_text(text: str) -> list[str]:
|
||||
normalized = _normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
return [
|
||||
tok
|
||||
for tok in _WORD_RE.findall(normalized)
|
||||
if len(tok) >= 2 and not tok.isdigit()
|
||||
]
|
||||
|
||||
|
||||
def _add_text_features(
|
||||
features: dict[str, float],
|
||||
tokens: list[str],
|
||||
*,
|
||||
unigram_weight: float,
|
||||
bigram_weight: float,
|
||||
char_ngram_weight: float,
|
||||
) -> None:
|
||||
for tok in tokens:
|
||||
features[tok] = features.get(tok, 0.0) + unigram_weight
|
||||
if char_ngram_weight > 0.0 and len(tok) >= 6 and tok.isalpha():
|
||||
for i in range(len(tok) - 3):
|
||||
ngram = f"c:{tok[i : i + 4]}"
|
||||
features[ngram] = features.get(ngram, 0.0) + char_ngram_weight
|
||||
|
||||
for i in range(len(tokens) - 1):
|
||||
bigram = f"b:{tokens[i]}_{tokens[i + 1]}"
|
||||
features[bigram] = features.get(bigram, 0.0) + bigram_weight
|
||||
|
||||
|
||||
def _amount_bucket(amount: Decimal | float | int | None) -> int:
|
||||
if amount is None:
|
||||
return 0
|
||||
val = abs(float(amount))
|
||||
if val < 1.0:
|
||||
return 0
|
||||
return int(math.floor(math.log10(val))) + 1
|
||||
|
||||
|
||||
def _extract_feature_parts(
|
||||
description: str,
|
||||
notes: str,
|
||||
account_id: int,
|
||||
tx_type: str,
|
||||
amount: Decimal | float | int | None,
|
||||
) -> tuple[dict[str, float], set[str], bool]:
|
||||
"""Extract combined features plus the set of tokens from description.
|
||||
|
||||
Returns ``(features, desc_feature_keys, has_description)``.
|
||||
"""
|
||||
desc_tokens = _tokenize_text(description)
|
||||
notes_tokens = _tokenize_text(notes)
|
||||
if not desc_tokens and not notes_tokens:
|
||||
return {}, set(), False
|
||||
|
||||
desc_features: dict[str, float] = {}
|
||||
if desc_tokens:
|
||||
_add_text_features(
|
||||
desc_features,
|
||||
desc_tokens,
|
||||
unigram_weight=2.0,
|
||||
bigram_weight=1.5,
|
||||
char_ngram_weight=0.6,
|
||||
)
|
||||
|
||||
features = dict(desc_features)
|
||||
if notes_tokens:
|
||||
_add_text_features(
|
||||
features,
|
||||
notes_tokens,
|
||||
unigram_weight=1.0,
|
||||
bigram_weight=0.75,
|
||||
char_ngram_weight=0.0,
|
||||
)
|
||||
|
||||
features[f"__type_{tx_type}"] = 0.5
|
||||
features[f"__acct_{account_id}"] = 0.6
|
||||
features[f"__amt_{_amount_bucket(amount)}"] = 0.4
|
||||
return features, set(desc_features.keys()), bool(desc_tokens)
|
||||
|
||||
|
||||
def extract_features(
|
||||
description: str,
|
||||
notes: str,
|
||||
account_id: int,
|
||||
tx_type: str,
|
||||
amount: Decimal | float | int | None,
|
||||
) -> dict[str, float]:
|
||||
"""Extract weighted text + structured features for a transaction.
|
||||
|
||||
Returns an empty dict when neither description nor notes yields any
|
||||
meaningful text tokens, preventing blind matches on account/type alone.
|
||||
"""
|
||||
features, _, _ = _extract_feature_parts(
|
||||
description=description,
|
||||
notes=notes,
|
||||
account_id=account_id,
|
||||
tx_type=tx_type,
|
||||
amount=amount,
|
||||
)
|
||||
return features
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransactionAutoTagger:
|
||||
"""Zero-dependency TF-IDF + Cosine Similarity k-NN classifier."""
|
||||
|
||||
idf: dict[str, float] = field(default_factory=dict)
|
||||
# Flat posting lists: [doc_idx_0, weight_0, doc_idx_1, weight_1, ...]
|
||||
inverted_index: dict[str, list[float | int]] = field(default_factory=dict)
|
||||
doc_categories: list[int | None] = field(default_factory=list)
|
||||
doc_tags: list[list[int]] = field(default_factory=list)
|
||||
doc_entities: list[list[int]] = field(default_factory=list)
|
||||
doc_counts: list[int] = field(default_factory=list)
|
||||
total_records: int = 0
|
||||
version: int = MODEL_VERSION
|
||||
|
||||
@property
|
||||
def num_docs(self) -> int:
|
||||
return self.total_records or len(self.doc_categories)
|
||||
|
||||
@classmethod
|
||||
def fit(cls, records: list[TrainingRecord]) -> TransactionAutoTagger:
|
||||
# Collapse identical (features, labels) into weighted prototypes
|
||||
prototypes: dict[tuple, int] = {}
|
||||
doc_features: list[dict[str, float]] = []
|
||||
doc_categories: list[int | None] = []
|
||||
doc_tags: list[list[int]] = []
|
||||
doc_entities: list[list[int]] = []
|
||||
doc_counts: list[int] = []
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
total_records = 0
|
||||
|
||||
for rec in records[:MAX_TRAINING_DOCS]:
|
||||
feats = extract_features(
|
||||
description=rec.description,
|
||||
notes=rec.notes,
|
||||
account_id=rec.account_id,
|
||||
tx_type=rec.tx_type,
|
||||
amount=rec.amount,
|
||||
)
|
||||
if not feats:
|
||||
continue
|
||||
total_records += 1
|
||||
for tok in feats:
|
||||
df[tok] += 1
|
||||
|
||||
sorted_tags = tuple(sorted(rec.tag_ids))
|
||||
sorted_entities = tuple(sorted(rec.entity_ids))
|
||||
sig = (
|
||||
tuple(sorted(feats.items())),
|
||||
rec.category_id,
|
||||
sorted_tags,
|
||||
sorted_entities,
|
||||
)
|
||||
existing_idx = prototypes.get(sig)
|
||||
if existing_idx is not None:
|
||||
doc_counts[existing_idx] += 1
|
||||
else:
|
||||
prototypes[sig] = len(doc_features)
|
||||
doc_features.append(feats)
|
||||
doc_categories.append(rec.category_id)
|
||||
doc_tags.append(list(sorted_tags))
|
||||
doc_entities.append(list(sorted_entities))
|
||||
doc_counts.append(1)
|
||||
|
||||
if total_records == 0:
|
||||
return cls()
|
||||
|
||||
idf = {
|
||||
tok: round(math.log((1.0 + total_records) / (1.0 + count)) + 1.0, 4)
|
||||
for tok, count in df.items()
|
||||
}
|
||||
|
||||
inverted_index: dict[str, list[float | int]] = defaultdict(list)
|
||||
for doc_idx, feats in enumerate(doc_features):
|
||||
weights = {
|
||||
tok: (1.0 + math.log(tf)) * idf[tok]
|
||||
for tok, tf in feats.items()
|
||||
}
|
||||
norm = math.sqrt(sum(w * w for w in weights.values()))
|
||||
if norm <= 0.0:
|
||||
continue
|
||||
for tok, w in weights.items():
|
||||
postings = inverted_index[tok]
|
||||
postings.append(doc_idx)
|
||||
postings.append(round(w / norm, 4))
|
||||
|
||||
return cls(
|
||||
idf=idf,
|
||||
inverted_index=dict(inverted_index),
|
||||
doc_categories=doc_categories,
|
||||
doc_tags=doc_tags,
|
||||
doc_entities=doc_entities,
|
||||
doc_counts=doc_counts,
|
||||
total_records=total_records,
|
||||
version=MODEL_VERSION,
|
||||
)
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
payload = {
|
||||
"v": self.version,
|
||||
"n": self.total_records,
|
||||
"idf": self.idf,
|
||||
"idx": self.inverted_index,
|
||||
"cat": self.doc_categories,
|
||||
"tag": self.doc_tags,
|
||||
"ent": self.doc_entities,
|
||||
"cnt": self.doc_counts,
|
||||
}
|
||||
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
return zlib.compress(raw, level=6)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, blob: bytes | memoryview) -> TransactionAutoTagger | None:
|
||||
if not blob:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(zlib.decompress(bytes(blob)).decode("utf-8"))
|
||||
except (zlib.error, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return None
|
||||
if payload.get("v") != MODEL_VERSION or "cnt" not in payload:
|
||||
return None
|
||||
return cls(
|
||||
idf=payload["idf"],
|
||||
inverted_index=payload["idx"],
|
||||
doc_categories=payload["cat"],
|
||||
doc_tags=payload["tag"],
|
||||
doc_entities=payload["ent"],
|
||||
doc_counts=payload["cnt"],
|
||||
total_records=payload.get("n", len(payload["cat"])),
|
||||
version=payload["v"],
|
||||
)
|
||||
|
||||
def predict(
|
||||
self,
|
||||
*,
|
||||
description: str,
|
||||
notes: str,
|
||||
account_id: int,
|
||||
tx_type: str,
|
||||
amount: Decimal | float | int | None,
|
||||
allowed_category_ids: set[int] | None = None,
|
||||
allowed_tag_ids: set[int] | None = None,
|
||||
allowed_entity_ids: set[int] | None = None,
|
||||
) -> Prediction:
|
||||
if self.num_docs == 0:
|
||||
return Prediction()
|
||||
|
||||
feats, desc_feature_keys, has_description = _extract_feature_parts(
|
||||
description=description,
|
||||
notes=notes,
|
||||
account_id=account_id,
|
||||
tx_type=tx_type,
|
||||
amount=amount,
|
||||
)
|
||||
if not feats:
|
||||
return Prediction()
|
||||
|
||||
query_weights: dict[str, float] = {}
|
||||
matched_text_tokens = 0
|
||||
unknown_idf = math.log(1.0 + self.num_docs) + 1.0
|
||||
norm_sq = 0.0
|
||||
|
||||
for tok, tf in feats.items():
|
||||
tok_idf = self.idf.get(tok)
|
||||
if tok_idf is not None:
|
||||
w = (1.0 + math.log(tf)) * tok_idf
|
||||
query_weights[tok] = w
|
||||
norm_sq += w * w
|
||||
if not tok.startswith("__"):
|
||||
matched_text_tokens += 1
|
||||
elif not tok.startswith("__"):
|
||||
# Penalize unknown description tokens so unrelated descriptions
|
||||
# abstain, but don't let unique notes (e.g. itemized details)
|
||||
# discard a matching description.
|
||||
if not has_description or tok in desc_feature_keys:
|
||||
w = (1.0 + math.log(tf)) * unknown_idf
|
||||
norm_sq += w * w
|
||||
|
||||
if matched_text_tokens == 0 or norm_sq <= 0.0:
|
||||
return Prediction()
|
||||
|
||||
norm = math.sqrt(norm_sq)
|
||||
|
||||
scores: dict[int, float] = defaultdict(float)
|
||||
for tok, w in query_weights.items():
|
||||
q_w = w / norm
|
||||
postings = self.inverted_index.get(tok)
|
||||
if postings:
|
||||
for i in range(0, len(postings), 2):
|
||||
scores[postings[i]] += q_w * postings[i + 1]
|
||||
|
||||
candidates = [
|
||||
(sim, doc_idx)
|
||||
for doc_idx, sim in scores.items()
|
||||
if sim >= MIN_SIMILARITY
|
||||
]
|
||||
if not candidates:
|
||||
return Prediction()
|
||||
|
||||
top_neighbors = heapq.nlargest(TOP_K, candidates, key=lambda x: x[0])
|
||||
total_support = sum(self.doc_counts[doc_idx] for _, doc_idx in top_neighbors)
|
||||
if total_support < MIN_SUPPORT_NEIGHBORS:
|
||||
if top_neighbors[0][0] < NEAR_EXACT_SIMILARITY:
|
||||
return Prediction()
|
||||
|
||||
total_weight = 0.0
|
||||
cat_weights: dict[int, float] = defaultdict(float)
|
||||
cat_present_weight = 0.0
|
||||
tag_weights: dict[int, float] = defaultdict(float)
|
||||
entity_weights: dict[int, float] = defaultdict(float)
|
||||
|
||||
for sim, doc_idx in top_neighbors:
|
||||
count = self.doc_counts[doc_idx]
|
||||
w = (sim * sim) * (1.0 + math.log(count))
|
||||
total_weight += w
|
||||
|
||||
cat_id = self.doc_categories[doc_idx]
|
||||
if cat_id is not None:
|
||||
cat_present_weight += w
|
||||
cat_weights[cat_id] += w
|
||||
|
||||
for tid in self.doc_tags[doc_idx]:
|
||||
tag_weights[tid] += w
|
||||
|
||||
for eid in self.doc_entities[doc_idx]:
|
||||
entity_weights[eid] += w
|
||||
|
||||
predicted_category: int | None = None
|
||||
if cat_weights and (cat_present_weight / total_weight) >= 0.5:
|
||||
best_cat, best_w = max(cat_weights.items(), key=lambda item: item[1])
|
||||
if (best_w / cat_present_weight) >= CATEGORY_CONFIDENCE_THRESHOLD:
|
||||
if allowed_category_ids is None or best_cat in allowed_category_ids:
|
||||
predicted_category = best_cat
|
||||
|
||||
predicted_tags = tuple(
|
||||
sorted(
|
||||
tid
|
||||
for tid, w in tag_weights.items()
|
||||
if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD
|
||||
and (allowed_tag_ids is None or tid in allowed_tag_ids)
|
||||
)
|
||||
)
|
||||
|
||||
predicted_entities = tuple(
|
||||
sorted(
|
||||
eid
|
||||
for eid, w in entity_weights.items()
|
||||
if (w / total_weight) >= MULTI_LABEL_CONFIDENCE_THRESHOLD
|
||||
and (allowed_entity_ids is None or eid in allowed_entity_ids)
|
||||
)
|
||||
)
|
||||
|
||||
return Prediction(
|
||||
category_id=predicted_category,
|
||||
tag_ids=predicted_tags,
|
||||
entity_ids=predicted_entities,
|
||||
)
|
||||
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import OrderedDict, defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.common.middleware.thread_local import (
|
||||
delete_current_user,
|
||||
get_current_user,
|
||||
write_current_user,
|
||||
)
|
||||
from apps.transactions.auto_tagging.classifier import (
|
||||
MAX_TRAINING_DOCS,
|
||||
MODEL_VERSION,
|
||||
Prediction,
|
||||
TrainingRecord,
|
||||
TransactionAutoTagger,
|
||||
)
|
||||
from apps.transactions.models import (
|
||||
AutoTaggerModel,
|
||||
Transaction,
|
||||
TransactionCategory,
|
||||
TransactionEntity,
|
||||
TransactionTag,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STALE_MODEL_AGE = timedelta(hours=1)
|
||||
_TAGGER_CACHE_MAXSIZE = 4
|
||||
_tagger_cache: OrderedDict[tuple[int, str, int], TransactionAutoTagger] = OrderedDict()
|
||||
|
||||
|
||||
def _get_cached_tagger(model_obj: AutoTaggerModel) -> TransactionAutoTagger | None:
|
||||
cache_key = (model_obj.user_id, model_obj.trained_at.isoformat(), model_obj.version)
|
||||
cached = _tagger_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
_tagger_cache.move_to_end(cache_key)
|
||||
return cached
|
||||
|
||||
tagger = TransactionAutoTagger.from_bytes(model_obj.data)
|
||||
if tagger is not None:
|
||||
_tagger_cache[cache_key] = tagger
|
||||
while len(_tagger_cache) > _TAGGER_CACHE_MAXSIZE:
|
||||
_tagger_cache.popitem(last=False)
|
||||
return tagger
|
||||
|
||||
|
||||
def _with_user_context(user):
|
||||
"""Context manager that temporarily sets the thread-local current user."""
|
||||
|
||||
class _UserContext:
|
||||
def __init__(self, target_user):
|
||||
self.target_user = target_user
|
||||
self.previous_user = None
|
||||
|
||||
def __enter__(self):
|
||||
self.previous_user = get_current_user()
|
||||
write_current_user(self.target_user)
|
||||
return self.target_user
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.previous_user is not None:
|
||||
write_current_user(self.previous_user)
|
||||
elif get_current_user() is not None:
|
||||
delete_current_user()
|
||||
|
||||
return _UserContext(user)
|
||||
|
||||
|
||||
def train_for_user(user) -> AutoTaggerModel:
|
||||
"""Train and persist the auto-tagging model for a single user."""
|
||||
with _with_user_context(user):
|
||||
qs = (
|
||||
Transaction.objects.filter(linked_transaction__isnull=True)
|
||||
.exclude(Q(description="") & Q(notes=""))
|
||||
.filter(
|
||||
Q(category__isnull=False)
|
||||
| Q(tags__isnull=False)
|
||||
| Q(entities__isnull=False)
|
||||
)
|
||||
.distinct()
|
||||
.order_by("-date", "-id")
|
||||
.values_list(
|
||||
"id",
|
||||
"description",
|
||||
"notes",
|
||||
"account_id",
|
||||
"type",
|
||||
"amount",
|
||||
"category_id",
|
||||
)[:MAX_TRAINING_DOCS]
|
||||
)
|
||||
rows = list(qs)
|
||||
|
||||
tx_ids = [row[0] for row in rows]
|
||||
tags_by_tx: dict[int, list[int]] = defaultdict(list)
|
||||
entities_by_tx: dict[int, list[int]] = defaultdict(list)
|
||||
|
||||
if tx_ids:
|
||||
for tx_id, tag_id in Transaction.tags.through.objects.filter(
|
||||
transaction_id__in=tx_ids
|
||||
).values_list("transaction_id", "transactiontag_id"):
|
||||
tags_by_tx[tx_id].append(tag_id)
|
||||
|
||||
for tx_id, entity_id in Transaction.entities.through.objects.filter(
|
||||
transaction_id__in=tx_ids
|
||||
).values_list("transaction_id", "transactionentity_id"):
|
||||
entities_by_tx[tx_id].append(entity_id)
|
||||
|
||||
records = [
|
||||
TrainingRecord(
|
||||
description=description or "",
|
||||
notes=notes or "",
|
||||
account_id=account_id,
|
||||
tx_type=tx_type,
|
||||
amount=amount,
|
||||
category_id=category_id,
|
||||
tag_ids=tuple(tags_by_tx.get(tx_id, ())),
|
||||
entity_ids=tuple(entities_by_tx.get(tx_id, ())),
|
||||
)
|
||||
for tx_id, description, notes, account_id, tx_type, amount, category_id in rows
|
||||
]
|
||||
|
||||
tagger = TransactionAutoTagger.fit(records)
|
||||
model_obj, _ = AutoTaggerModel.objects.update_or_create(
|
||||
user=user,
|
||||
defaults={
|
||||
"data": tagger.to_bytes(),
|
||||
"transaction_count": tagger.num_docs,
|
||||
"version": MODEL_VERSION,
|
||||
},
|
||||
)
|
||||
return model_obj
|
||||
|
||||
|
||||
def ensure_model_fresh_async(user_id: int) -> None:
|
||||
"""Defer a background retrain if Procrastinate is available."""
|
||||
from apps.transactions.tasks import train_user_auto_tagger
|
||||
|
||||
try:
|
||||
train_user_auto_tagger.configure(
|
||||
queueing_lock=f"train_auto_tagger_{user_id}"
|
||||
).defer(user_id=user_id)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not defer auto-tagger training for user %s",
|
||||
user_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
def _load_or_train_tagger(user) -> TransactionAutoTagger | None:
|
||||
model_obj = AutoTaggerModel.objects.filter(user=user).first()
|
||||
if model_obj is None or model_obj.version != MODEL_VERSION:
|
||||
model_obj = train_for_user(user)
|
||||
elif timezone.now() - model_obj.trained_at > STALE_MODEL_AGE:
|
||||
ensure_model_fresh_async(user.id)
|
||||
|
||||
return _get_cached_tagger(model_obj)
|
||||
|
||||
|
||||
def apply_auto_tagging(instance: Transaction, user) -> bool:
|
||||
"""Auto-assign category, tags, and entities to an unclassified transaction.
|
||||
|
||||
Only runs when:
|
||||
- The user has auto_tagging enabled in UserSettings
|
||||
- The transaction is not deleted and not part of a transfer
|
||||
- The transaction currently has NO category, NO tags, and NO entities
|
||||
"""
|
||||
user_settings = getattr(user, "settings", None)
|
||||
if user_settings is None or not user_settings.auto_tagging:
|
||||
return False
|
||||
|
||||
if instance.deleted or instance.linked_transaction_id is not None:
|
||||
return False
|
||||
|
||||
if (
|
||||
instance.category_id is not None
|
||||
or Transaction.tags.through.objects.filter(transaction_id=instance.pk).exists()
|
||||
or Transaction.entities.through.objects.filter(
|
||||
transaction_id=instance.pk
|
||||
).exists()
|
||||
):
|
||||
return False
|
||||
|
||||
with _with_user_context(user):
|
||||
tagger = _load_or_train_tagger(user)
|
||||
if tagger is None or tagger.num_docs == 0:
|
||||
return False
|
||||
|
||||
allowed_category_ids = set(
|
||||
TransactionCategory.objects.filter(
|
||||
active=True, auto_match=True
|
||||
).values_list("id", flat=True)
|
||||
)
|
||||
allowed_tag_ids = set(
|
||||
TransactionTag.objects.filter(
|
||||
active=True, auto_match=True
|
||||
).values_list("id", flat=True)
|
||||
)
|
||||
allowed_entity_ids = set(
|
||||
TransactionEntity.objects.filter(
|
||||
active=True, auto_match=True
|
||||
).values_list("id", flat=True)
|
||||
)
|
||||
|
||||
prediction: Prediction = tagger.predict(
|
||||
description=instance.description,
|
||||
notes=instance.notes,
|
||||
account_id=instance.account_id,
|
||||
tx_type=instance.type,
|
||||
amount=instance.amount,
|
||||
allowed_category_ids=allowed_category_ids,
|
||||
allowed_tag_ids=allowed_tag_ids,
|
||||
allowed_entity_ids=allowed_entity_ids,
|
||||
)
|
||||
|
||||
if prediction.is_empty:
|
||||
return False
|
||||
|
||||
if prediction.category_id is not None:
|
||||
Transaction.all_objects.filter(pk=instance.pk).update(
|
||||
emit_signal=False,
|
||||
category_id=prediction.category_id,
|
||||
)
|
||||
instance.category_id = prediction.category_id
|
||||
|
||||
if prediction.tag_ids:
|
||||
instance.tags.add(*prediction.tag_ids)
|
||||
|
||||
if prediction.entity_ids:
|
||||
instance.entities.add(*prediction.entity_ids)
|
||||
|
||||
return True
|
||||
@@ -1091,7 +1091,7 @@ class InstallmentPlanForm(SubTransactionsFormMixin, forms.ModelForm):
|
||||
class TransactionTagForm(forms.ModelForm):
|
||||
class Meta:
|
||||
model = TransactionTag
|
||||
fields = ["name", "active"]
|
||||
fields = ["name", "active", "auto_match"]
|
||||
labels = {"name": _("Tag name")}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
@@ -1100,7 +1100,11 @@ class TransactionTagForm(forms.ModelForm):
|
||||
self.helper = FormHelper()
|
||||
self.helper.form_tag = False
|
||||
self.helper.form_method = "post"
|
||||
self.helper.layout = Layout(Field("name", css_class="mb-3"), Switch("active"))
|
||||
self.helper.layout = Layout(
|
||||
Field("name", css_class="mb-3"),
|
||||
Switch("active"),
|
||||
Switch("auto_match"),
|
||||
)
|
||||
|
||||
if self.instance and self.instance.pk:
|
||||
self.helper.layout.append(
|
||||
@@ -1119,7 +1123,7 @@ class TransactionTagForm(forms.ModelForm):
|
||||
class TransactionEntityForm(forms.ModelForm):
|
||||
class Meta:
|
||||
model = TransactionEntity
|
||||
fields = ["name", "active"]
|
||||
fields = ["name", "active", "auto_match"]
|
||||
labels = {"name": _("Entity name")}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
@@ -1128,7 +1132,11 @@ class TransactionEntityForm(forms.ModelForm):
|
||||
self.helper = FormHelper()
|
||||
self.helper.form_tag = False
|
||||
self.helper.form_method = "post"
|
||||
self.helper.layout = Layout(Field("name"), Switch("active"))
|
||||
self.helper.layout = Layout(
|
||||
Field("name"),
|
||||
Switch("active"),
|
||||
Switch("auto_match"),
|
||||
)
|
||||
|
||||
if self.instance and self.instance.pk:
|
||||
self.helper.layout.append(
|
||||
@@ -1147,7 +1155,7 @@ class TransactionEntityForm(forms.ModelForm):
|
||||
class TransactionCategoryForm(forms.ModelForm):
|
||||
class Meta:
|
||||
model = TransactionCategory
|
||||
fields = ["name", "mute", "active"]
|
||||
fields = ["name", "mute", "active", "auto_match"]
|
||||
labels = {"name": _("Category name")}
|
||||
help_texts = {
|
||||
"mute": _("Muted categories won't be displayed on monthly summaries")
|
||||
@@ -1159,7 +1167,12 @@ class TransactionCategoryForm(forms.ModelForm):
|
||||
self.helper = FormHelper()
|
||||
self.helper.form_tag = False
|
||||
self.helper.form_method = "post"
|
||||
self.helper.layout = Layout(Field("name"), Switch("mute"), Switch("active"))
|
||||
self.helper.layout = Layout(
|
||||
Field("name"),
|
||||
Switch("mute"),
|
||||
Switch("active"),
|
||||
Switch("auto_match"),
|
||||
)
|
||||
|
||||
if self.instance and self.instance.pk:
|
||||
self.helper.layout.append(
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-19 21:22
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('transactions', '0054_filterpreset_relative_to'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='transactioncategory',
|
||||
name='auto_match',
|
||||
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this category based on past transactions', verbose_name='Auto-match'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='transactionentity',
|
||||
name='auto_match',
|
||||
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this entity based on past transactions', verbose_name='Auto-match'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='transactiontag',
|
||||
name='auto_match',
|
||||
field=models.BooleanField(default=True, help_text='Allow auto-tagging to automatically assign this tag based on past transactions', verbose_name='Auto-match'),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AutoTaggerModel',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('data', models.BinaryField(default=b'')),
|
||||
('transaction_count', models.PositiveIntegerField(default=0)),
|
||||
('version', models.PositiveSmallIntegerField(default=1)),
|
||||
('trained_at', models.DateTimeField(auto_now=True)),
|
||||
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='auto_tagger_model', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Auto-Tagger Model',
|
||||
'verbose_name_plural': 'Auto-Tagger Models',
|
||||
'db_table': 'auto_tagger_models',
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -261,6 +261,13 @@ class TransactionCategory(SharedObject):
|
||||
"Deactivated categories won't be able to be selected when creating new transactions"
|
||||
),
|
||||
)
|
||||
auto_match = models.BooleanField(
|
||||
default=True,
|
||||
verbose_name=_("Auto-match"),
|
||||
help_text=_(
|
||||
"Allow auto-tagging to automatically assign this category based on past transactions"
|
||||
),
|
||||
)
|
||||
|
||||
objects = SharedObjectManager()
|
||||
all_objects = models.Manager() # Unfiltered manager
|
||||
@@ -285,6 +292,13 @@ class TransactionTag(SharedObject):
|
||||
"Deactivated tags won't be able to be selected when creating new transactions"
|
||||
),
|
||||
)
|
||||
auto_match = models.BooleanField(
|
||||
default=True,
|
||||
verbose_name=_("Auto-match"),
|
||||
help_text=_(
|
||||
"Allow auto-tagging to automatically assign this tag based on past transactions"
|
||||
),
|
||||
)
|
||||
|
||||
objects = SharedObjectManager()
|
||||
all_objects = models.Manager() # Unfiltered manager
|
||||
@@ -309,6 +323,13 @@ class TransactionEntity(SharedObject):
|
||||
"Deactivated entities won't be able to be selected when creating new transactions"
|
||||
),
|
||||
)
|
||||
auto_match = models.BooleanField(
|
||||
default=True,
|
||||
verbose_name=_("Auto-match"),
|
||||
help_text=_(
|
||||
"Allow auto-tagging to automatically assign this entity based on past transactions"
|
||||
),
|
||||
)
|
||||
|
||||
objects = SharedObjectManager()
|
||||
all_objects = models.Manager() # Unfiltered manager
|
||||
@@ -1567,3 +1588,23 @@ class QuickTransaction(TransferTemplate, OwnedObject):
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class AutoTaggerModel(models.Model):
|
||||
user = models.OneToOneField(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="auto_tagger_model",
|
||||
)
|
||||
data = models.BinaryField(default=b"")
|
||||
transaction_count = models.PositiveIntegerField(default=0)
|
||||
version = models.PositiveSmallIntegerField(default=1)
|
||||
trained_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("Auto-Tagger Model")
|
||||
verbose_name_plural = _("Auto-Tagger Models")
|
||||
db_table = "auto_tagger_models"
|
||||
|
||||
def __str__(self):
|
||||
return f"AutoTaggerModel({self.user_id}, n={self.transaction_count})"
|
||||
@@ -2,12 +2,15 @@ import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from cachalot.api import cachalot_disabled, invalidate
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db.models import Max
|
||||
from django.utils import timezone
|
||||
from django.conf import settings
|
||||
|
||||
from procrastinate.contrib.django import app
|
||||
|
||||
from apps.transactions.models import RecurringTransaction, Transaction
|
||||
from apps.transactions.auto_tagging.service import _with_user_context, train_for_user
|
||||
from apps.transactions.models import AutoTaggerModel, RecurringTransaction, Transaction
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,3 +53,48 @@ def cleanup_deleted_transactions(timestamp=None):
|
||||
deleted_count, _ = old_transactions.hard_delete()
|
||||
|
||||
return f"Hard deleted {deleted_count} objects older than {settings.KEEP_DELETED_TRANSACTIONS_FOR} days."
|
||||
|
||||
|
||||
@app.task(name="train_user_auto_tagger")
|
||||
def train_user_auto_tagger(user_id: int):
|
||||
user = get_user_model().objects.filter(id=user_id).first()
|
||||
if user is None:
|
||||
return "User not found."
|
||||
|
||||
user_settings = getattr(user, "settings", None)
|
||||
if user_settings is None or not user_settings.auto_tagging:
|
||||
return "Auto-tagging disabled for user."
|
||||
|
||||
with cachalot_disabled():
|
||||
model_obj = train_for_user(user)
|
||||
return f"Trained auto-tagger for user {user_id} on {model_obj.transaction_count} transactions."
|
||||
|
||||
|
||||
@app.periodic(cron="30 2 * * *")
|
||||
@app.task(lock="retrain_auto_taggers", name="retrain_auto_taggers")
|
||||
def retrain_auto_taggers(timestamp=None):
|
||||
User = get_user_model()
|
||||
users = User.objects.filter(settings__auto_tagging=True, is_active=True).select_related(
|
||||
"settings"
|
||||
)
|
||||
existing_models = {
|
||||
m.user_id: m.trained_at
|
||||
for m in AutoTaggerModel.objects.filter(user__in=users)
|
||||
}
|
||||
|
||||
retrained = 0
|
||||
with cachalot_disabled():
|
||||
for user in users:
|
||||
trained_at = existing_models.get(user.id)
|
||||
with _with_user_context(user):
|
||||
latest_tx = Transaction.objects.aggregate(latest=Max("updated_at"))[
|
||||
"latest"
|
||||
]
|
||||
if trained_at is not None and (
|
||||
latest_tx is None or latest_tx <= trained_at
|
||||
):
|
||||
continue
|
||||
train_for_user(user)
|
||||
retrained += 1
|
||||
|
||||
return f"Retrained auto-tagger for {retrained} user(s)."
|
||||
@@ -0,0 +1,474 @@
|
||||
import datetime
|
||||
from decimal import Decimal
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
|
||||
from apps.accounts.models import Account, AccountGroup
|
||||
from apps.common.middleware.thread_local import (
|
||||
delete_current_user,
|
||||
get_current_user,
|
||||
write_current_user,
|
||||
)
|
||||
from apps.currencies.models import Currency
|
||||
from apps.rules.models import TransactionRule, TransactionRuleAction
|
||||
from apps.rules.tasks import check_for_transaction_rules
|
||||
from apps.transactions.auto_tagging.classifier import (
|
||||
TrainingRecord,
|
||||
TransactionAutoTagger,
|
||||
)
|
||||
from apps.transactions.auto_tagging.service import apply_auto_tagging, train_for_user
|
||||
from apps.transactions.forms import (
|
||||
TransactionCategoryForm,
|
||||
TransactionEntityForm,
|
||||
TransactionTagForm,
|
||||
)
|
||||
from apps.transactions.models import (
|
||||
AutoTaggerModel,
|
||||
Transaction,
|
||||
TransactionCategory,
|
||||
TransactionEntity,
|
||||
TransactionTag,
|
||||
)
|
||||
from apps.transactions.tasks import retrain_auto_taggers, train_user_auto_tagger
|
||||
from apps.users.forms import UserSettingsForm
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
class TransactionAutoTaggerClassifierTests(TestCase):
|
||||
def _sample_records(self) -> list[TrainingRecord]:
|
||||
return [
|
||||
TrainingRecord(
|
||||
description="UBER *TRIP HELP.UBER.COM 12/01",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("24.50"),
|
||||
category_id=10,
|
||||
tag_ids=(100,),
|
||||
entity_ids=(200,),
|
||||
),
|
||||
TrainingRecord(
|
||||
description="UBER *TRIP 15/01/2025",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("18.90"),
|
||||
category_id=10,
|
||||
tag_ids=(100,),
|
||||
entity_ids=(200,),
|
||||
),
|
||||
TrainingRecord(
|
||||
description="UBER *TRIP 2025-01-20",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("31.00"),
|
||||
category_id=10,
|
||||
tag_ids=(100,),
|
||||
entity_ids=(200,),
|
||||
),
|
||||
TrainingRecord(
|
||||
description="PIX PADARIA SÃO JOÃO",
|
||||
notes="breakfast",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("15.00"),
|
||||
category_id=20,
|
||||
tag_ids=(101, 102),
|
||||
entity_ids=(201,),
|
||||
),
|
||||
TrainingRecord(
|
||||
description="PIX PADARIA SAO JOAO",
|
||||
notes="coffee",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("12.00"),
|
||||
category_id=20,
|
||||
tag_ids=(101, 102),
|
||||
entity_ids=(201,),
|
||||
),
|
||||
]
|
||||
|
||||
def test_predicts_category_tags_and_entities_with_noise_and_accents(self):
|
||||
tagger = TransactionAutoTagger.fit(self._sample_records())
|
||||
|
||||
uber_pred = tagger.predict(
|
||||
description="UBER *TRIP 03/02/2025",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("22.00"),
|
||||
)
|
||||
self.assertEqual(uber_pred.category_id, 10)
|
||||
self.assertEqual(uber_pred.tag_ids, (100,))
|
||||
self.assertEqual(uber_pred.entity_ids, (200,))
|
||||
|
||||
bakery_pred = tagger.predict(
|
||||
description="PIX PADARIA SAO JOAO 04/02",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("14.00"),
|
||||
)
|
||||
self.assertEqual(bakery_pred.category_id, 20)
|
||||
self.assertEqual(bakery_pred.tag_ids, (101, 102))
|
||||
self.assertEqual(bakery_pred.entity_ids, (201,))
|
||||
|
||||
def test_serialization_roundtrip_and_invalid_blob(self):
|
||||
tagger = TransactionAutoTagger.fit(self._sample_records())
|
||||
blob = tagger.to_bytes()
|
||||
restored = TransactionAutoTagger.from_bytes(blob)
|
||||
|
||||
self.assertIsNotNone(restored)
|
||||
self.assertEqual(restored.num_docs, tagger.num_docs)
|
||||
pred = restored.predict(
|
||||
description="UBER *TRIP",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("20.00"),
|
||||
)
|
||||
self.assertEqual(pred.category_id, 10)
|
||||
|
||||
self.assertIsNone(TransactionAutoTagger.from_bytes(b""))
|
||||
self.assertIsNone(TransactionAutoTagger.from_bytes(b"not-valid-zlib"))
|
||||
|
||||
def test_abstains_on_empty_text_or_unrelated_description(self):
|
||||
tagger = TransactionAutoTagger.fit(self._sample_records())
|
||||
|
||||
empty_pred = tagger.predict(
|
||||
description="",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("20.00"),
|
||||
)
|
||||
self.assertTrue(empty_pred.is_empty)
|
||||
|
||||
unrelated_pred = tagger.predict(
|
||||
description="COMPLETELY UNKNOWN MERCHANT XYZ",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("20.00"),
|
||||
)
|
||||
self.assertTrue(unrelated_pred.is_empty)
|
||||
|
||||
# Sharing only 1 weak token among mostly unknown words must still abstain
|
||||
weak_overlap_pred = tagger.predict(
|
||||
description="COMPLETELY RANDOM TRIP THING",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("20.00"),
|
||||
)
|
||||
self.assertTrue(weak_overlap_pred.is_empty)
|
||||
|
||||
def test_unique_notes_do_not_discard_matching_description(self):
|
||||
tagger = TransactionAutoTagger.fit(self._sample_records())
|
||||
|
||||
pred = tagger.predict(
|
||||
description="UBER *TRIP 05/02",
|
||||
notes="airport pickup for visiting cousins with heavy suitcases",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("25.00"),
|
||||
)
|
||||
self.assertEqual(pred.category_id, 10)
|
||||
self.assertEqual(pred.tag_ids, (100,))
|
||||
self.assertEqual(pred.entity_ids, (200,))
|
||||
|
||||
def test_respects_allowed_label_sets(self):
|
||||
tagger = TransactionAutoTagger.fit(self._sample_records())
|
||||
pred = tagger.predict(
|
||||
description="UBER *TRIP",
|
||||
notes="",
|
||||
account_id=1,
|
||||
tx_type="EX",
|
||||
amount=Decimal("20.00"),
|
||||
allowed_category_ids=set(),
|
||||
allowed_tag_ids={100},
|
||||
allowed_entity_ids=set(),
|
||||
)
|
||||
self.assertIsNone(pred.category_id)
|
||||
self.assertEqual(pred.tag_ids, (100,))
|
||||
self.assertEqual(pred.entity_ids, ())
|
||||
|
||||
|
||||
class AutoTaggingServiceAndRulesTests(TestCase):
|
||||
def setUp(self):
|
||||
self.user = User.objects.create_user(
|
||||
email="alice@example.com", password="password123"
|
||||
)
|
||||
self.other_user = User.objects.create_user(
|
||||
email="bob@example.com", password="password123"
|
||||
)
|
||||
|
||||
self.currency = Currency.objects.create(
|
||||
code="USD", name="US Dollar", decimal_places=2, prefix="$ "
|
||||
)
|
||||
self.group = AccountGroup.objects.create(name="Checking", owner=self.user)
|
||||
self.account = Account.objects.create(
|
||||
name="Main Card",
|
||||
group=self.group,
|
||||
currency=self.currency,
|
||||
owner=self.user,
|
||||
visibility="private",
|
||||
)
|
||||
self.second_account = Account.objects.create(
|
||||
name="Savings",
|
||||
group=self.group,
|
||||
currency=self.currency,
|
||||
owner=self.user,
|
||||
visibility="private",
|
||||
)
|
||||
|
||||
self.cat_transport = TransactionCategory.objects.create(
|
||||
name="Transport", owner=self.user, visibility="private"
|
||||
)
|
||||
self.cat_other = TransactionCategory.objects.create(
|
||||
name="Other", owner=self.user, visibility="private"
|
||||
)
|
||||
self.tag_commute = TransactionTag.objects.create(
|
||||
name="Commute", owner=self.user, visibility="private"
|
||||
)
|
||||
self.tag_rule = TransactionTag.objects.create(
|
||||
name="RuleApplied", owner=self.user, visibility="private"
|
||||
)
|
||||
self.entity_uber = TransactionEntity.objects.create(
|
||||
name="Uber", owner=self.user, visibility="private"
|
||||
)
|
||||
|
||||
# Seed historical labeled transactions for Alice
|
||||
for day in (1, 5, 10):
|
||||
tx = Transaction.objects.create(
|
||||
owner=self.user,
|
||||
account=self.account,
|
||||
type=Transaction.Type.EXPENSE,
|
||||
date=datetime.date(2025, 1, day),
|
||||
amount=Decimal("25.00"),
|
||||
description=f"UBER *TRIP 0{day}/01",
|
||||
category=self.cat_transport,
|
||||
)
|
||||
tx.tags.add(self.tag_commute)
|
||||
tx.entities.add(self.entity_uber)
|
||||
|
||||
def tearDown(self):
|
||||
if get_current_user() is not None:
|
||||
delete_current_user()
|
||||
|
||||
def _tag_ids(self, tx: Transaction) -> list[int]:
|
||||
return list(
|
||||
Transaction.tags.through.objects.filter(transaction_id=tx.pk).values_list(
|
||||
"transactiontag_id", flat=True
|
||||
)
|
||||
)
|
||||
|
||||
def _entity_ids(self, tx: Transaction) -> list[int]:
|
||||
return list(
|
||||
Transaction.entities.through.objects.filter(
|
||||
transaction_id=tx.pk
|
||||
).values_list("transactionentity_id", flat=True)
|
||||
)
|
||||
|
||||
def _create_unclassified_uber_tx(self, **overrides) -> Transaction:
|
||||
defaults = {
|
||||
"owner": self.user,
|
||||
"account": self.account,
|
||||
"type": Transaction.Type.EXPENSE,
|
||||
"date": datetime.date(2025, 2, 1),
|
||||
"amount": Decimal("27.00"),
|
||||
"description": "UBER *TRIP 01/02",
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return Transaction.objects.create(**defaults)
|
||||
|
||||
def test_auto_tags_unclassified_transaction_after_rules(self):
|
||||
tx = self._create_unclassified_uber_tx()
|
||||
|
||||
check_for_transaction_rules(
|
||||
instance_id=tx.id,
|
||||
user_id=self.user.id,
|
||||
signal="transaction_created",
|
||||
)
|
||||
|
||||
tx.refresh_from_db()
|
||||
self.assertEqual(tx.category_id, self.cat_transport.id)
|
||||
self.assertEqual(self._tag_ids(tx), [self.tag_commute.id])
|
||||
self.assertEqual(self._entity_ids(tx), [self.entity_uber.id])
|
||||
|
||||
def test_skips_when_user_already_set_category_tag_or_entity(self):
|
||||
train_for_user(self.user)
|
||||
|
||||
tx_with_cat = self._create_unclassified_uber_tx(category=self.cat_other)
|
||||
self.assertFalse(apply_auto_tagging(tx_with_cat, self.user))
|
||||
tx_with_cat.refresh_from_db()
|
||||
self.assertEqual(tx_with_cat.category_id, self.cat_other.id)
|
||||
self.assertEqual(self._tag_ids(tx_with_cat), [])
|
||||
self.assertEqual(self._entity_ids(tx_with_cat), [])
|
||||
|
||||
tx_with_tag = self._create_unclassified_uber_tx()
|
||||
tx_with_tag.tags.add(self.tag_rule)
|
||||
self.assertFalse(apply_auto_tagging(tx_with_tag, self.user))
|
||||
tx_with_tag.refresh_from_db()
|
||||
self.assertIsNone(tx_with_tag.category_id)
|
||||
self.assertEqual(self._entity_ids(tx_with_tag), [])
|
||||
|
||||
tx_with_entity = self._create_unclassified_uber_tx()
|
||||
tx_with_entity.entities.add(self.entity_uber)
|
||||
self.assertFalse(apply_auto_tagging(tx_with_entity, self.user))
|
||||
tx_with_entity.refresh_from_db()
|
||||
self.assertIsNone(tx_with_entity.category_id)
|
||||
self.assertEqual(self._tag_ids(tx_with_entity), [])
|
||||
|
||||
def test_rules_run_before_auto_tagging_and_prevent_auto_tagging_if_rule_classifies(self):
|
||||
rule = TransactionRule.objects.create(
|
||||
owner=self.user,
|
||||
visibility="private",
|
||||
name="Tag Uber via Rule",
|
||||
trigger="'UBER' in description",
|
||||
on_create=True,
|
||||
active=True,
|
||||
)
|
||||
TransactionRuleAction.objects.create(
|
||||
rule=rule,
|
||||
field=TransactionRuleAction.Field.tags,
|
||||
value=f"['{self.tag_rule.name}']",
|
||||
)
|
||||
|
||||
tx = self._create_unclassified_uber_tx()
|
||||
check_for_transaction_rules(
|
||||
instance_id=tx.id,
|
||||
user_id=self.user.id,
|
||||
signal="transaction_created",
|
||||
)
|
||||
|
||||
tx.refresh_from_db()
|
||||
# Rule set the tag; auto-tagging should see the transaction is already classified and skip it
|
||||
self.assertEqual(self._tag_ids(tx), [self.tag_rule.id])
|
||||
self.assertIsNone(tx.category_id)
|
||||
self.assertEqual(self._entity_ids(tx), [])
|
||||
|
||||
def test_user_settings_disable_auto_tagging(self):
|
||||
self.user.settings.auto_tagging = False
|
||||
self.user.settings.save()
|
||||
|
||||
tx = self._create_unclassified_uber_tx()
|
||||
check_for_transaction_rules(
|
||||
instance_id=tx.id,
|
||||
user_id=self.user.id,
|
||||
signal="transaction_created",
|
||||
)
|
||||
|
||||
tx.refresh_from_db()
|
||||
self.assertIsNone(tx.category_id)
|
||||
self.assertEqual(self._tag_ids(tx), [])
|
||||
self.assertEqual(self._entity_ids(tx), [])
|
||||
|
||||
def test_per_item_auto_match_opt_out_and_inactive_items(self):
|
||||
train_for_user(self.user)
|
||||
|
||||
self.cat_transport.auto_match = False
|
||||
self.cat_transport.save()
|
||||
self.tag_commute.active = False
|
||||
self.tag_commute.save()
|
||||
|
||||
tx = self._create_unclassified_uber_tx()
|
||||
applied = apply_auto_tagging(tx, self.user)
|
||||
self.assertTrue(applied)
|
||||
|
||||
tx.refresh_from_db()
|
||||
self.assertIsNone(tx.category_id)
|
||||
self.assertEqual(self._tag_ids(tx), [])
|
||||
self.assertEqual(self._entity_ids(tx), [self.entity_uber.id])
|
||||
|
||||
def test_skips_transfers(self):
|
||||
train_for_user(self.user)
|
||||
|
||||
tx_from = self._create_unclassified_uber_tx()
|
||||
tx_from.add_transfer_side(self.second_account, Decimal("27.00"))
|
||||
tx_from.refresh_from_db()
|
||||
|
||||
self.assertFalse(apply_auto_tagging(tx_from, self.user))
|
||||
self.assertIsNone(tx_from.category_id)
|
||||
|
||||
def test_multi_user_isolation(self):
|
||||
train_for_user(self.user)
|
||||
|
||||
bob_group = AccountGroup.objects.create(name="Bob Group", owner=self.other_user)
|
||||
bob_account = Account.objects.create(
|
||||
name="Bob Card",
|
||||
group=bob_group,
|
||||
currency=self.currency,
|
||||
owner=self.other_user,
|
||||
visibility="private",
|
||||
)
|
||||
bob_tx = Transaction.objects.create(
|
||||
owner=self.other_user,
|
||||
account=bob_account,
|
||||
type=Transaction.Type.EXPENSE,
|
||||
date=datetime.date(2025, 2, 1),
|
||||
amount=Decimal("25.00"),
|
||||
description="UBER *TRIP 01/02",
|
||||
)
|
||||
|
||||
self.assertFalse(apply_auto_tagging(bob_tx, self.other_user))
|
||||
bob_tx.refresh_from_db()
|
||||
self.assertIsNone(bob_tx.category_id)
|
||||
self.assertFalse(bob_tx.tags.exists())
|
||||
self.assertFalse(bob_tx.entities.exists())
|
||||
|
||||
def test_procrastinate_training_tasks(self):
|
||||
msg = train_user_auto_tagger(self.user.id)
|
||||
self.assertIn("Trained auto-tagger", msg)
|
||||
model_obj = AutoTaggerModel.objects.get(user=self.user)
|
||||
self.assertEqual(model_obj.transaction_count, 3)
|
||||
|
||||
# Periodic task skips already-up-to-date user
|
||||
with patch(
|
||||
"apps.transactions.tasks.train_for_user", wraps=train_for_user
|
||||
) as mock_train:
|
||||
retrain_auto_taggers()
|
||||
trained_user_ids = [call.args[0].id for call in mock_train.call_args_list]
|
||||
self.assertNotIn(self.user.id, trained_user_ids)
|
||||
|
||||
def test_retrain_auto_taggers_detects_shared_account_updates(self):
|
||||
# Train Bob initially (0 transactions visible)
|
||||
train_for_user(self.other_user)
|
||||
self.assertEqual(
|
||||
AutoTaggerModel.objects.get(user=self.other_user).transaction_count, 0
|
||||
)
|
||||
|
||||
# Share Alice's account with Bob and add a new transaction owned by Alice
|
||||
self.account.shared_with.add(self.other_user)
|
||||
Transaction.objects.create(
|
||||
owner=self.user,
|
||||
account=self.account,
|
||||
type=Transaction.Type.EXPENSE,
|
||||
date=datetime.date(2025, 2, 2),
|
||||
amount=Decimal("30.00"),
|
||||
description="UBER *TRIP SHARED",
|
||||
category=self.cat_transport,
|
||||
)
|
||||
|
||||
retrain_auto_taggers()
|
||||
|
||||
bob_model = AutoTaggerModel.objects.get(user=self.other_user)
|
||||
self.assertGreater(bob_model.transaction_count, 0)
|
||||
|
||||
def test_forms_expose_auto_tagging_and_auto_match_fields(self):
|
||||
write_current_user(self.user)
|
||||
settings_form = UserSettingsForm(instance=self.user.settings)
|
||||
self.assertIn("auto_tagging", settings_form.fields)
|
||||
|
||||
cat_form = TransactionCategoryForm(instance=self.cat_transport)
|
||||
self.assertIn("auto_match", cat_form.fields)
|
||||
|
||||
tag_form = TransactionTagForm(instance=self.tag_commute)
|
||||
self.assertIn("auto_match", tag_form.fields)
|
||||
|
||||
entity_form = TransactionEntityForm(instance=self.entity_uber)
|
||||
self.assertIn("auto_match", entity_form.fields)
|
||||
@@ -158,6 +158,7 @@ class UserSettingsForm(forms.ModelForm):
|
||||
"number_format",
|
||||
"default_account",
|
||||
"combine_transfers",
|
||||
"auto_tagging",
|
||||
"volume",
|
||||
]
|
||||
widgets = {
|
||||
@@ -185,6 +186,7 @@ class UserSettingsForm(forms.ModelForm):
|
||||
"start_page",
|
||||
"default_account",
|
||||
Switch("combine_transfers"),
|
||||
Switch("auto_tagging"),
|
||||
HTML('<hr class="hr my-3" />'),
|
||||
Range("volume"),
|
||||
FormActions(
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.17 on 2026-11-19 21:22
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('users', '0029_usersettings_combine_transfers'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='usersettings',
|
||||
name='auto_tagging',
|
||||
field=models.BooleanField(default=True, help_text='Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions', verbose_name='Auto-tagging'),
|
||||
),
|
||||
]
|
||||
@@ -484,6 +484,13 @@ class UserSettings(models.Model):
|
||||
verbose_name=_("Combine transfers"),
|
||||
help_text=_("Show both sides of a transfer as a single item"),
|
||||
)
|
||||
auto_tagging = models.BooleanField(
|
||||
default=True,
|
||||
verbose_name=_("Auto-tagging"),
|
||||
help_text=_(
|
||||
"Automatically assign categories, tags, and entities to new unclassified transactions based on your past transactions"
|
||||
),
|
||||
)
|
||||
volume = models.PositiveIntegerField(
|
||||
default=10,
|
||||
validators=[MinValueValidator(0), MaxValueValidator(10)],
|
||||
|
||||
Reference in new issue
Block a user