From 70be7b3161b65f652c60aa54d970b398d2d8fc29 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Thu, 12 Nov 2026 10:13:11 +0000 Subject: [PATCH 1/2] fix(import): make QIF deduplication account-aware and keep identical transactions QIF imports deduplicate each transaction by storing a SHA-256 of its raw lines as the transaction's internal_id and skipping rows whose hash is already taken. The hash didn't include the account, which broke two cases: - Importing the same file into two accounts (possible since the account can be picked per run) skipped every row of the second import. - Two real transactions with the same date, amount and payee in one file hashed to the same ID, so the second one was always dropped. The internal ID now hashes the account's ID and the transaction's occurrence (how many identical transactions came before it in the file) together with the raw lines. Using the occurrence instead of the position in the file keeps the IDs stable when a later export of the same statement has more transactions before or between the repeated ones. Transactions imported before this change keep their old IDs. So that re-importing an old file doesn't duplicate them, the first occurrence of a transaction is also matched against the previous hash, limited to the account being imported into. Repeats of identical transactions, which the old hash always skipped, are imported now. The duplicate check also looks at soft-deleted transactions, like the CSV/Excel deduplication does. internal_id is unique, so a soft-deleted transaction with the same ID used to fail the import with an integrity error instead of being skipped. Fixes #671 --- app/apps/import_app/services/v1.py | 42 +++++- app/apps/import_app/tests/test_qif_import.py | 136 +++++++++++++++++++ 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index 38cdb571..b4e0879e 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -4,7 +4,9 @@ import logging import os import re import zipfile +from collections import Counter from django.db import transaction +from django.db.models import Q from datetime import datetime, date from decimal import Decimal, InvalidOperation from pathlib import Path @@ -968,12 +970,28 @@ class ImportService: f"Invalid {self.settings.file_type.upper()} file format: {str(e)}" ) + @staticmethod + def _qif_internal_id(raw_content: str, account: Account, occurrence: int) -> str: + """ + Builds the internal ID of a QIF transaction from its raw lines. + + The account is part of the hash so the same file can be imported into + different accounts, and the occurrence (how many identical transactions + came before it in the file) tells identical transactions apart, e.g. two + purchases with the same date, amount and payee. + """ + return hashlib.sha256( + f"{account.pk}:{occurrence}:{raw_content}".encode("utf-8") + ).hexdigest() + def _parse_and_import_qif(self, content_lines: list[str], filename: str) -> None: # Infer account from filename (remove extension) account_name = os.path.splitext(os.path.basename(filename))[0] current_transaction = {} raw_lines_buffer = [] + # How many times each transaction's raw lines have been seen in this file + occurrences = Counter() account = ( self.import_run.account or Account.objects.filter(name=account_name).first() @@ -994,17 +1012,31 @@ class ImportService: if current_transaction: # Deduplication using hash of raw lines raw_content = "".join(raw_lines_buffer) - internal_id = hashlib.sha256( - raw_content.encode("utf-8") - ).hexdigest() + occurrence = occurrences[raw_content] + occurrences[raw_content] += 1 + internal_id = self._qif_internal_id( + raw_content, account, occurrence + ) # Reset buffer for next transaction raw_lines_buffer = [] + duplicate_filter = Q(internal_id=internal_id) + if occurrence == 0: + # Transactions imported before the account and the + # occurrence were part of the hash used the raw lines + # alone, and only ever kept the first of identical ones + duplicate_filter |= Q( + internal_id=hashlib.sha256( + raw_content.encode("utf-8") + ).hexdigest(), + account=account, + ) + try: with transaction.atomic(): - if Transaction.objects.filter( - internal_id=internal_id + if Transaction.all_objects.filter( + duplicate_filter ).exists(): self._increment_totals("skipped", 1) self._log( diff --git a/app/apps/import_app/tests/test_qif_import.py b/app/apps/import_app/tests/test_qif_import.py index d2b7e9db..2b628d96 100644 --- a/app/apps/import_app/tests/test_qif_import.py +++ b/app/apps/import_app/tests/test_qif_import.py @@ -1,4 +1,6 @@ +from datetime import date from decimal import Decimal +import hashlib import os import shutil from django.test import TestCase, override_settings @@ -12,6 +14,30 @@ from apps.transactions.models import ( Transaction, ) +QIF_YAML_CONFIG = """ +settings: + file_type: qif + importing: transactions + date_format: "%d/%m/%Y" +mapping: {} +""" + +# A salary, then two purchases with the same date, amount and payee +IDENTICAL_TRANSACTIONS_CONTENT = """!Type:Bank +D04/01/2015 +T100.00 +PSalary +^ +D05/01/2015 +T-4.50 +PCoffee Shop +^ +D05/01/2015 +T-4.50 +PCoffee Shop +^ +""" + class QIFImportTests(TestCase): def setUp(self): @@ -45,6 +71,23 @@ class QIFImportTests(TestCase): if os.path.exists(self.test_dir): shutil.rmtree(self.test_dir) + def _import_qif(self, content, account=None, filename="bradesco-checking.qif"): + file_path = os.path.join(self.test_dir, filename) + with open(file_path, "w", encoding="utf-8") as f: + f.write(content) + + profile = ImportProfile.objects.create( + name="QIF Profile", + yaml_config=QIF_YAML_CONFIG, + version=ImportProfile.Versions.VERSION_1, + ) + run = ImportRun.objects.create( + profile=profile, file_name=filename, account=account + ) + ImportService(run).process_file(file_path) + run.refresh_from_db() + return run + def test_import_single_qif_valid_mapping(self): content = """!Type:Bank D04/01/2015 @@ -257,3 +300,96 @@ mapping: {} self.assertEqual( Transaction.objects.first().description, "" ) # empty desc if no memo + + def test_import_same_file_into_two_accounts(self): + savings = Account.objects.create( + name="bradesco-savings", + group=self.group, + currency=self.currency, + owner=self.user, + ) + + into_checking = self._import_qif( + IDENTICAL_TRANSACTIONS_CONTENT, account=self.account + ) + into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings) + into_savings_again = self._import_qif( + IDENTICAL_TRANSACTIONS_CONTENT, account=savings + ) + + self.assertEqual(into_checking.successful_rows, 3) + self.assertEqual(into_savings.successful_rows, 3) + self.assertEqual(into_savings.skipped_rows, 0) + self.assertEqual(into_savings_again.successful_rows, 0) + self.assertEqual(into_savings_again.skipped_rows, 3) + self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3) + self.assertEqual(Transaction.objects.filter(account=savings).count(), 3) + + def test_import_identical_transactions_in_one_file(self): + first_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT) + + self.assertEqual(first_run.successful_rows, 3) + self.assertEqual(first_run.skipped_rows, 0) + coffees = Transaction.objects.filter(entities__name="Coffee Shop") + self.assertEqual(coffees.count(), 2) + self.assertEqual(coffees.values("internal_id").distinct().count(), 2) + + # Re-importing the file still recognizes both of them + second_run = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT) + + self.assertEqual(second_run.successful_rows, 0) + self.assertEqual(second_run.skipped_rows, 3) + self.assertEqual(Transaction.objects.count(), 3) + + def test_import_skips_transactions_imported_with_the_previous_hash(self): + # Imports made before the account was part of the hash used the raw + # lines alone, and skipped every repeat of identical transactions + legacy_internal_id = hashlib.sha256( + "D05/01/2015T-4.50PCoffee Shop^".encode("utf-8") + ).hexdigest() + Transaction.objects.create( + account=self.account, + type=Transaction.Type.EXPENSE, + date=date(2015, 1, 5), + amount=Decimal("4.50"), + internal_id=legacy_internal_id, + ) + savings = Account.objects.create( + name="bradesco-savings", + group=self.group, + currency=self.currency, + owner=self.user, + ) + + into_checking = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT) + into_savings = self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT, account=savings) + + # The first coffee is the one imported before, the second one was + # skipped back then and is imported now + self.assertEqual(into_checking.successful_rows, 2) + self.assertEqual(into_checking.skipped_rows, 1) + self.assertEqual(Transaction.objects.filter(account=self.account).count(), 3) + # The previous hash doesn't say which account it was for, so it only + # counts as a duplicate in the account the transaction is in + self.assertEqual(into_savings.successful_rows, 3) + self.assertEqual(into_savings.skipped_rows, 0) + self.assertEqual(Transaction.objects.filter(account=savings).count(), 3) + + @override_settings(ENABLE_SOFT_DELETE=True) + def test_import_skips_soft_deleted_transactions(self): + content = """!Type:Bank +D04/01/2015 +T100.00 +POK +^ +""" + self._import_qif(content) + Transaction.objects.get().delete() + self.assertTrue(Transaction.deleted_objects.exists()) + + run = self._import_qif(content) + + self.assertEqual(run.status, ImportRun.Status.FINISHED) + self.assertEqual(run.successful_rows, 0) + self.assertEqual(run.skipped_rows, 1) + self.assertFalse(Transaction.objects.exists()) From d763d0ffb8c76c7fc1d9a3b95c3f84214862fa5a Mon Sep 17 00:00:00 2001 From: Herculino Trotta Date: Sat, 14 Nov 2026 12:16:15 -0300 Subject: [PATCH 2/2] fix(import): leave QIF headers out of the hash and skip non-transaction sections The !Type header line was buffered into the first transaction's raw lines, so a transaction hashed differently depending on whether it opened the export. An overlapping export with a different first transaction imported it twice. Header lines are now left out of the internal ID. The previous-hash check still hashes the full lines, headers included, and runs on the first appearance of each of those, so a first transaction and an identical later one imported before this change are both still recognized. Account blocks (!Account and !Option:AutoSwitch account lists) and non-transaction sections (!Type:Cat, !Type:Class, !Type:Memorized and any type not in QIF_TRANSACTION_SECTIONS) were read with the transaction codes, where T is an account's type and D a description, so files containing them failed to import. They are now skipped, and each skipped section is logged. --- app/apps/import_app/services/v1.py | 54 ++++++++- app/apps/import_app/tests/test_qif_import.py | 114 +++++++++++++++++++ 2 files changed, 163 insertions(+), 5 deletions(-) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index b4e0879e..487bee2d 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -41,6 +41,17 @@ from apps.import_app.schemas.v1 import ( logger = logging.getLogger(__name__) +# QIF sections that hold transactions. Others, like category lists (!Type:Cat) +# or memorized transactions (!Type:Memorized), are skipped +QIF_TRANSACTION_SECTIONS = { + "!type:bank", + "!type:cash", + "!type:ccard", + "!type:invst", + "!type:oth a", + "!type:oth l", +} + class ImportService: def __init__(self, import_run: ImportRun): @@ -992,6 +1003,10 @@ class ImportService: raw_lines_buffer = [] # How many times each transaction's raw lines have been seen in this file occurrences = Counter() + legacy_contents_seen = set() + in_account_list = False + in_account_block = False + in_other_section = False account = ( self.import_run.account or Account.objects.filter(name=account_name).first() @@ -1006,12 +1021,39 @@ class ImportService: if not line: continue + # Account blocks (!Account, or an !Option:AutoSwitch account list) + # and non-transaction sections use codes that mean something else + # in a transaction, e.g. T is an account's type and D its description + header = line.lower() + if header == "!option:autoswitch": + in_account_list = True + elif header == "!clear:autoswitch": + in_account_list = False + elif header == "!account": + in_account_block = True + elif header.startswith("!type:"): + in_other_section = header not in QIF_TRANSACTION_SECTIONS + if in_other_section: + self._log("info", f"Skipped {line} section in {filename}") + if in_account_list or in_account_block or in_other_section: + if line == "^": + in_account_block = False + raw_lines_buffer = [] + continue + raw_lines_buffer.append(line) if line == "^": if current_transaction: - # Deduplication using hash of raw lines - raw_content = "".join(raw_lines_buffer) + # Deduplication using hash of raw lines. Header lines (!Type, + # !Account) are left out so a transaction hashes the same + # whether or not it comes right after one + legacy_content = "".join(raw_lines_buffer) + raw_content = "".join( + raw_line + for raw_line in raw_lines_buffer + if not raw_line.startswith("!") + ) occurrence = occurrences[raw_content] occurrences[raw_content] += 1 internal_id = self._qif_internal_id( @@ -1022,13 +1064,15 @@ class ImportService: raw_lines_buffer = [] duplicate_filter = Q(internal_id=internal_id) - if occurrence == 0: + if legacy_content not in legacy_contents_seen: # Transactions imported before the account and the # occurrence were part of the hash used the raw lines - # alone, and only ever kept the first of identical ones + # alone, headers included, and only ever kept the first + # of identical ones + legacy_contents_seen.add(legacy_content) duplicate_filter |= Q( internal_id=hashlib.sha256( - raw_content.encode("utf-8") + legacy_content.encode("utf-8") ).hexdigest(), account=account, ) diff --git a/app/apps/import_app/tests/test_qif_import.py b/app/apps/import_app/tests/test_qif_import.py index 2b628d96..8c2b98fd 100644 --- a/app/apps/import_app/tests/test_qif_import.py +++ b/app/apps/import_app/tests/test_qif_import.py @@ -375,6 +375,120 @@ mapping: {} self.assertEqual(into_savings.skipped_rows, 0) self.assertEqual(Transaction.objects.filter(account=savings).count(), 3) + def test_import_overlapping_exports_with_a_different_first_transaction(self): + # The first coffee opens the second export, right after the header + later_export = """!Type:Bank +D05/01/2015 +T-4.50 +PCoffee Shop +^ +D06/01/2015 +T-20.00 +PMarket +^ +""" + self._import_qif(IDENTICAL_TRANSACTIONS_CONTENT) + + run = self._import_qif(later_export) + + self.assertEqual(run.successful_rows, 1) + self.assertEqual(run.skipped_rows, 1) + self.assertEqual(Transaction.objects.count(), 4) + + def test_import_skips_previous_hash_of_a_first_transaction_repeated_later(self): + # The previous hash included the header in the first transaction, so a + # first transaction and an identical later one were both imported + content = """!Type:Bank +D05/01/2015 +T-4.50 +PCoffee Shop +^ +D05/01/2015 +T-4.50 +PCoffee Shop +^ +""" + for raw_content in ( + "!Type:BankD05/01/2015T-4.50PCoffee Shop^", + "D05/01/2015T-4.50PCoffee Shop^", + ): + Transaction.objects.create( + account=self.account, + type=Transaction.Type.EXPENSE, + date=date(2015, 1, 5), + amount=Decimal("4.50"), + internal_id=hashlib.sha256(raw_content.encode("utf-8")).hexdigest(), + ) + + run = self._import_qif(content) + + self.assertEqual(run.successful_rows, 0) + self.assertEqual(run.skipped_rows, 2) + self.assertEqual(Transaction.objects.count(), 2) + + def test_import_skips_account_blocks(self): + # Quicken-style export: an account list, then an account block before + # the transactions. T and D mean the account type and description there + content = """!Option:AutoSwitch +!Account +NChecking +TBank +DMain checking account +^ +NSavings +TBank +^ +!Clear:AutoSwitch +!Account +NChecking +TBank +DMain checking account +^ +!Type:Bank +D04/01/2015 +T100.00 +PSalary +^ +""" + run = self._import_qif(content) + + self.assertEqual(run.status, ImportRun.Status.FINISHED) + self.assertEqual(run.successful_rows, 1) + self.assertEqual(run.failed_rows, 0) + transaction = Transaction.objects.get() + self.assertEqual(transaction.amount, Decimal("100.00")) + self.assertEqual(transaction.date, date(2015, 1, 4)) + + def test_import_skips_non_transaction_sections(self): + # Category and class lists, and memorized transactions (templates, not + # real transactions), around a section of real ones + content = """!Type:Cat +NFood +DGroceries and restaurants +E +^ +!Type:Class +NPersonal +DPersonal expenses +^ +!Type:Bank +D04/01/2015 +T100.00 +PSalary +^ +!Type:Memorized +KC +T-50.00 +PRent +^ +""" + run = self._import_qif(content) + + self.assertEqual(run.status, ImportRun.Status.FINISHED) + self.assertEqual(run.successful_rows, 1) + self.assertEqual(run.failed_rows, 0) + self.assertEqual(Transaction.objects.get().amount, Decimal("100.00")) + @override_settings(ENABLE_SOFT_DELETE=True) def test_import_skips_soft_deleted_transactions(self): content = """!Type:Bank