diff --git a/.env.example b/.env.example index 914162f..b1aa9c5 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,5 @@ WEB_CONCURRENCY=4 ENABLE_SOFT_DELETE=false # If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all. KEEP_DELETED_TRANSACTIONS_FOR=365 + +TASK_WORKERS=1 # This only work if you're using the single container option. Increase to have more open queues via procrastinate, you probably don't need to increase this. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95a0cc5..f0221d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,8 @@ name: Release Pipeline on: release: types: [created] + push: + branches: [ main ] env: IMAGE_NAME: wygiwyh @@ -29,7 +31,21 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push image + - name: Build and push nightly image + if: github.event_name == 'push' + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/prod/django/Dockerfile + push: true + provenance: false + tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push release image + if: github.event_name == 'release' uses: docker/build-push-action@v6 with: context: . diff --git a/app/WYGIWYH/settings.py b/app/WYGIWYH/settings.py index 636956b..14d9a09 100644 --- a/app/WYGIWYH/settings.py +++ b/app/WYGIWYH/settings.py @@ -222,7 +222,7 @@ SESSION_COOKIE_SECURE = os.getenv("HTTPS_ENABLED", "false").lower() == "true" DEBUG_TOOLBAR_CONFIG = { "ROOT_TAG_EXTRA_ATTRS": "hx-preserve", - "SHOW_TOOLBAR_CALLBACK": lambda r: False, # disables it} + # "SHOW_TOOLBAR_CALLBACK": lambda r: False, # disables it } DEBUG_TOOLBAR_PANELS = [ "debug_toolbar.panels.history.HistoryPanel", diff --git a/app/apps/common/tasks.py b/app/apps/common/tasks.py index 9e0d1aa..3cee864 100644 --- a/app/apps/common/tasks.py +++ b/app/apps/common/tasks.py @@ -1,5 +1,8 @@ import logging +from asgiref.sync import sync_to_async +from django.core import management + from procrastinate import builtin_tasks from procrastinate.contrib.django import app @@ -24,3 +27,16 @@ async def remove_old_jobs(context, timestamp): exc_info=True, ) raise e + + +@app.periodic(cron="0 6 1 * *") +@app.task(queueing_lock="remove_expired_sessions") +async def remove_expired_sessions(timestamp=None): + """Cleanup expired sessions by using Django management command.""" + try: + await sync_to_async(management.call_command)("clearsessions", verbosity=0) + except Exception: + logger.error( + "Error while executing 'remove_expired_sessions' task", + exc_info=True, + ) diff --git a/app/apps/import_app/schemas/v1.py b/app/apps/import_app/schemas/v1.py index 01ae643..7289140 100644 --- a/app/apps/import_app/schemas/v1.py +++ b/app/apps/import_app/schemas/v1.py @@ -65,7 +65,7 @@ class CSVImportSettings(BaseModel): class ColumnMapping(BaseModel): - source: Optional[str] = Field( + source: Optional[str] | Optional[list[str]] = Field( default=None, description="CSV column header. If None, the field will be generated from transformations", ) diff --git a/app/apps/import_app/services/v1.py b/app/apps/import_app/services/v1.py index d84935e..1d72b1e 100644 --- a/app/apps/import_app/services/v1.py +++ b/app/apps/import_app/services/v1.py @@ -486,8 +486,18 @@ class ImportService: mapped_data = {} for field, mapping in self.mapping.items(): - # If source is None, use None as the initial value - value = row.get(mapping.source) if mapping.source else None + value = None + + if isinstance(mapping.source, str): + value = row.get(mapping.source) + elif isinstance(mapping.source, list): + for source in mapping.source: + value = row.get(source) + if value is not None: + break + else: + # If source is None, use None as the initial value + value = None # Use default_value if value is None if value is None: diff --git a/app/apps/monthly_overview/views.py b/app/apps/monthly_overview/views.py index 0289a4d..73933e1 100644 --- a/app/apps/monthly_overview/views.py +++ b/app/apps/monthly_overview/views.py @@ -30,6 +30,8 @@ def index(request): @login_required @require_http_methods(["GET"]) def monthly_overview(request, month: int, year: int): + order = request.session.get("monthly_transactions_order", "default") + if month < 1 or month > 12: from django.http import Http404 @@ -54,6 +56,7 @@ def monthly_overview(request, month: int, year: int): "previous_month": previous_month, "previous_year": previous_year, "filter": f, + "order": order, }, ) @@ -62,7 +65,12 @@ def monthly_overview(request, month: int, year: int): @login_required @require_http_methods(["GET"]) def transactions_list(request, month: int, year: int): - order = request.GET.get("order") + order = request.session.get("monthly_transactions_order", "default") + + if "order" in request.GET: + order = request.GET["order"] + if order != request.session.get("monthly_transactions_order", "default"): + request.session["monthly_transactions_order"] = order f = TransactionsFilter(request.GET) transactions_filtered = ( diff --git a/app/apps/transactions/views/transactions.py b/app/apps/transactions/views/transactions.py index 33156de..23c9ec8 100644 --- a/app/apps/transactions/views/transactions.py +++ b/app/apps/transactions/views/transactions.py @@ -313,15 +313,23 @@ def transaction_pay(request, transaction_id): @login_required @require_http_methods(["GET"]) def transaction_all_index(request): + order = request.session.get("all_transactions_order", "default") f = TransactionsFilter(request.GET) - return render(request, "transactions/pages/transactions.html", {"filter": f}) + return render( + request, "transactions/pages/transactions.html", {"filter": f, "order": order} + ) @only_htmx @login_required @require_http_methods(["GET"]) def transaction_all_list(request): - order = request.GET.get("order") + order = request.session.get("all_transactions_order", "default") + + if "order" in request.GET: + order = request.GET["order"] + if order != request.session.get("all_transactions_order", "default"): + request.session["all_transactions_order"] = order transactions = Transaction.objects.prefetch_related( "account", diff --git a/app/templates/monthly_overview/pages/overview.html b/app/templates/monthly_overview/pages/overview.html index 03e817d..e1acf1f 100644 --- a/app/templates/monthly_overview/pages/overview.html +++ b/app/templates/monthly_overview/pages/overview.html @@ -113,9 +113,9 @@
diff --git a/app/templates/transactions/pages/transactions.html b/app/templates/transactions/pages/transactions.html index b44872e..6336628 100644 --- a/app/templates/transactions/pages/transactions.html +++ b/app/templates/transactions/pages/transactions.html @@ -32,9 +32,9 @@
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 133d522..d632e4f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,13 +3,13 @@ volumes: wygiwyh_temp: services: - web: &django + web: build: context: . dockerfile: ./docker/dev/django/Dockerfile image: wygiwyh_dev_server container_name: wygiwyh_dev_server - command: /start + command: /start-supervisor volumes: - ./app/:/usr/src/app/:z - ./frontend/:/usr/src/frontend:z @@ -54,12 +54,12 @@ services: - '${SQL_PORT}:5432' restart: unless-stopped - procrastinate: - <<: *django - image: wygiwyh_dev_procrastinate - container_name: wygiwyh_dev_procrastinate - depends_on: - - db - ports: [ ] - command: /start-procrastinate - restart: unless-stopped +# procrastinate: +# <<: *django +# image: wygiwyh_dev_procrastinate +# container_name: wygiwyh_dev_procrastinate +# depends_on: +# - db +# ports: [ ] +# command: /start-procrastinate +# restart: unless-stopped diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b840e46..c0c10a0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,15 +2,13 @@ services: web: image: eitchtee/wygiwyh:latest container_name: ${SERVER_NAME} - command: /start + command: /start-single ports: - "${OUTBOUND_PORT}:8000" env_file: - .env depends_on: - db - volumes: - - wygiwyh_temp:/usr/src/app/temp/ restart: unless-stopped db: @@ -23,18 +21,3 @@ services: - POSTGRES_USER=${SQL_USER} - POSTGRES_PASSWORD=${SQL_PASSWORD} - POSTGRES_DB=${SQL_DATABASE} - - procrastinate: - image: eitchtee/wygiwyh:latest - container_name: ${PROCRASTINATE_NAME} - depends_on: - - db - env_file: - - .env - volumes: - - wygiwyh_temp:/usr/src/app/temp/ - command: /start-procrastinate - restart: unless-stopped - -volumes: - wygiwyh_temp: diff --git a/docker/dev/django/Dockerfile b/docker/dev/django/Dockerfile index c215782..920ca66 100644 --- a/docker/dev/django/Dockerfile +++ b/docker/dev/django/Dockerfile @@ -18,7 +18,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ COPY --from=python-build-stage /usr/src/app/wheels /wheels/ RUN apt-get update && \ - apt-get install --no-install-recommends -y gettext && \ + apt-get install --no-install-recommends -y gettext supervisor && \ rm -rf /var/lib/apt/lists/* && \ pip install --upgrade pip && \ pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* && \ @@ -26,9 +26,15 @@ RUN apt-get update && \ COPY ./docker/dev/django/start /start COPY ./docker/dev/procrastinate/start /start-procrastinate +COPY ./docker/dev/supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY ./docker/dev/supervisord/supervisord.conf /etc/supervisord.conf +COPY ./docker/dev/supervisord/start /start-supervisor + RUN sed -i 's/\r$//g' /start && \ chmod +x /start && \ sed -i 's/\r$//g' /start-procrastinate && \ - chmod +x /start-procrastinate + chmod +x /start-procrastinate && \ + sed -i 's/\r$//g' /start-supervisor && \ + chmod +x /start-supervisor COPY ./app . diff --git a/docker/dev/supervisord/start b/docker/dev/supervisord/start new file mode 100644 index 0000000..cad0431 --- /dev/null +++ b/docker/dev/supervisord/start @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +export TASK_WORKERS=${TASK_WORKERS:=1} + +exec supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/docker/dev/supervisord/supervisord.conf b/docker/dev/supervisord/supervisord.conf new file mode 100644 index 0000000..85112ff --- /dev/null +++ b/docker/dev/supervisord/supervisord.conf @@ -0,0 +1,39 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 +pidfile=/tmp/supervisord.pid +user=root + +[supervisorctl] +serverurl=unix:///run/supervisord.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[unix_http_server] +file=/run/supervisord.sock +chmod=0700 + +[program:web] +directory=/usr/src/app +command=/bin/bash /start +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 +autorestart=true +startretries=5 + +[program:procrastinate] +directory=/usr/src/app +command=/bin/bash /start-procrastinate +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_TASK_WORKERS)s +numprocs_start=1 +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 +autorestart=true +startretries=5 diff --git a/docker/prod/django/Dockerfile b/docker/prod/django/Dockerfile index 2615701..bd9684e 100644 --- a/docker/prod/django/Dockerfile +++ b/docker/prod/django/Dockerfile @@ -31,7 +31,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ COPY --from=python-build-stage /usr/src/app/wheels /wheels/ RUN --mount=type=cache,target=/root/.cache/apt \ apt-get update && \ - apt-get install --no-install-recommends -y gettext && \ + apt-get install --no-install-recommends -y gettext supervisor && \ rm -rf /var/lib/apt/lists/* && \ pip install --upgrade pip && \ pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* && \ @@ -39,10 +39,15 @@ RUN --mount=type=cache,target=/root/.cache/apt \ COPY --chown=app:app ./docker/prod/django/start /start COPY --chown=app:app ./docker/prod/procrastinate/start /start-procrastinate +COPY --chown=app:app ./docker/prod/supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY --chown=app:app ./docker/prod/supervisord/supervisord.conf /etc/supervisord.conf +COPY --chown=app:app ./docker/prod/supervisord/start /start-single RUN sed -i 's/\r$//g' /start && \ chmod +x /start && \ sed -i 's/\r$//g' /start-procrastinate && \ - chmod +x /start-procrastinate + chmod +x /start-procrastinate && \ + sed -i 's/\r$//g' /start-single && \ + chmod +x /start-single COPY --chown=app:app ./app . diff --git a/docker/prod/supervisord/start b/docker/prod/supervisord/start new file mode 100644 index 0000000..cad0431 --- /dev/null +++ b/docker/prod/supervisord/start @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +export TASK_WORKERS=${TASK_WORKERS:=1} + +exec supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/docker/prod/supervisord/supervisord.conf b/docker/prod/supervisord/supervisord.conf new file mode 100644 index 0000000..553ffe2 --- /dev/null +++ b/docker/prod/supervisord/supervisord.conf @@ -0,0 +1,37 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 +pidfile=/tmp/supervisord.pid + +[supervisorctl] +serverurl=unix:///tmp/supervisord.sock + +[unix_http_server] +file=/tmp/supervisord.sock +chmod=0700 + +[program:web] +user=app +directory=/usr/src/app +command=/bin/bash /start +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 +autorestart=true +startretries=5 + +[program:procrastinate] +user=app +directory=/usr/src/app +command=/bin/bash /start-procrastinate +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_TASK_WORKERS)s +numprocs_start=1 +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 +autorestart=true +startretries=5