Initial commit: DeltaGlider - 99.9% compression for S3 storage

DeltaGlider reduces storage costs by storing only binary deltas between
similar files. Achieves 99.9% compression for versioned artifacts.

Key features:
- Intelligent file type detection (delta for archives, direct for others)
- Drop-in S3 replacement with automatic compression
- SHA256 integrity verification on every operation
- Clean hexagonal architecture
- Full test coverage
- Production tested with 200K+ files

Case study: ReadOnlyREST reduced 4TB to 5GB (99.9% compression)
This commit is contained in:
Simone Scarduzio
2025-09-22 15:49:31 +02:00
commit 7562064832
50 changed files with 4520 additions and 0 deletions

200
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,200 @@
name: CI
on:
push:
branches: [main, develop]
tags: ["v*"]
pull_request:
branches: [main]
env:
UV_VERSION: "0.5.13"
PYTHON_VERSION: "3.12"
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install UV
run: |
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install dependencies
run: |
uv pip install --system ruff
- name: Run ruff check
run: |
uv run ruff check src tests
- name: Run ruff format check
run: |
uv run ruff format --check src tests
typecheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install UV
run: |
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install dependencies
run: |
uv pip install --system mypy boto3-stubs[s3] types-python-dateutil
- name: Run mypy
run: |
uv run mypy src tests
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install UV
run: |
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install xdelta3
run: |
sudo apt-get update
sudo apt-get install -y xdelta3
- name: Install dependencies
run: |
uv pip install --system -e ".[dev]"
- name: Run unit tests
run: |
uv run pytest tests/unit -v --tb=short
- name: Run integration tests
run: |
uv run pytest tests/integration -v --tb=short
docker-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker image
uses: docker/build-push-action@v5
with:
context: .
push: false
tags: deltaglider:test
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Test Docker image
run: |
docker run --rm deltaglider:test --help
e2e-test:
runs-on: ubuntu-latest
services:
localstack:
image: localstack/localstack:latest
ports:
- 4566:4566
env:
SERVICES: s3
DEBUG: 0
DATA_DIR: /tmp/localstack/data
options: >-
--health-cmd "curl -f http://localhost:4566/_localstack/health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- name: Install UV
run: |
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install xdelta3
run: |
sudo apt-get update
sudo apt-get install -y xdelta3
- name: Install dependencies
run: |
uv pip install --system -e ".[dev]"
- name: Run E2E tests
env:
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL: http://localhost:4566
run: |
uv run pytest tests/e2e -v -m e2e --tb=short
docker-push:
needs: [lint, typecheck, test, docker-build, e2e-test]
runs-on: ubuntu-latest
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: deltaglider/deltaglider
tags: |
type=ref,event=tag
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max