mirror of
https://github.com/beshu-tech/deltaglider.git
synced 2026-03-18 07:23:42 +01:00
Initial commit: DeltaGlider - 99.9% compression for S3 storage
DeltaGlider reduces storage costs by storing only binary deltas between similar files. Achieves 99.9% compression for versioned artifacts. Key features: - Intelligent file type detection (delta for archives, direct for others) - Drop-in S3 replacement with automatic compression - SHA256 integrity verification on every operation - Clean hexagonal architecture - Full test coverage - Production tested with 200K+ files Case study: ReadOnlyREST reduced 4TB to 5GB (99.9% compression)
This commit is contained in:
200
.github/workflows/ci.yml
vendored
Normal file
200
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,200 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.5.13"
|
||||
PYTHON_VERSION: "3.12"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install UV
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install --system ruff
|
||||
|
||||
- name: Run ruff check
|
||||
run: |
|
||||
uv run ruff check src tests
|
||||
|
||||
- name: Run ruff format check
|
||||
run: |
|
||||
uv run ruff format --check src tests
|
||||
|
||||
typecheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install UV
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install --system mypy boto3-stubs[s3] types-python-dateutil
|
||||
|
||||
- name: Run mypy
|
||||
run: |
|
||||
uv run mypy src tests
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install UV
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install xdelta3
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y xdelta3
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install --system -e ".[dev]"
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
uv run pytest tests/unit -v --tb=short
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
uv run pytest tests/integration -v --tb=short
|
||||
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: false
|
||||
tags: deltaglider:test
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Test Docker image
|
||||
run: |
|
||||
docker run --rm deltaglider:test --help
|
||||
|
||||
e2e-test:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
localstack:
|
||||
image: localstack/localstack:latest
|
||||
ports:
|
||||
- 4566:4566
|
||||
env:
|
||||
SERVICES: s3
|
||||
DEBUG: 0
|
||||
DATA_DIR: /tmp/localstack/data
|
||||
options: >-
|
||||
--health-cmd "curl -f http://localhost:4566/_localstack/health"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install UV
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install xdelta3
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y xdelta3
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install --system -e ".[dev]"
|
||||
|
||||
- name: Run E2E tests
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
AWS_ENDPOINT_URL: http://localhost:4566
|
||||
run: |
|
||||
uv run pytest tests/e2e -v -m e2e --tb=short
|
||||
|
||||
docker-push:
|
||||
needs: [lint, typecheck, test, docker-build, e2e-test]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: deltaglider/deltaglider
|
||||
tags: |
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
Reference in New Issue
Block a user