Initial commit: DeltaGlider - 99.9% compression for S3 storage

DeltaGlider reduces storage costs by storing only binary deltas between
similar files. Achieves 99.9% compression for versioned artifacts.

Key features:
- Intelligent file type detection (delta for archives, direct for others)
- Drop-in S3 replacement with automatic compression
- SHA256 integrity verification on every operation
- Clean hexagonal architecture
- Full test coverage
- Production tested with 200K+ files

Case study: ReadOnlyREST reduced 4TB to 5GB (99.9% compression)

src/deltaglider/adapters/__init__.py

@@ -0,0 +1,19 @@
+"""Adapters for DeltaGlider."""
+
+from .cache_fs import FsCacheAdapter
+from .clock_utc import UtcClockAdapter
+from .diff_xdelta import XdeltaAdapter
+from .hash_sha import Sha256Adapter
+from .logger_std import StdLoggerAdapter
+from .metrics_noop import NoopMetricsAdapter
+from .storage_s3 import S3StorageAdapter
+
+__all__ = [
+    "S3StorageAdapter",
+    "XdeltaAdapter",
+    "Sha256Adapter",
+    "FsCacheAdapter",
+    "UtcClockAdapter",
+    "StdLoggerAdapter",
+    "NoopMetricsAdapter",
+]

src/deltaglider/adapters/cache_fs.py

@@ -0,0 +1,49 @@
+"""Filesystem cache adapter."""
+
+import shutil
+from pathlib import Path
+
+from ..ports.cache import CachePort
+from ..ports.hash import HashPort
+
+
+class FsCacheAdapter(CachePort):
+    """Filesystem implementation of CachePort."""
+
+    def __init__(self, base_dir: Path, hasher: HashPort):
+        """Initialize with base directory."""
+        self.base_dir = base_dir
+        self.hasher = hasher
+
+    def ref_path(self, bucket: str, leaf: str) -> Path:
+        """Get path where reference should be cached."""
+        cache_dir = self.base_dir / bucket / leaf
+        return cache_dir / "reference.bin"
+
+    def has_ref(self, bucket: str, leaf: str, sha: str) -> bool:
+        """Check if reference exists and matches SHA."""
+        path = self.ref_path(bucket, leaf)
+        if not path.exists():
+            return False
+
+        actual_sha = self.hasher.sha256(path)
+        return actual_sha == sha
+
+    def write_ref(self, bucket: str, leaf: str, src: Path) -> Path:
+        """Cache reference file."""
+        path = self.ref_path(bucket, leaf)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, path)
+        return path
+
+    def evict(self, bucket: str, leaf: str) -> None:
+        """Remove cached reference."""
+        path = self.ref_path(bucket, leaf)
+        if path.exists():
+            path.unlink()
+        # Clean up empty directories
+        try:
+            path.parent.rmdir()
+            (path.parent.parent).rmdir()
+        except OSError:
+            pass  # Directory not empty

src/deltaglider/adapters/clock_utc.py

@@ -0,0 +1,13 @@
+"""UTC clock adapter."""
+
+from datetime import UTC, datetime
+
+from ..ports.clock import ClockPort
+
+
+class UtcClockAdapter(ClockPort):
+    """UTC implementation of ClockPort."""
+
+    def now(self) -> datetime:
+        """Get current UTC time."""
+        return datetime.now(UTC).replace(tzinfo=None)

src/deltaglider/adapters/diff_xdelta.py

@@ -0,0 +1,55 @@
+"""Xdelta3 diff adapter."""
+
+import subprocess
+from pathlib import Path
+
+from ..ports.diff import DiffPort
+
+
+class XdeltaAdapter(DiffPort):
+    """Xdelta3 implementation of DiffPort."""
+
+    def __init__(self, xdelta_path: str = "xdelta3"):
+        """Initialize with xdelta3 path."""
+        self.xdelta_path = xdelta_path
+
+    def encode(self, base: Path, target: Path, out: Path) -> None:
+        """Create delta from base to target."""
+        cmd = [
+            self.xdelta_path,
+            "-e",  # encode
+            "-f",  # force overwrite
+            "-9",  # compression level
+            "-s", str(base),  # source file
+            str(target),  # target file
+            str(out),  # output delta
+        ]
+
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+        )
+
+        if result.returncode != 0:
+            raise RuntimeError(f"xdelta3 encode failed: {result.stderr}")
+
+    def decode(self, base: Path, delta: Path, out: Path) -> None:
+        """Apply delta to base to recreate target."""
+        cmd = [
+            self.xdelta_path,
+            "-d",  # decode
+            "-f",  # force overwrite
+            "-s", str(base),  # source file
+            str(delta),  # delta file
+            str(out),  # output file
+        ]
+
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+        )
+
+        if result.returncode != 0:
+            raise RuntimeError(f"xdelta3 decode failed: {result.stderr}")

src/deltaglider/adapters/hash_sha.py

@@ -0,0 +1,29 @@
+"""SHA256 hash adapter."""
+
+import hashlib
+from pathlib import Path
+from typing import BinaryIO
+
+from ..ports.hash import HashPort
+
+
+class Sha256Adapter(HashPort):
+    """SHA256 implementation of HashPort."""
+
+    def sha256(self, path_or_stream: Path | BinaryIO) -> str:
+        """Compute SHA256 hash."""
+        hasher = hashlib.sha256()
+
+        if isinstance(path_or_stream, Path):
+            with open(path_or_stream, "rb") as f:
+                while chunk := f.read(8192):
+                    hasher.update(chunk)
+        else:
+            # Reset position if possible
+            if hasattr(path_or_stream, "seek"):
+                path_or_stream.seek(0)
+
+            while chunk := path_or_stream.read(8192):
+                hasher.update(chunk)
+
+        return hasher.hexdigest()

src/deltaglider/adapters/logger_std.py

@@ -0,0 +1,67 @@
+"""Standard logger adapter."""
+
+import json
+import logging
+import sys
+from typing import Any
+
+from ..ports.logger import LoggerPort
+
+
+class StdLoggerAdapter(LoggerPort):
+    """Standard logging implementation of LoggerPort."""
+
+    def __init__(self, name: str = "deltaglider", level: str = "INFO"):
+        """Initialize logger."""
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(getattr(logging, level.upper()))
+
+        if not self.logger.handlers:
+            handler = logging.StreamHandler(sys.stderr)
+            formatter = logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+            )
+            handler.setFormatter(formatter)
+            self.logger.addHandler(handler)
+
+    def debug(self, message: str, **kwargs: Any) -> None:
+        """Log debug message."""
+        self._log(logging.DEBUG, message, kwargs)
+
+    def info(self, message: str, **kwargs: Any) -> None:
+        """Log info message."""
+        self._log(logging.INFO, message, kwargs)
+
+    def warning(self, message: str, **kwargs: Any) -> None:
+        """Log warning message."""
+        self._log(logging.WARNING, message, kwargs)
+
+    def error(self, message: str, **kwargs: Any) -> None:
+        """Log error message."""
+        self._log(logging.ERROR, message, kwargs)
+
+    def log_operation(
+        self,
+        op: str,
+        key: str,
+        leaf: str,
+        sizes: dict[str, int],
+        durations: dict[str, float],
+        cache_hit: bool = False,
+    ) -> None:
+        """Log structured operation data."""
+        data = {
+            "op": op,
+            "key": key,
+            "leaf": leaf,
+            "sizes": sizes,
+            "durations": durations,
+            "cache_hit": cache_hit,
+        }
+        self.info(f"Operation: {op}", **data)
+
+    def _log(self, level: int, message: str, data: dict[str, Any]) -> None:
+        """Log with structured data."""
+        if data:
+            message = f"{message} - {json.dumps(data)}"
+        self.logger.log(level, message)

src/deltaglider/adapters/metrics_noop.py

@@ -0,0 +1,20 @@
+"""No-op metrics adapter."""
+
+
+from ..ports.metrics import MetricsPort
+
+
+class NoopMetricsAdapter(MetricsPort):
+    """No-op implementation of MetricsPort."""
+
+    def increment(self, name: str, value: int = 1, tags: dict[str, str] | None = None) -> None:
+        """No-op increment counter."""
+        pass
+
+    def gauge(self, name: str, value: float, tags: dict[str, str] | None = None) -> None:
+        """No-op set gauge."""
+        pass
+
+    def timing(self, name: str, value: float, tags: dict[str, str] | None = None) -> None:
+        """No-op record timing."""
+        pass

src/deltaglider/adapters/storage_s3.py

@@ -0,0 +1,136 @@
+"""S3 storage adapter."""
+
+import os
+from collections.abc import Iterator
+from pathlib import Path
+from typing import TYPE_CHECKING, BinaryIO, Optional
+
+import boto3
+from botocore.exceptions import ClientError
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3.client import S3Client
+
+from ..ports.storage import ObjectHead, PutResult, StoragePort
+
+
+class S3StorageAdapter(StoragePort):
+    """S3 implementation of StoragePort."""
+
+    def __init__(
+        self,
+        client: Optional["S3Client"] = None,
+        endpoint_url: str | None = None,
+    ):
+        """Initialize with S3 client."""
+        if client is None:
+            self.client = boto3.client(
+                "s3",
+                endpoint_url=endpoint_url or os.environ.get("AWS_ENDPOINT_URL"),
+            )
+        else:
+            self.client = client
+
+    def head(self, key: str) -> ObjectHead | None:
+        """Get object metadata."""
+        bucket, object_key = self._parse_key(key)
+
+        try:
+            response = self.client.head_object(Bucket=bucket, Key=object_key)
+            return ObjectHead(
+                key=object_key,
+                size=response["ContentLength"],
+                etag=response["ETag"].strip('"'),
+                last_modified=response["LastModified"],
+                metadata=self._extract_metadata(response.get("Metadata", {})),
+            )
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                return None
+            raise
+
+    def list(self, prefix: str) -> Iterator[ObjectHead]:
+        """List objects by prefix."""
+        bucket, prefix_key = self._parse_key(prefix)
+
+        paginator = self.client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=bucket, Prefix=prefix_key)
+
+        for page in pages:
+            for obj in page.get("Contents", []):
+                # Get full metadata
+                head = self.head(f"{bucket}/{obj['Key']}")
+                if head:
+                    yield head
+
+    def get(self, key: str) -> BinaryIO:
+        """Get object content as stream."""
+        bucket, object_key = self._parse_key(key)
+
+        try:
+            response = self.client.get_object(Bucket=bucket, Key=object_key)
+            return response["Body"]
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "NoSuchKey":
+                raise FileNotFoundError(f"Object not found: {key}") from e
+            raise
+
+    def put(
+        self,
+        key: str,
+        body: BinaryIO | bytes | Path,
+        metadata: dict[str, str],
+        content_type: str = "application/octet-stream",
+    ) -> PutResult:
+        """Put object with metadata."""
+        bucket, object_key = self._parse_key(key)
+
+        # Prepare body
+        if isinstance(body, Path):
+            with open(body, "rb") as f:
+                body_data = f.read()
+        elif isinstance(body, bytes):
+            body_data = body
+        else:
+            body_data = body.read()
+
+        # AWS requires lowercase metadata keys
+        clean_metadata = {k.lower(): v for k, v in metadata.items()}
+
+        try:
+            response = self.client.put_object(
+                Bucket=bucket,
+                Key=object_key,
+                Body=body_data,
+                ContentType=content_type,
+                Metadata=clean_metadata,
+            )
+            return PutResult(
+                etag=response["ETag"].strip('"'),
+                version_id=response.get("VersionId"),
+            )
+        except ClientError as e:
+            raise RuntimeError(f"Failed to put object: {e}") from e
+
+    def delete(self, key: str) -> None:
+        """Delete object."""
+        bucket, object_key = self._parse_key(key)
+
+        try:
+            self.client.delete_object(Bucket=bucket, Key=object_key)
+        except ClientError as e:
+            if e.response["Error"]["Code"] != "NoSuchKey":
+                raise
+
+    def _parse_key(self, key: str) -> tuple[str, str]:
+        """Parse bucket/key from combined key."""
+        parts = key.split("/", 1)
+        if len(parts) != 2:
+            raise ValueError(f"Invalid key format: {key}")
+        return parts[0], parts[1]
+
+    def _extract_metadata(self, raw_metadata: dict[str, str]) -> dict[str, str]:
+        """Extract user metadata from S3 response."""
+        # S3 returns user metadata as-is (already lowercase)
+        return raw_metadata
+