mirror of
https://github.com/beshu-tech/deltaglider.git
synced 2026-01-11 22:50:24 +01:00
feat: Add AWS credential parameters to create_client()
- Add aws_access_key_id, aws_secret_access_key, aws_session_token, and region_name parameters - Pass credentials through to S3StorageAdapter and boto3.client() - Enables multi-tenant scenarios with different AWS accounts - Maintains backward compatibility (uses boto3 default credential chain when omitted) - Add comprehensive tests for credential handling - Add examples/credentials_example.py with usage examples Fixes credential conflicts when multiple SDK instances need different credentials. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -12,11 +12,11 @@
|
||||
|
||||
**Store 4TB of similar files in 5GB. No, that's not a typo.**
|
||||
|
||||
DeltaGlider is a drop-in S3 replacement that achieves 99.9% compression for versioned artifacts, backups, and release archives through intelligent binary delta compression.
|
||||
DeltaGlider is a drop-in S3 replacement that may achieve 99.9% size reduction for versioned compressed artifacts, backups, and release archives through intelligent binary delta compression (via xdelta3).
|
||||
|
||||
## The Problem We Solved
|
||||
|
||||
You're storing hundreds of versions of your releases. Each 100MB build differs by <1% from the previous version. You're paying to store 100GB of what's essentially 100MB of unique data.
|
||||
You're storing hundreds of versions of your software releases. Each 100MB build differs by <1% from the previous version. You're paying to store 100GB of what's essentially 100MB of unique data.
|
||||
|
||||
Sound familiar?
|
||||
|
||||
@@ -435,8 +435,8 @@ DeltaGlider uses a clean hexagonal architecture:
|
||||
- Any versioned binary data
|
||||
|
||||
❌ **Not ideal for:**
|
||||
- Already compressed unique files
|
||||
- Streaming media files
|
||||
- Already compressed **unique** files
|
||||
- Streaming or multimedia files
|
||||
- Frequently changing unstructured data
|
||||
- Files smaller than 1MB
|
||||
|
||||
|
||||
101
examples/credentials_example.py
Normal file
101
examples/credentials_example.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Example: Using explicit AWS credentials with DeltaGlider.
|
||||
|
||||
This example demonstrates how to pass AWS credentials directly to
|
||||
DeltaGlider's create_client() function, which is useful when:
|
||||
|
||||
1. You need to use different credentials than your environment default
|
||||
2. You're working with temporary credentials (session tokens)
|
||||
3. You want to avoid relying on environment variables
|
||||
4. You're implementing multi-tenant systems with different AWS accounts
|
||||
"""
|
||||
|
||||
from deltaglider import create_client
|
||||
|
||||
|
||||
def example_basic_credentials():
|
||||
"""Use basic AWS credentials (access key + secret key)."""
|
||||
client = create_client(
|
||||
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
region_name="us-west-2",
|
||||
)
|
||||
|
||||
# Now use the client normally
|
||||
# client.put_object(Bucket="my-bucket", Key="file.zip", Body=b"data")
|
||||
print("✓ Created client with explicit credentials")
|
||||
|
||||
|
||||
def example_temporary_credentials():
|
||||
"""Use temporary AWS credentials (with session token)."""
|
||||
client = create_client(
|
||||
aws_access_key_id="ASIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
aws_session_token="FwoGZXIvYXdzEBEaDH...", # From STS
|
||||
region_name="us-east-1",
|
||||
)
|
||||
|
||||
print("✓ Created client with temporary credentials")
|
||||
|
||||
|
||||
def example_environment_credentials():
|
||||
"""Use default credential chain (environment variables, IAM role, etc.)."""
|
||||
# When credentials are omitted, DeltaGlider uses boto3's default credential chain:
|
||||
# 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
# 2. AWS credentials file (~/.aws/credentials)
|
||||
# 3. IAM role (for EC2 instances)
|
||||
client = create_client()
|
||||
|
||||
print("✓ Created client with default credential chain")
|
||||
|
||||
|
||||
def example_minio_credentials():
|
||||
"""Use credentials for MinIO or other S3-compatible services."""
|
||||
client = create_client(
|
||||
endpoint_url="http://localhost:9000",
|
||||
aws_access_key_id="minioadmin",
|
||||
aws_secret_access_key="minioadmin",
|
||||
)
|
||||
|
||||
print("✓ Created client for MinIO with custom credentials")
|
||||
|
||||
|
||||
def example_multi_tenant():
|
||||
"""Example: Different credentials for different tenants."""
|
||||
|
||||
# Tenant A uses one AWS account
|
||||
tenant_a_client = create_client(
|
||||
aws_access_key_id="TENANT_A_KEY",
|
||||
aws_secret_access_key="TENANT_A_SECRET",
|
||||
region_name="us-west-2",
|
||||
)
|
||||
|
||||
# Tenant B uses a different AWS account
|
||||
tenant_b_client = create_client(
|
||||
aws_access_key_id="TENANT_B_KEY",
|
||||
aws_secret_access_key="TENANT_B_SECRET",
|
||||
region_name="eu-west-1",
|
||||
)
|
||||
|
||||
print("✓ Created separate clients for multi-tenant scenario")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("DeltaGlider Credentials Examples\n" + "=" * 40)
|
||||
|
||||
print("\n1. Basic credentials:")
|
||||
example_basic_credentials()
|
||||
|
||||
print("\n2. Temporary credentials:")
|
||||
example_temporary_credentials()
|
||||
|
||||
print("\n3. Environment credentials:")
|
||||
example_environment_credentials()
|
||||
|
||||
print("\n4. MinIO credentials:")
|
||||
example_minio_credentials()
|
||||
|
||||
print("\n5. Multi-tenant scenario:")
|
||||
example_multi_tenant()
|
||||
|
||||
print("\n" + "=" * 40)
|
||||
print("All examples completed successfully!")
|
||||
@@ -21,13 +21,31 @@ class S3StorageAdapter(StoragePort):
|
||||
self,
|
||||
client: Optional["S3Client"] = None,
|
||||
endpoint_url: str | None = None,
|
||||
boto3_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
"""Initialize with S3 client."""
|
||||
"""Initialize with S3 client.
|
||||
|
||||
Args:
|
||||
client: Pre-configured S3 client (if None, one will be created)
|
||||
endpoint_url: S3 endpoint URL override (for MinIO, LocalStack, etc.)
|
||||
boto3_kwargs: Additional kwargs to pass to boto3.client() including:
|
||||
- aws_access_key_id: AWS access key
|
||||
- aws_secret_access_key: AWS secret key
|
||||
- aws_session_token: AWS session token (for temporary credentials)
|
||||
- region_name: AWS region name
|
||||
"""
|
||||
if client is None:
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint_url or os.environ.get("AWS_ENDPOINT_URL"),
|
||||
)
|
||||
# Build boto3 client parameters
|
||||
client_params: dict[str, Any] = {
|
||||
"service_name": "s3",
|
||||
"endpoint_url": endpoint_url or os.environ.get("AWS_ENDPOINT_URL"),
|
||||
}
|
||||
|
||||
# Merge in any additional boto3 kwargs (credentials, region, etc.)
|
||||
if boto3_kwargs:
|
||||
client_params.update(boto3_kwargs)
|
||||
|
||||
self.client = boto3.client(**client_params)
|
||||
else:
|
||||
self.client = client
|
||||
|
||||
|
||||
@@ -1396,6 +1396,10 @@ def create_client(
|
||||
endpoint_url: str | None = None,
|
||||
log_level: str = "INFO",
|
||||
cache_dir: str = "/tmp/.deltaglider/cache",
|
||||
aws_access_key_id: str | None = None,
|
||||
aws_secret_access_key: str | None = None,
|
||||
aws_session_token: str | None = None,
|
||||
region_name: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> DeltaGliderClient:
|
||||
"""Create a DeltaGlider client with boto3-compatible APIs.
|
||||
@@ -1411,18 +1415,28 @@ def create_client(
|
||||
endpoint_url: Optional S3 endpoint URL (for MinIO, R2, etc.)
|
||||
log_level: Logging level
|
||||
cache_dir: Directory for reference cache
|
||||
aws_access_key_id: AWS access key ID (None to use environment/IAM)
|
||||
aws_secret_access_key: AWS secret access key (None to use environment/IAM)
|
||||
aws_session_token: AWS session token for temporary credentials (None if not using)
|
||||
region_name: AWS region name (None for default)
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
DeltaGliderClient instance
|
||||
|
||||
Examples:
|
||||
>>> # Boto3-compatible usage
|
||||
>>> # Boto3-compatible usage with default credentials
|
||||
>>> client = create_client()
|
||||
>>> client.put_object(Bucket='my-bucket', Key='file.zip', Body=b'data')
|
||||
>>> response = client.get_object(Bucket='my-bucket', Key='file.zip')
|
||||
>>> data = response['Body'].read()
|
||||
|
||||
>>> # With explicit credentials
|
||||
>>> client = create_client(
|
||||
... aws_access_key_id='AKIAIOSFODNN7EXAMPLE',
|
||||
... aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
||||
... )
|
||||
|
||||
>>> # Batch operations
|
||||
>>> results = client.upload_batch(['v1.zip', 'v2.zip'], 's3://bucket/releases/')
|
||||
|
||||
@@ -1441,9 +1455,20 @@ def create_client(
|
||||
XdeltaAdapter,
|
||||
)
|
||||
|
||||
# Build boto3 client kwargs
|
||||
boto3_kwargs = {}
|
||||
if aws_access_key_id is not None:
|
||||
boto3_kwargs["aws_access_key_id"] = aws_access_key_id
|
||||
if aws_secret_access_key is not None:
|
||||
boto3_kwargs["aws_secret_access_key"] = aws_secret_access_key
|
||||
if aws_session_token is not None:
|
||||
boto3_kwargs["aws_session_token"] = aws_session_token
|
||||
if region_name is not None:
|
||||
boto3_kwargs["region_name"] = region_name
|
||||
|
||||
# Create adapters
|
||||
hasher = Sha256Adapter()
|
||||
storage = S3StorageAdapter(endpoint_url=endpoint_url)
|
||||
storage = S3StorageAdapter(endpoint_url=endpoint_url, boto3_kwargs=boto3_kwargs)
|
||||
diff = XdeltaAdapter()
|
||||
cache = FsCacheAdapter(Path(cache_dir), hasher)
|
||||
clock = UtcClockAdapter()
|
||||
|
||||
@@ -146,6 +146,68 @@ def client(tmp_path):
|
||||
return client
|
||||
|
||||
|
||||
class TestCredentialHandling:
|
||||
"""Test AWS credential passing."""
|
||||
|
||||
def test_create_client_with_explicit_credentials(self, tmp_path):
|
||||
"""Test that credentials can be passed directly to create_client."""
|
||||
# This test verifies the API accepts credentials, not that they work
|
||||
# (we'd need a real S3 or LocalStack for that)
|
||||
client = create_client(
|
||||
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
region_name="us-west-2",
|
||||
cache_dir=str(tmp_path / "cache"),
|
||||
)
|
||||
|
||||
# Verify the client was created
|
||||
assert client is not None
|
||||
assert client.service is not None
|
||||
|
||||
# Verify credentials were passed to the storage adapter's boto3 client
|
||||
# The storage adapter should have a client with these credentials
|
||||
storage = client.service.storage
|
||||
assert hasattr(storage, "client")
|
||||
|
||||
# Check that the boto3 client was configured with our credentials
|
||||
# Note: boto3 doesn't expose credentials directly, but we can verify
|
||||
# the client was created (if credentials were invalid, this would fail)
|
||||
assert storage.client is not None
|
||||
|
||||
def test_create_client_with_session_token(self, tmp_path):
|
||||
"""Test passing temporary credentials with session token."""
|
||||
client = create_client(
|
||||
aws_access_key_id="ASIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
aws_session_token="FwoGZXIvYXdzEBEaDH...",
|
||||
cache_dir=str(tmp_path / "cache"),
|
||||
)
|
||||
|
||||
assert client is not None
|
||||
assert client.service.storage.client is not None
|
||||
|
||||
def test_create_client_without_credentials_uses_environment(self, tmp_path):
|
||||
"""Test that omitting credentials falls back to environment/IAM."""
|
||||
# This should use boto3's default credential chain
|
||||
client = create_client(cache_dir=str(tmp_path / "cache"))
|
||||
|
||||
assert client is not None
|
||||
assert client.service.storage.client is not None
|
||||
|
||||
def test_create_client_with_endpoint_and_credentials(self, tmp_path):
|
||||
"""Test passing both endpoint URL and credentials."""
|
||||
client = create_client(
|
||||
endpoint_url="http://localhost:9000",
|
||||
aws_access_key_id="minioadmin",
|
||||
aws_secret_access_key="minioadmin",
|
||||
cache_dir=str(tmp_path / "cache"),
|
||||
)
|
||||
|
||||
assert client is not None
|
||||
# Endpoint should be available
|
||||
assert client.endpoint_url == "http://localhost:9000"
|
||||
|
||||
|
||||
class TestBoto3Compatibility:
|
||||
"""Test boto3-compatible methods."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user