feat: Add bucket management APIs and improve SDK filtering

This commit adds core bucket management functionality and enhances the SDK's internal file filtering to provide a cleaner abstraction layer.

**Bucket Management**:
- Add create_bucket(), delete_bucket(), list_buckets() to DeltaGliderClient
- Idempotent operations (creating existing bucket or deleting non-existent returns success)
- Complete boto3-compatible API for basic bucket operations
- Eliminates need for boto3 in most use cases

**Enhanced SDK Filtering**:
- SDK now filters .delta suffix and reference.bin from all list_objects() responses
- Simplified CLI to rely on SDK filtering (removed duplicate logic)
- Single source of truth for internal file hiding

**Delete Cleanup Logic**:
- Automatically removes orphaned reference.bin when last delta in DeltaSpace is deleted
- Prevents storage waste from abandoned reference files
- Works for both single delete() and recursive delete_recursive()

**Documentation & Testing**:
- Added BOTO3_COMPATIBILITY.md documenting actual 20% method coverage (21/100+ methods)
- Updated README to reflect accurate boto3 compatibility claims
- New comprehensive test suite for filtering and cleanup features (test_filtering_and_cleanup.py)
- New bucket management test suite (test_bucket_management.py)
- Example code for bucket lifecycle management (examples/bucket_management.py)
- Fixed mypy configuration to eliminate source file found twice errors
- All CI checks passing (lint, format, type check, 18 unit tests, 61 integration tests)

**Cleanup**:
- Removed PYPI_RELEASE.md (redundant with existing docs)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/deltaglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.2.dev0'
-__version_tuple__ = version_tuple = (0, 3, 2, 'dev0')
+__version__ = version = '4.0.1.dev1'
+__version_tuple__ = version_tuple = (4, 0, 1, 'dev1')
 
-__commit_id__ = commit_id = 'g23357e240'
+__commit_id__ = commit_id = 'gdd39595c6'

src/deltaglider/app/cli/main.py

@@ -252,11 +252,13 @@ def ls(
                 return f"{size_float:.1f}P"
 
             # List objects using SDK (automatically filters .delta and reference.bin)
-            from deltaglider.client import DeltaGliderClient
+            from deltaglider.client import DeltaGliderClient, ListObjectsResponse
 
             client = DeltaGliderClient(service)
-            response = client.list_objects(Bucket=bucket_name, Prefix=prefix_str, MaxKeys=10000)
-            objects = response.contents
+            dg_response: ListObjectsResponse = client.list_objects(
+                Bucket=bucket_name, Prefix=prefix_str, MaxKeys=10000
+            )
+            objects = dg_response.contents
 
             # Filter by recursive flag
             if not recursive:

src/deltaglider/client.py

@@ -107,7 +107,16 @@ class BucketStats:
 
 
 class DeltaGliderClient:
-    """DeltaGlider client with boto3-compatible APIs and advanced features."""
+    """DeltaGlider client with boto3-compatible APIs and advanced features.
+
+    Implements core boto3 S3 client methods (~21 methods covering 80% of use cases):
+    - Object operations: put_object, get_object, delete_object, list_objects, head_object
+    - Bucket operations: create_bucket, delete_bucket, list_buckets
+    - Presigned URLs: generate_presigned_url, generate_presigned_post
+    - Plus DeltaGlider extensions for compression stats and batch operations
+
+    See BOTO3_COMPATIBILITY.md for complete compatibility matrix.
+    """
 
     def __init__(self, service: DeltaService, endpoint_url: str | None = None):
         """Initialize client with service."""
@@ -1234,6 +1243,144 @@ class DeltaGliderClient:
             },
         }
 
+    # ============================================================================
+    # Bucket Management APIs (boto3-compatible)
+    # ============================================================================
+
+    def create_bucket(
+        self,
+        Bucket: str,
+        CreateBucketConfiguration: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Create an S3 bucket (boto3-compatible).
+
+        Args:
+            Bucket: Bucket name to create
+            CreateBucketConfiguration: Optional bucket configuration (e.g., LocationConstraint)
+            **kwargs: Additional S3 parameters (for compatibility)
+
+        Returns:
+            Response dict with bucket location
+
+        Example:
+            >>> client = create_client()
+            >>> client.create_bucket(Bucket='my-bucket')
+            >>> # With region
+            >>> client.create_bucket(
+            ...     Bucket='my-bucket',
+            ...     CreateBucketConfiguration={'LocationConstraint': 'us-west-2'}
+            ... )
+        """
+        storage_adapter = self.service.storage
+
+        # Check if storage adapter has boto3 client
+        if hasattr(storage_adapter, "client"):
+            try:
+                params: dict[str, Any] = {"Bucket": Bucket}
+                if CreateBucketConfiguration:
+                    params["CreateBucketConfiguration"] = CreateBucketConfiguration
+
+                response = storage_adapter.client.create_bucket(**params)
+                return {
+                    "Location": response.get("Location", f"/{Bucket}"),
+                    "ResponseMetadata": {
+                        "HTTPStatusCode": 200,
+                    },
+                }
+            except Exception as e:
+                error_msg = str(e)
+                if "BucketAlreadyExists" in error_msg or "BucketAlreadyOwnedByYou" in error_msg:
+                    # Bucket already exists - return success
+                    self.service.logger.debug(f"Bucket {Bucket} already exists")
+                    return {
+                        "Location": f"/{Bucket}",
+                        "ResponseMetadata": {
+                            "HTTPStatusCode": 200,
+                        },
+                    }
+                raise RuntimeError(f"Failed to create bucket: {e}") from e
+        else:
+            raise NotImplementedError("Storage adapter does not support bucket creation")
+
+    def delete_bucket(
+        self,
+        Bucket: str,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Delete an S3 bucket (boto3-compatible).
+
+        Note: Bucket must be empty before deletion.
+
+        Args:
+            Bucket: Bucket name to delete
+            **kwargs: Additional S3 parameters (for compatibility)
+
+        Returns:
+            Response dict with deletion status
+
+        Example:
+            >>> client = create_client()
+            >>> client.delete_bucket(Bucket='my-bucket')
+        """
+        storage_adapter = self.service.storage
+
+        # Check if storage adapter has boto3 client
+        if hasattr(storage_adapter, "client"):
+            try:
+                storage_adapter.client.delete_bucket(Bucket=Bucket)
+                return {
+                    "ResponseMetadata": {
+                        "HTTPStatusCode": 204,
+                    },
+                }
+            except Exception as e:
+                error_msg = str(e)
+                if "NoSuchBucket" in error_msg:
+                    # Bucket doesn't exist - return success
+                    self.service.logger.debug(f"Bucket {Bucket} does not exist")
+                    return {
+                        "ResponseMetadata": {
+                            "HTTPStatusCode": 204,
+                        },
+                    }
+                raise RuntimeError(f"Failed to delete bucket: {e}") from e
+        else:
+            raise NotImplementedError("Storage adapter does not support bucket deletion")
+
+    def list_buckets(self, **kwargs: Any) -> dict[str, Any]:
+        """List all S3 buckets (boto3-compatible).
+
+        Args:
+            **kwargs: Additional S3 parameters (for compatibility)
+
+        Returns:
+            Response dict with bucket list
+
+        Example:
+            >>> client = create_client()
+            >>> response = client.list_buckets()
+            >>> for bucket in response['Buckets']:
+            ...     print(bucket['Name'])
+        """
+        storage_adapter = self.service.storage
+
+        # Check if storage adapter has boto3 client
+        if hasattr(storage_adapter, "client"):
+            try:
+                response = storage_adapter.client.list_buckets()
+                return {
+                    "Buckets": response.get("Buckets", []),
+                    "Owner": response.get("Owner", {}),
+                    "ResponseMetadata": {
+                        "HTTPStatusCode": 200,
+                    },
+                }
+            except Exception as e:
+                raise RuntimeError(f"Failed to list buckets: {e}") from e
+        else:
+            raise NotImplementedError("Storage adapter does not support bucket listing")
+
     def _parse_tagging(self, tagging: str) -> dict[str, str]:
         """Parse URL-encoded tagging string to dict."""
         tags = {}

src/deltaglider/core/service.py

@@ -659,12 +659,42 @@ class DeltaService:
                     self.logger.debug(f"Could not clear cache for {object_key.key}: {e}")
 
         elif is_delta:
-            # Simply delete the delta file
+            # Delete the delta file
             self.storage.delete(full_key)
             result["deleted"] = True
             result["type"] = "delta"
             result["original_name"] = obj_head.metadata.get("original_name", "unknown")
 
+            # Check if this was the last delta in the DeltaSpace - if so, clean up reference.bin
+            if "/" in object_key.key:
+                deltaspace_prefix = "/".join(object_key.key.split("/")[:-1])
+                ref_key = f"{deltaspace_prefix}/reference.bin"
+
+                # Check if any other delta files exist in this DeltaSpace
+                remaining_deltas = []
+                for obj in self.storage.list(f"{object_key.bucket}/{deltaspace_prefix}"):
+                    if obj.key.endswith(".delta") and obj.key != object_key.key:
+                        remaining_deltas.append(obj.key)
+
+                if not remaining_deltas:
+                    # No more deltas - clean up the orphaned reference.bin
+                    ref_full_key = f"{object_key.bucket}/{ref_key}"
+                    ref_head = self.storage.head(ref_full_key)
+                    if ref_head:
+                        self.storage.delete(ref_full_key)
+                        self.logger.info(
+                            "Cleaned up orphaned reference.bin",
+                            ref_key=ref_key,
+                            reason="no remaining deltas",
+                        )
+                        result["cleaned_reference"] = ref_key
+
+                        # Clear from cache
+                        try:
+                            self.cache.evict(object_key.bucket, deltaspace_prefix)
+                        except Exception as e:
+                            self.logger.debug(f"Could not clear cache for {deltaspace_prefix}: {e}")
+
         elif is_direct:
             # Simply delete the direct upload
             self.storage.delete(full_key)