feat: Enhance S3 migration CLI with new commands and EC2 detection option

This commit is contained in:
Simone Scarduzio
2025-10-12 23:12:32 +02:00
parent b2ca59490b
commit aea5cb5d9a
5 changed files with 54 additions and 30 deletions

View File

@@ -283,12 +283,14 @@ def copy_s3_to_s3(
tmp_path = Path(tmp.name)
# Write stream to temp file
with open(tmp_path, 'wb') as f:
with open(tmp_path, "wb") as f:
shutil.copyfileobj(source_stream, f)
try:
# Use DeltaService.put() with override_name to preserve original filename
summary = service.put(tmp_path, dest_deltaspace, max_ratio, override_name=original_filename)
summary = service.put(
tmp_path, dest_deltaspace, max_ratio, override_name=original_filename
)
if not quiet:
if summary.delta_size:
@@ -371,7 +373,9 @@ def migrate_s3_to_s3(
click.echo(f"Migrating from s3://{source_bucket}/{source_prefix}")
click.echo(f" to s3://{dest_bucket}/{effective_dest_prefix}")
else:
click.echo(f"Migrating from s3://{source_bucket}/{source_prefix} to s3://{dest_bucket}/{dest_prefix}")
click.echo(
f"Migrating from s3://{source_bucket}/{source_prefix} to s3://{dest_bucket}/{dest_prefix}"
)
click.echo("Scanning source and destination buckets...")
# List source objects
@@ -396,7 +400,9 @@ def migrate_s3_to_s3(
source_objects.append(obj)
# List destination objects to detect what needs copying
dest_list_prefix = f"{dest_bucket}/{effective_dest_prefix}" if effective_dest_prefix else dest_bucket
dest_list_prefix = (
f"{dest_bucket}/{effective_dest_prefix}" if effective_dest_prefix else dest_bucket
)
dest_keys = set()
for obj in service.storage.list(dest_list_prefix):
@@ -429,6 +435,7 @@ def migrate_s3_to_s3(
return
if not quiet:
def format_bytes(size: int) -> str:
size_float = float(size)
for unit in ["B", "KB", "MB", "GB", "TB"]:
@@ -487,7 +494,7 @@ def migrate_s3_to_s3(
dest_s3_url,
quiet=True,
max_ratio=max_ratio,
no_delta=no_delta
no_delta=no_delta,
)
successful += 1
@@ -517,10 +524,13 @@ def migrate_s3_to_s3(
if successful > 0 and not no_delta:
try:
from ...client import DeltaGliderClient
client = DeltaGliderClient(service)
dest_stats = client.get_bucket_stats(dest_bucket, detailed_stats=False)
if dest_stats.delta_objects > 0:
click.echo(f"\nCompression achieved: {dest_stats.average_compression_ratio:.1%}")
click.echo(
f"\nCompression achieved: {dest_stats.average_compression_ratio:.1%}"
)
click.echo(f"Space saved: {format_bytes(dest_stats.space_saved)}")
except Exception:
pass # Ignore stats errors

View File

@@ -129,7 +129,14 @@ def _version_callback(ctx: click.Context, param: click.Parameter, value: bool) -
@click.group()
@click.option("--debug", is_flag=True, help="Enable debug logging")
@click.option("--version", is_flag=True, is_eager=True, expose_value=False, callback=_version_callback, help="Show version and exit")
@click.option(
"--version",
is_flag=True,
is_eager=True,
expose_value=False,
callback=_version_callback,
help="Show version and exit",
)
@click.pass_context
def cli(ctx: click.Context, debug: bool) -> None:
"""DeltaGlider - Delta-aware S3 file storage wrapper."""
@@ -662,7 +669,9 @@ def verify(service: DeltaService, s3_url: str) -> None:
@click.option("--max-ratio", type=float, help="Max delta/file ratio (default: 0.5)")
@click.option("--dry-run", is_flag=True, help="Show what would be migrated without migrating")
@click.option("--yes", "-y", is_flag=True, help="Skip confirmation prompt")
@click.option("--no-preserve-prefix", is_flag=True, help="Don't preserve source prefix in destination")
@click.option(
"--no-preserve-prefix", is_flag=True, help="Don't preserve source prefix in destination"
)
@click.option("--endpoint-url", help="Override S3 endpoint URL")
@click.option("--region", help="AWS region")
@click.option("--profile", help="AWS profile to use")