emmi_data_management.cli_aws

Attributes

RICH_MARKUP_MODE

CLI_HELP_MARKDOWN

CTX

aws_s3_app

Functions

docs()

AWS S3 commands

aws_estimate([bucket, prefix, extension])

aws_file([bucket, key, local_dir])

aws_fetch([bucket, prefix, local_dir, extension, ...])

Module Contents

emmi_data_management.cli_aws.RICH_MARKUP_MODE: Literal['markdown', 'rich'] = 'rich'
emmi_data_management.cli_aws.CLI_HELP_MARKDOWN = Multiline-String
Show Value
"""
    AWS S3 commands

    **Examples**
    ```
    # estimate bytes under a prefix
    emmi-data aws estimate my-bucket data/prefix/ --extension .pt

    # download a single object
    emmi-data aws file my-bucket data/prefix/file.pt ./data

    # download a prefix (parallel) and write a provenance manifest with hashes
    emmi-data aws fetch my-bucket data/prefix/ ./data       --extension .pt       --jobs 16       --manifest-out ./s3-manifest.json       --hash --hash-jobs 8 --hash-chunk-mb 8
    ```
    """
emmi_data_management.cli_aws.CTX
emmi_data_management.cli_aws.aws_s3_app
emmi_data_management.cli_aws.docs()

AWS S3 commands

Examples

# estimate bytes under a prefix
emmi-data aws estimate my-bucket data/prefix/ --extension .pt

# download a single object
emmi-data aws file my-bucket data/prefix/file.pt ./data

# download a prefix (parallel) and write a provenance manifest with hashes
emmi-data aws fetch my-bucket data/prefix/ ./data          --extension .pt          --jobs 16          --manifest-out ./s3-manifest.json          --hash --hash-jobs 8 --hash-chunk-mb 8
Return type:

None

emmi_data_management.cli_aws.aws_estimate(bucket=typer.Argument(..., help='S3 bucket name'), prefix=typer.Argument(..., help='S3 key prefix'), extension=typer.Option(None, '--extension', '-e', help='Filter by extension'))
Parameters:

  • bucket (str)

  • prefix (str)

  • extension (str | None)

Return type:

None

emmi_data_management.cli_aws.aws_file(bucket=typer.Argument(..., help='S3 bucket'), key=typer.Argument(..., help='S3 object key'), local_dir=typer.Argument(..., dir_okay=True, file_okay=False, help='Destination directory'))
Parameters:
Return type:

None

emmi_data_management.cli_aws.aws_fetch(bucket=typer.Argument(..., help='S3 bucket'), prefix=typer.Argument(..., help='S3 key prefix'), local_dir=typer.Argument(..., dir_okay=True, file_okay=False, help='Destination directory'), extension=typer.Option(None, '--extension', '-e', help='Filter by extension'), jobs=typer.Option(8, '--jobs', '-j', help='Parallel downloads'), manifest_out=typer.Option(None, '--manifest-out', '-m', help='Write a provenance manifest JSON'), hash_after=typer.Option(True, '--hash/--no-hash', help='Compute SHA-256 for manifest entries'), hash_jobs=typer.Option(8, '--hash-jobs', help='Parallel hash workers'), hash_chunk_mb=typer.Option(8, '--hash-chunk-mb', help='Hash chunk size (MiB)'), dry_run=typer.Option(False, '--dry-run', help='Only report counts/size; do not download'), strip_prefix=typer.Option(False, '--strip-prefix/--no-strip-prefix', help='Save keys relative to the provided prefix'), flatten=typer.Option(False, '--flatten', help='Save by basename only; error on collisions'), strict_hash=typer.Option(False, '--strict-hash/--no-strict-hash', help='Exit non-zero if any hash fails'))
Parameters:
Return type:

None