import os import boto3 from huggingface_hub import snapshot_download def upload_directory_to_s3(local_path, bucket_name, s3_prefix): s3 = boto3.client('s3') for root, dirs, files in os.walk(local_path): for file in files: local_file_path = os.path.join(root, file) # Create S3 key by joining prefix and relative path relative_path = os.path.relpath(local_file_path, local_path) s3_key = os.path.join(s3_prefix, relative_path).replace("\\", "/") print(f"Uploading {file} to s3://{bucket_name}/{s3_key}") s3.upload_file(local_file_path, bucket_name, s3_key) # 1. Download from Hugging Face model_id = "bert-base-uncased" local_dir = "./model_temp" snapshot_download( repo_id=model_id, local_dir=local_dir, local_dir_use_symlinks=False # Crucial for direct file access ) # 2. Upload to S3 upload_directory_to_s3(local_dir, "my-awesome-bucket", "models/bert-base") Use code with caution. Method 2: Using AWS SageMaker SDK
If you are already working within the AWS ecosystem, the SageMaker Python SDK provides a streamlined way to package and upload models. download huggingface model to s3
A Hugging Face User Access Token (if downloading private or gated models). Method 1: The Direct Python Script (Recommended) Method 2: Using AWS SageMaker SDK If you
The most flexible way to handle the transfer is using a Python script. This method downloads the files to a temporary local directory and immediately uploads them to S3 using boto3 . This method downloads the files to a temporary
huggingface-cli download bert-base-uncased --local-dir ./my-model Use code with caution. :
s3_path = sess.upload_data( path=local_dir, bucket="my-bucket", key_prefix="models/my-model" ) Use code with caution. Method 3: CLI One-Liners (Quick & Dirty)