style: Format all Python files with ruff
- Formatted 103 files to comply with ruff format requirements - No code logic changes, only formatting/whitespace - Fixes CI formatting check failures
This commit is contained in:
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
BOTO3_AVAILABLE = True
|
||||
except ImportError:
|
||||
BOTO3_AVAILABLE = False
|
||||
@@ -63,33 +64,30 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if not BOTO3_AVAILABLE:
|
||||
raise ImportError(
|
||||
"boto3 is required for S3 storage. "
|
||||
"Install with: pip install boto3"
|
||||
)
|
||||
raise ImportError("boto3 is required for S3 storage. Install with: pip install boto3")
|
||||
|
||||
if 'bucket' not in kwargs:
|
||||
if "bucket" not in kwargs:
|
||||
raise ValueError("bucket parameter is required for S3 storage")
|
||||
|
||||
self.bucket = kwargs['bucket']
|
||||
self.region = kwargs.get('region', os.getenv('AWS_DEFAULT_REGION', 'us-east-1'))
|
||||
self.bucket = kwargs["bucket"]
|
||||
self.region = kwargs.get("region", os.getenv("AWS_DEFAULT_REGION", "us-east-1"))
|
||||
|
||||
# Initialize S3 client
|
||||
client_kwargs = {
|
||||
'region_name': self.region,
|
||||
"region_name": self.region,
|
||||
}
|
||||
|
||||
if 'endpoint_url' in kwargs:
|
||||
client_kwargs['endpoint_url'] = kwargs['endpoint_url']
|
||||
if "endpoint_url" in kwargs:
|
||||
client_kwargs["endpoint_url"] = kwargs["endpoint_url"]
|
||||
|
||||
if 'aws_access_key_id' in kwargs:
|
||||
client_kwargs['aws_access_key_id'] = kwargs['aws_access_key_id']
|
||||
if "aws_access_key_id" in kwargs:
|
||||
client_kwargs["aws_access_key_id"] = kwargs["aws_access_key_id"]
|
||||
|
||||
if 'aws_secret_access_key' in kwargs:
|
||||
client_kwargs['aws_secret_access_key'] = kwargs['aws_secret_access_key']
|
||||
if "aws_secret_access_key" in kwargs:
|
||||
client_kwargs["aws_secret_access_key"] = kwargs["aws_secret_access_key"]
|
||||
|
||||
self.s3_client = boto3.client('s3', **client_kwargs)
|
||||
self.s3_resource = boto3.resource('s3', **client_kwargs)
|
||||
self.s3_client = boto3.client("s3", **client_kwargs)
|
||||
self.s3_resource = boto3.resource("s3", **client_kwargs)
|
||||
|
||||
def upload_file(
|
||||
self, local_path: str, remote_path: str, metadata: dict[str, str] | None = None
|
||||
@@ -101,14 +99,14 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
|
||||
extra_args = {}
|
||||
if metadata:
|
||||
extra_args['Metadata'] = metadata
|
||||
extra_args["Metadata"] = metadata
|
||||
|
||||
try:
|
||||
self.s3_client.upload_file(
|
||||
str(local_file),
|
||||
self.bucket,
|
||||
remote_path,
|
||||
ExtraArgs=extra_args if extra_args else None
|
||||
ExtraArgs=extra_args if extra_args else None,
|
||||
)
|
||||
return f"s3://{self.bucket}/{remote_path}"
|
||||
except ClientError as e:
|
||||
@@ -120,50 +118,41 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
local_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
self.s3_client.download_file(
|
||||
self.bucket,
|
||||
remote_path,
|
||||
str(local_file)
|
||||
)
|
||||
self.s3_client.download_file(self.bucket, remote_path, str(local_file))
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
raise FileNotFoundError(f"Remote file not found: {remote_path}") from e
|
||||
raise Exception(f"S3 download failed: {e}") from e
|
||||
|
||||
def delete_file(self, remote_path: str) -> None:
|
||||
"""Delete file from S3."""
|
||||
try:
|
||||
self.s3_client.delete_object(
|
||||
Bucket=self.bucket,
|
||||
Key=remote_path
|
||||
)
|
||||
self.s3_client.delete_object(Bucket=self.bucket, Key=remote_path)
|
||||
except ClientError as e:
|
||||
raise Exception(f"S3 deletion failed: {e}") from e
|
||||
|
||||
def list_files(
|
||||
self, prefix: str = "", max_results: int = 1000
|
||||
) -> list[StorageObject]:
|
||||
def list_files(self, prefix: str = "", max_results: int = 1000) -> list[StorageObject]:
|
||||
"""List files in S3 bucket."""
|
||||
try:
|
||||
paginator = self.s3_client.get_paginator('list_objects_v2')
|
||||
paginator = self.s3_client.get_paginator("list_objects_v2")
|
||||
page_iterator = paginator.paginate(
|
||||
Bucket=self.bucket,
|
||||
Prefix=prefix,
|
||||
PaginationConfig={'MaxItems': max_results}
|
||||
Bucket=self.bucket, Prefix=prefix, PaginationConfig={"MaxItems": max_results}
|
||||
)
|
||||
|
||||
files = []
|
||||
for page in page_iterator:
|
||||
if 'Contents' not in page:
|
||||
if "Contents" not in page:
|
||||
continue
|
||||
|
||||
for obj in page['Contents']:
|
||||
files.append(StorageObject(
|
||||
key=obj['Key'],
|
||||
size=obj['Size'],
|
||||
last_modified=obj['LastModified'].isoformat(),
|
||||
etag=obj.get('ETag', '').strip('"')
|
||||
))
|
||||
for obj in page["Contents"]:
|
||||
files.append(
|
||||
StorageObject(
|
||||
key=obj["Key"],
|
||||
size=obj["Size"],
|
||||
last_modified=obj["LastModified"].isoformat(),
|
||||
etag=obj.get("ETag", "").strip('"'),
|
||||
)
|
||||
)
|
||||
|
||||
return files
|
||||
except ClientError as e:
|
||||
@@ -172,13 +161,10 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
def file_exists(self, remote_path: str) -> bool:
|
||||
"""Check if file exists in S3."""
|
||||
try:
|
||||
self.s3_client.head_object(
|
||||
Bucket=self.bucket,
|
||||
Key=remote_path
|
||||
)
|
||||
self.s3_client.head_object(Bucket=self.bucket, Key=remote_path)
|
||||
return True
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
return False
|
||||
raise Exception(f"S3 head_object failed: {e}") from e
|
||||
|
||||
@@ -186,12 +172,9 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
"""Generate presigned URL for S3 object."""
|
||||
try:
|
||||
url = self.s3_client.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={
|
||||
'Bucket': self.bucket,
|
||||
'Key': remote_path
|
||||
},
|
||||
ExpiresIn=expires_in
|
||||
"get_object",
|
||||
Params={"Bucket": self.bucket, "Key": remote_path},
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
return url
|
||||
except ClientError as e:
|
||||
@@ -200,16 +183,9 @@ class S3StorageAdaptor(BaseStorageAdaptor):
|
||||
def copy_file(self, source_path: str, dest_path: str) -> None:
|
||||
"""Copy file within S3 bucket (server-side copy)."""
|
||||
try:
|
||||
copy_source = {
|
||||
'Bucket': self.bucket,
|
||||
'Key': source_path
|
||||
}
|
||||
self.s3_client.copy_object(
|
||||
CopySource=copy_source,
|
||||
Bucket=self.bucket,
|
||||
Key=dest_path
|
||||
)
|
||||
copy_source = {"Bucket": self.bucket, "Key": source_path}
|
||||
self.s3_client.copy_object(CopySource=copy_source, Bucket=self.bucket, Key=dest_path)
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
raise FileNotFoundError(f"Source file not found: {source_path}") from e
|
||||
raise Exception(f"S3 copy failed: {e}") from e
|
||||
|
||||
Reference in New Issue
Block a user