import re from logging import Logger from typing import List, Tuple, Optional from botocore.client import BaseClient class KeyMatcher: def match(self, key) -> bool: raise NotImplementedError class RegexKeyMatcher(KeyMatcher): def __init__(self, pattern: str) -> None: super().__init__() self.__regex = re.compile(pattern) def match(self, key) -> bool: if self.__regex.fullmatch(key): return True return False class Bucket: def __init__(self, logger: Logger, s3_client: BaseClient, name: str) -> None: super().__init__() self.__logger = logger self.__s3_client = s3_client self.name = name def find_all(self, prefix: str, matcher: KeyMatcher = None) -> List[str]: paginator = self.__s3_client.get_paginator('list_objects') keys = [] for page in paginator.paginate(Bucket=self.name, Prefix=prefix): if 'Contents' not in page: continue for obj in page['Contents']: if not matcher or matcher.match(obj['Key']): keys.append(obj['Key']) if not keys: self.__logger.warning('No keys found in bucket `%s` with prefix `%s`', self.name, prefix) return keys def copy(self, key: str, dst_bucket: str, dst_key: str): self.__logger.debug('Copying file `s3://%s/%s` -> `s3://%s/%s`', self.name, key, dst_bucket, dst_key) self.__s3_client.copy(CopySource={'Bucket': self.name, 'Key': key}, Bucket=dst_bucket, Key=dst_key) self.__logger.debug('Object copying complete `s3://%s/%s` -> `s3://%s/%s`', self.name, key, dst_bucket, dst_key) def size(self, key: str): return int(self.__s3_client.head_object(Bucket=self.name, Key=key)['ContentLength']) def read(self, key: str, byte_range: Optional[Tuple[int, int]] = None) -> bytes: kw = { 'Bucket': self.name, 'Key': key, } if byte_range: kw['Range'] = f'bytes={byte_range[0]}-{byte_range[1] - 1}' return self.__s3_client.get_object(**kw)['Body'].read() def add_tags(self, key: str, **tags): tagging = self.__s3_client.get_object_tagging(Bucket=self.name, Key=key) tags_list = [x for x in tagging['TagSet'] if x['Key'] not in tags.keys()] for tag in tags.keys(): tags_list.append({ 'Key': tag, 'Value': tags[tag] }) self.__s3_client.put_object_tagging( Bucket=self.name, Key=key, Tagging={ 'TagSet': tags_list } ) def get_tag(self, key: str, tag_key: str) -> Optional[str]: tagging = self.__s3_client.get_object_tagging(Bucket=self.name, Key=key) tags = [x for x in tagging['TagSet'] if x['Key'] == tag_key] if len(tags) == 0: self.__logger.warning('Requested tag with key `%s` does not exist for object %s.', tag_key, key) return None if len(tags) > 1: self.__logger.warning('More than one tag with key `%s` found', tag_key) return tags[0]['Value']