import os from datetime import datetime, timedelta from google.cloud.bigtable import Client project_id = os.environ.get("GCP_PROJECT", "delphi-prod") instance_id = os.environ.get("INSTANCE_ID", "delphi-consumer-analytics-hdd") cluster_id = os.environ.get("CLUSTER_ID", "stg-delphi-ca-hdd-us-east4-b") expiration_days = int(os.environ.get("EXPIRATION_DAYS", "1")) client = Client(project=project_id, admin=True) instance = client.instance(instance_id) def get_backup_id(table_name: str) -> str: """Helper function, adds current date as a suffix to a given string :param table_name: The name of the table :return: table name with a current date """ from time import gmtime, strftime return """{}-{}""".format(table_name, strftime("%m%d%Y", gmtime())) def get_expire_time() -> datetime: """Helper function, returns current date +{expiration_days} days :return: datetime object, the current date +{expiration_days} days """ date = datetime.now() date += timedelta(days=expiration_days) return date def main(event, context): """Background Cloud Function to be triggered by Pub/Sub. Args: event (dict): The dictionary with data specific to this type of event. The `data` field contains the PubsubMessage message. The `attributes` field will contain custom attributes if there are any. context (google.cloud.functions.Context): The Cloud Functions event metadata. The `event_id` field contains the Pub/Sub message ID. The `timestamp` field contains the publish time. """ print( """This Function was triggered by messageId {} published at {} """.format( context.event_id, context.timestamp ) ) import base64 import json if "data" in event: data = json.loads(base64.b64decode(event["data"]).decode("utf-8")) if "tables" in data: for table_id in data["tables"]: table = instance.table(table_id) if not table.exists(): raise Exception(f"Table {table_id} doesn't exists.") backup_id = get_backup_id(table_id) expire_time = get_expire_time() print(f"Backup for {table_id} has started.") backup = table.backup(backup_id, cluster_id, expire_time) backup.create() print(f"Backup has {table_id} finished.") return 0 else: raise Exception("expected list of tables not found in data") else: raise Exception("data is missing")