from os import path import logging from celery import Celery from raven import Client from raven.contrib.celery import register_signal from raven.contrib.celery import register_logger_signal from app import celeryconfig from app import config from app.logic import converter celery = Celery('tasks', backend=config.celery_results_db, broker=config.celery_broker) celery.config_from_object(celeryconfig) if config.raven_dsn: client = Client(dsn=config.raven_dsn) # register a custom filter to filter out duplicate logs register_logger_signal(client) # hook into the Celery error handler register_signal(client) # The register_logger_signal function can also take an optional argument # `loglevel` which is the level used for the handler created. # Defaults to `logging.ERROR` # TODO (OrCharles): have this be configurable? register_logger_signal(client, loglevel=logging.INFO) @celery.task(queue=config.broker_queue_name) def convert_json_to_xlsx(owner_type, owner_id, filename, xlsx_filename, template): """Conversion of JSON to XLSX """ bucket = config.bucket_name # since the files can be very large, store them on the filesystem # instead of in memory # build the temp filesystem paths xlsx_fs_folder = converter.build_fs_path(config.temp_dir, 'xlsx', owner_type, owner_id) json_fs_folder = converter.build_fs_path(config.temp_dir, 'json', owner_type, owner_id) # make sure the temp directories for this owner are created converter.makedirs(xlsx_fs_folder, exist_ok=True) converter.makedirs(json_fs_folder, exist_ok=True) # create the temporary json file to hold the s3 file contents full_path = path.join( converter.build_fs_path(config.temp_dir, 'json', owner_type, owner_id), filename) json_fp = open(full_path, mode='wb') # create the temporary xlsx file to hold the s3 file contents template_fp = converter.open_fp(config.temp_dir, 'xlsx', None, None, template, 'wb') # build the s3 folders template_s3_folder = converter.build_s3_path(config.prefix, 'xlsx', None, None) xlsx_s3_folder = converter.build_s3_path(config.prefix, 'xlsx', owner_type, owner_id) json_s3_folder = converter.build_s3_path(config.prefix, 'json', owner_type, owner_id) # get a connection to s3 s3 = converter.get_s3() # get the json file from s3 converter.get_file_from_s3(s3, bucket, json_s3_folder, filename, json_fp) # get the template file from s3 converter.get_file_from_s3(s3, bucket, template_s3_folder, template, template_fp) # reopen it as read json_fp.close() json_fp = open(full_path, mode='r', encoding='utf-8') # reopen the template file template_fp = converter.reopen_fp(config.temp_dir, 'xlsx', None, None, template, 'rb', template_fp) # create the xlsx file to be populated xlsx_fp = converter.open_fp(config.temp_dir, 'xlsx', owner_type, owner_id, xlsx_filename, 'wb') # process its contents to xlsx converter.convert_json_to_xlsx(json_fp, xlsx_fp, filename, template_fp) json_fp.close() # reopen the xlsx file as read xlsx_fp = converter.reopen_fp(config.temp_dir, 'xlsx', owner_type, owner_id, xlsx_filename, 'rb', xlsx_fp) # upload the json back to s3 converter.put_file_to_s3(s3, bucket, xlsx_s3_folder, xlsx_filename, xlsx_fp) return xlsx_s3_folder + '/' + xlsx_filename