"""Common functions regarding file open, read, write.""" import codecs from contextlib import contextmanager import os import subprocess from requests.packages import chardet THIS_DIR = os.path.dirname(os.path.abspath(__file__)) def remove_bom(file_source_path, file_destination_path): """Remove BOM from CSV file. Args: file_source_path (str): Source tmp location. file_destination_path (str): Destination tmp location. """ file_source_path = os.path.realpath(os.path.expanduser(file_source_path)) # Read from file # @todo for now we are not reading file in chunk # as vudu files are of some kb in size file_open = open(file_source_path, mode='r+b') raw = file_open.read() file_open.close() # Decode raw = raw.decode(chardet.detect(raw)['encoding']) # Encode to UTF-8 raw = raw.encode('utf8') # Remove BOM if raw.startswith(codecs.BOM_UTF8): raw = raw.lstrip(codecs.BOM_UTF8) # Write to file file_open = open(file_destination_path, 'wb') file_open.write(raw) file_open.close() def remove_footer(destination_tmp_path, number_of_line, footer_with_text=None): """Remove specified number of line from file footer. Args: destination_tmp_path (str): Destination tmp location. number_of_line(int): Number of lines to be removed. footer_with_text (str): String to match footer to be sure we remove only footer if it exist. By default if no footer_with_text, it will remove last number_of_line. """ file_open = open(destination_tmp_path, mode='r') raw = file_open.readlines() file_open.close() if footer_with_text and raw[-number_of_line].find(footer_with_text) == -1: return file_open = open(destination_tmp_path, mode='w') raw = raw[:-number_of_line] file_open.writelines(raw) file_open.close() @contextmanager def open_with_encodings(filename, encodings=None): """Get file object with a best guess codec. This function attempts the fallback codecs, and creates a new file object with that encoding. This is not fool-proof, and is just a good enough guess that essentially suppresses the UnicodeDecodeError. The data can still be incorrect! Also, for big files this can be slow. It is best if you knew the encoding before hand to avoid using this. Args: filename (str): Filename to open under different encodings. encodings (list): Encoding names to try in order. Yields: file object: File object opened with a proper encoding. Raises: UnicodeError: If none of the fallback codecs worked. """ if encodings is None: encodings = [] encodings.insert(0, 'utf-8') for encoding in encodings: try: fh = open(filename, encoding=encoding) fh.read() fh.seek(0) yield fh fh.close() return except: # noqa fh.close() raise UnicodeError def split_file(source_file, destination_path, chunk_size='200m', cut_header_lines=0, archiver='gzip', compression_level=5): """Split file into chunks.""" args = [ 'bash', f'{THIS_DIR}/split_file.sh', '-s', source_file, '-t', destination_path, '-z', chunk_size, '-c', str(compression_level), '-d', str(cut_header_lines) if cut_header_lines else '0', '-a', archiver, ] completed_process = subprocess.run( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) returncode = completed_process.returncode return { 'result': returncode == 0, 'return_code': returncode, 'stderr': completed_process.stderr, 'stdout': completed_process.stdout, } def upload_path_to_s3(path, destination, num_threads=20): """Upload all files in dir to S3 recursively in parallel.""" args = [ 'bash', f'{THIS_DIR}/s3_upload.sh', '-s', path, '-t', destination, '-n', str(num_threads), ] completed_process = subprocess.run( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=os.environ) returncode = completed_process.returncode return { 'result': returncode == 0, 'return_code': returncode, 'stderr': completed_process.stderr, 'stdout': completed_process.stdout, }