"""Utils for checking and correcting wavforms.""" import os # basic os operations like paths import subprocess import tempfile import wave from io import BytesIO from awsretry import AWSRetry import boto3 from constants.lambda_const import sox_bin_location class WavFileCheckFailure(Exception): """Error for failed upload.""" pass class WavFileWriteFailure(Exception): """Error for failed upload.""" pass class WavFileOpenFailure(Exception): """Error for failed upload.""" pass class WavFileUploadFailure(Exception): """Error for failed upload.""" pass class WavFileFixFailure(Exception): """Error for failed upload.""" pass class WavFileConverted(Exception): """Error for failed upload.""" pass def validate_wav_key(bucket, key): """Validate that a key being passed is in fact a wav.""" key_path = bucket + '/' + key file_root, file_ext = os.path.splitext(key) if file_root[0] == '.': msg = '{}: Hidden file. Skipping'.format(key_path) print(msg) return False if file_ext != '.wav': msg = '{}: Not a WAV file.'.format(key_path) print(msg) return False return True # def convert_flac(bucket, key): # """Convert wav file to flac.""" # key_path = bucket + '/' + key # fix_output_file = BytesIO() # # # Log to console # msg = 'Flac file. Converting: {}'.format(key_path) # print(msg) # # # Open file # try: # # Open wave file # wav_file = get_wav_from_s3(bucket, key) # except Exception as e: # Open and convert fails # print(e)) # return # # # Transcode the file # subprocess.run( # [ # sox_bin_location, # '{}'.format(wav_file), # '-t', # 'wavpcm', # '{}'.format(fix_output_file) # ], check=True) # Flac Convert # wav_file = fix_output_file # # return fix_output_file @AWSRetry.backoff(added_exceptions=['Forbidden', '403']) def head_object_backoff(bucket_name, key, **kwargs): """HEAD an object with backoff.""" s3_client = boto3.client('s3') return s3_client.head_object(Bucket=bucket_name, Key=key, **kwargs) @AWSRetry.backoff(added_exceptions=['Forbidden', '403']) def get_object_backoff(bucket_name, key, **kwargs): """GET an object with backoff.""" s3_client = boto3.client('s3') return s3_client.get_object(Bucket=bucket_name, Key=key, **kwargs) def get_wav_from_s3(bucket, key): """Open a WAV file. Args: bucket: (str) The S3 bucket Name key: (str) The S3 object key """ if not validate_wav_key(bucket, key): raise WavFileCheckFailure('File does not end in `.wav`') fixed_file = False key_path = bucket + '/' + key solution = '' error_log = '' # Checking if wav converted head = head_object_backoff(bucket, key) metadata = head.get('Metadata') if not metadata: raise WavFileOpenFailure( 'Did not get file from S3. File has no metadata.') if metadata.get('converted'): raise WavFileConverted('File already converted.') print('Converting object to wav') file_byte_string = get_object_backoff(bucket, key)[ 'Body'].read() wav_bytes_file = BytesIO(file_byte_string) temp_wav_file = tempfile.NamedTemporaryFile(suffix='.wav') wav = None print('Opening WAV file.') try: # Open wave file wav = wave.open(wav_bytes_file, 'rb') print('File Opened.') # Writing temp wav_file try: with wave.open(temp_wav_file.name, 'wb') as wav_out: wav_out.setnchannels(wav.getnchannels()) wav_out.setsampwidth(wav.getsampwidth()) wav_out.setframerate(wav.getframerate()) wav_out.setnframes(wav.getnframes()) wav_out.writeframes(wav.readframes(wav.getnframes())) print('Temp WAV file written: {}'.format(temp_wav_file.name)) except Exception as e: print('Error writing out temp file: {}'.format(str(e))) except Exception as e: # Open and convert fails error_log = str(e) # Log msg = 'Exception: Could not open {} - {}'.format( wav_bytes_file, str(e)) print(msg) if not wav: print('Could not open file: {}'.format(key_path)) try: print('Attempting to fix file.') temp_wav_file, solution = fix_file( wav_bytes_file, error_log, bucket, key) print('Opening fixed file.') wav = wave.open(temp_wav_file.name) fixed_file = True except (WavFileWriteFailure, WavFileFixFailure): raise except Exception as e: print('Fix File error: {}'.format(str(e))) raise WavFileOpenFailure('File did not open.') # Get Wav Params wav_params = wav.getparams() # Print log to console logger_msg = 'track: {}| channels: {}| sampwidth: {}| framerate: {}| ' \ 'frames: {}| comptype: {}| compname: {}| solution: {}' \ .format(bucket + '/' + key, wav_params.nchannels, wav_params.sampwidth, wav_params.framerate, wav_params.nframes, wav_params.comptype, wav_params.compname, solution) print(logger_msg) # Return wav file return temp_wav_file, fixed_file, solution def save_wav_to_s3(wav_file, bucket, key, solution=None): """Save a Pillow image object to an S3 target from memory.""" if not solution: solution = 'via lambda' metadata = {} # file_name = os.path.split(key)[1] # buffer = BytesIO() # # try: # print('Saving: {} as {}'.format(file_name, key)) # with wave.open(buffer, 'wb') as wav_out: # wav_out.writeframes(wav_file) # # except Exception as e: # print('Save failed!') # print('Error - {} ({}):, {}'.format(type(e), e.args, str(e))) # raise WavFileWriteFailure( # 'Failed to write wav data to BytesIO buffer') # buffer.seek(0) # Rewind buffer for saving to S3 print('Getting Metadata from existing object') response = head_object_backoff(bucket, key) if response.get('Metadata'): metadata = response['Metadata'] metadata['converted'] = solution print('Puting {} to {}'.format(key, bucket)) s3_client = boto3.client('s3') sent_data = s3_client.put_object( Body=wav_file, Key=key, Bucket=bucket, ContentType='audio/wav', Metadata=metadata) if sent_data['ResponseMetadata']['HTTPStatusCode'] != 200: print('Upload failure!') raise WavFileUploadFailure( 'Failed to upload wav file {} to bucket {}'.format(key, bucket)) def fix_file(wav_bytes_file, error_msg, bucket, key): """Fix a wav file which won't open.""" key_path = bucket + '/' + key fix_output_file = tempfile.NamedTemporaryFile(suffix='.wav') temp_wav_file = tempfile.NamedTemporaryFile(suffix='.wav') fixed_file = False solution = '' print('Fixing file.') if 'file does not start with RIFF ID' in error_msg: msg = '{}: File is not a WAV.'.format(key_path) print(msg) raise WavFileOpenFailure('File does not start with a RIFF ID.') try: print('Writing Bytes to tempfile') with open(temp_wav_file.name, 'wb') as outfile: outfile.write(wav_bytes_file.getbuffer()) except Exception as e: msg = 'Error Writing fixed file: {}'.format(str(e)) print(msg) raise WavFileWriteFailure(msg) print('Input file: {}'.format(temp_wav_file.name)) print('Output file: {}'.format(fix_output_file.name)) # Convert with sox if 'unknown format: 65534' in error_msg: # Update user msg = '{}: Converting from WAVE_FORMAT_EXTENSIBLE to ' \ 'WAVE_FORMAT_PCM.'.format(key_path) print(msg) # Convert with sox try: subprocess.run( [ sox_bin_location, '{}'.format(temp_wav_file.name), '-t', 'wavpcm', '{}'.format(fix_output_file.name) ], check=True) # WAV_FORMAT_EXTENSIBLE except subprocess.CalledProcessError as e: msg = 'Exception: {}'.format(str(e)) print(msg) raise WavFileFixFailure('Failed to fix WAVE_FORMAT_EXTENSIBLE.') else: solution = 'Converted to WAV_FORMAT_PCM' fixed_file = True elif 'unknown format: 3' in error_msg: msg = '{}: Converting from 32-bit to 24 bit.'.format(key_path) print(msg) # Convert with sox try: subprocess.run( [ sox_bin_location, '{}'.format(temp_wav_file.name), '-b', '24', '-t', 'wavpcm', '{}'.format(fix_output_file.name) ], check=True) # Convert 32bit to 24bit except subprocess.CalledProcessError as e: msg = 'Exception: {}'.format(str(e)) print(msg) WavFileFixFailure('Failed to Convert from 32-bit to 24-bit.') else: solution = 'Converted to 24-bit' fixed_file = True if fixed_file: fix_output_file.seek(0) # wav_bytes_file = fix_output_file return fix_output_file, solution def check_wav(temp_wav_file, bucket, key): """Check wav files.""" key_path = bucket + '/' + key solution = '' fix_output_file = tempfile.NamedTemporaryFile(suffix='.wav') fixed_file = False print('Checking WAV file.') try: wav = wave.open(temp_wav_file.name, 'rb') except Exception as e: msg = 'Error opening temp file for checking: {}' print(msg.format(str(e))) raise wav_params = wav.getparams() # Bit depth > 24 bit if int(wav_params.sampwidth) > 3: msg = '{}: Converting from sampwidth {} to 3 (24-bit).'.format( key_path, wav_params.sampwidth) print(msg) # Convert with sox try: bit_depth_proc = subprocess.run( # noqa [ sox_bin_location, '{}'.format(temp_wav_file.name), '-b', '24', '-t', 'wavpcm', '{}'.format(fix_output_file.name) ], check=True) # Convert Sample Width to 24-bit except subprocess.CalledProcessError as e: msg = 'Exception: {}'.format(str(e)) print(msg) raise WavFileCheckFailure('Failed to convert sampwidth to 24bit.') temp_wav_file = fix_output_file solution = 'Converted to 24-bit' fixed_file = True # Bitrate > 48000 (96000, 192000, etc.) # if int(wav_params.framerate) > 48000: # REMOVED return temp_wav_file, fixed_file, solution