"""Frame-by-frame video processing for smooth crop interpolation. This module processes videos frame-by-frame to avoid FFmpeg expression bugs and provide truly smooth crop transitions using scipy interpolation. """ import gc import subprocess import sys from pathlib import Path from typing import TYPE_CHECKING import cv2 import numpy as np from scipy.interpolate import interp1d from .config import settings from .ffmpeg_utils import get_cv2_rotation_code from .models import CropKeyframe, CropPath if TYPE_CHECKING: from .captioning import CaptionRenderer class CropInterpolator: """Linear interpolation of crop positions with scene-boundary support. Uses linear interpolation between keyframes. For keyframes with scene_change=True, creates separate interpolators for each scene segment to ensure instant cuts at scene boundaries. """ def __init__( self, keyframes: list[CropKeyframe], segment_start: float, ): """Initialize interpolator. Args: keyframes: List of crop keyframes (must have at least 1) segment_start: Start time of the segment in seconds """ if len(keyframes) < 1: raise ValueError("Need at least 1 keyframe for interpolation") self.keyframes = keyframes self.crop_width = keyframes[0].width self.crop_height = keyframes[0].height # Single keyframe = constant position if len(keyframes) == 1: self.x_interp = lambda t: keyframes[0].x self.y_interp = lambda t: keyframes[0].y self.interpolation_method = "constant" self.interpolators = None # No segments needed return # Split keyframes into segments at scene boundaries self.segments = self._split_into_segments(keyframes) # Create interpolator for each segment self.interpolators = [] for segment_kfs in self.segments: times = np.array([kf.time for kf in segment_kfs]) x_positions = np.array([kf.x for kf in segment_kfs]) y_positions = np.array([kf.y for kf in segment_kfs]) # Linear interpolation (suitable for scene_equilibrium static crops) kind = "linear" # Create segment interpolator if len(segment_kfs) == 1: # Single keyframe in segment = constant # Use closure to properly capture the value def make_const_interp(val): return lambda t: float(val) x_interp = make_const_interp(x_positions[0]) y_interp = make_const_interp(y_positions[0]) else: x_interp = interp1d( times, x_positions, kind=kind, fill_value="extrapolate", assume_sorted=True, bounds_error=False, ) y_interp = interp1d( times, y_positions, kind=kind, fill_value="extrapolate", assume_sorted=True, bounds_error=False, ) self.interpolators.append({ 'time_start': times[0], 'time_end': times[-1], 'x_interp': x_interp, 'y_interp': y_interp, }) # Set interpolation method for reporting self.interpolation_method = "linear" if self.interpolators else "constant" def _split_into_segments(self, keyframes: list[CropKeyframe]) -> list[list[CropKeyframe]]: """Split keyframes into segments at scene_change boundaries. Args: keyframes: List of crop keyframes Returns: List of keyframe segments, where each segment is a list of keyframes belonging to the same scene. """ if not keyframes: return [] segments = [] current_segment = [] for kf in keyframes: if kf.scene_change and current_segment: # End current segment segments.append(current_segment) # Start new segment with this keyframe current_segment = [kf] else: current_segment.append(kf) if current_segment: segments.append(current_segment) return segments def get_position(self, time: float, frame_width: int, frame_height: int) -> tuple[int, int]: """Get interpolated crop position at given time. For scene_change keyframes, returns position from the appropriate segment interpolator, ensuring instant cuts at scene boundaries. Args: time: Time in seconds frame_width: Width of the video frame (for boundary checking) frame_height: Height of the video frame (for boundary checking) Returns: Tuple of (x, y) crop position, clipped to valid bounds """ # If no segments (single keyframe case), use simple interpolation if self.interpolators is None: x = float(self.keyframes[0].x) y = float(self.keyframes[0].y) else: # Find which segment this time falls into # Use tolerance for floating point comparison x, y = None, None tolerance = 0.001 # 1ms tolerance for floating point errors for segment in self.interpolators: # Only apply tolerance to start time to avoid segment overlap at boundaries # This ensures frames at exact scene boundaries use the NEW scene's segment if segment['time_start'] - tolerance <= time <= segment['time_end']: try: x = float(segment['x_interp'](time)) y = float(segment['y_interp'](time)) break except Exception as e: # If interpolation fails, use segment start position print(f"Warning: Interpolation failed at t={time:.2f}s: {e}", file=sys.stderr) # Find the keyframe at segment start for kf in self.keyframes: if abs(kf.time - segment['time_start']) < tolerance: x, y = float(kf.x), float(kf.y) break if x is not None: break # Fallback: use nearest keyframe (handles edge cases) if x is None or y is None: closest_kf = min(self.keyframes, key=lambda kf: abs(kf.time - time)) x, y = float(closest_kf.x), float(closest_kf.y) # Clip to valid bounds (ensure crop doesn't go out of frame) x = int(np.clip(x, 0, frame_width - self.crop_width)) y = int(np.clip(y, 0, frame_height - self.crop_height)) return x, y def process_segment_frame_by_frame( input_path: Path, output_path: Path, crop_path: CropPath, caption_renderer: "CaptionRenderer | None" = None, audio_source_path: Path | None = None, audio_start: float = 0.0, audio_duration: float | None = None, ) -> Path: """Process video segment frame-by-frame with smooth crop interpolation. This avoids FFmpeg expression bugs by processing in Python with scipy interpolation. This is the same approach used by professional tools like Google's AutoFlip. Args: input_path: Path to input video file (visual source) output_path: Path for output video file crop_path: CropPath with segment and keyframes audio_source_path: Optional separate audio source (e.g. main video for UGC clips). When set, audio is drawn from this file at [audio_start, audio_start+duration] instead of from input_path. audio_start: Start time in audio_source_path to extract audio from (seconds). audio_duration: How many seconds of audio to use (None = use clip duration). Returns: Path to the generated output file Raises: RuntimeError: If video processing fails ValueError: If crop parameters are invalid """ print(f"\n{'='*60}", file=sys.stderr) print(f"FRAME-BY-FRAME PROCESSING: {len(crop_path.keyframes)} keyframes", file=sys.stderr) print(f"{'='*60}\n", file=sys.stderr) # Open video cap = cv2.VideoCapture(str(input_path)) if not cap.isOpened(): raise RuntimeError(f"Failed to open video: {input_path}") # Get video properties fps = cap.get(cv2.CAP_PROP_FPS) width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # Correct for rotation metadata (e.g. iPhone portrait videos stored as landscape) _rotate_code = get_cv2_rotation_code(input_path) if _rotate_code in (cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE): width, height = height, width print(f"Video: {width}x{height} @ {fps:.2f} fps ({total_frames} frames)", file=sys.stderr) # Seek to segment start start_frame = round(crop_path.segment.start * fps) cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) # Calculate total frames to process duration = crop_path.segment.end - crop_path.segment.start num_frames = int(duration * fps) print(f"Segment: {crop_path.segment.start:.2f}s - {crop_path.segment.end:.2f}s " f"({duration:.2f}s, {num_frames} frames)", file=sys.stderr) # Create interpolator interpolator = CropInterpolator(crop_path.keyframes, crop_path.segment.start) # Get crop dimensions crop_width = interpolator.crop_width crop_height = interpolator.crop_height print(f"Crop: {crop_width}x{crop_height}", file=sys.stderr) print(f"Interpolation method: {interpolator.interpolation_method}", file=sys.stderr) # Pipe cropped frames directly to FFmpeg — avoids intermediate temp file # and the mp4v → libx264 double-encode. clip_duration = audio_duration if audio_duration is not None else duration if audio_source_path is not None: # Use a separate audio source (e.g. main video for UGC clips) audio_file = str(audio_source_path) audio_seek = str(audio_start) else: # Default: audio from the same input video at segment timestamps audio_file = str(input_path) audio_seek = str(crop_path.segment.start) ffmpeg_cmd = [ str(settings.ffmpeg_path), # Raw BGR24 frames from stdin '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', f'{crop_width}x{crop_height}', '-r', str(fps), '-i', 'pipe:0', # Audio source (seek before -i for accuracy) '-ss', audio_seek, '-t', str(clip_duration), '-i', audio_file, # Stream mapping '-map', '0:v:0', '-map', '1:a:0?', # Encoding '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', '-c:a', 'aac', '-b:a', '192k', '-y', str(output_path), ] ffmpeg_proc = subprocess.Popen( ffmpeg_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, ) print(f"\nProcessing frames...", file=sys.stderr) frames_processed = 0 frames_failed = 0 try: for frame_idx in range(num_frames): ret, frame = cap.read() if not ret: frames_failed += 1 print(f"Warning: Failed to read frame {frame_idx}", file=sys.stderr) break if _rotate_code is not None: frame = cv2.rotate(frame, _rotate_code) # Calculate current time current_time = crop_path.segment.start + (frame_idx / fps) # Get interpolated crop position crop_x, crop_y = interpolator.get_position(current_time, width, height) # Crop frame cropped = frame[crop_y:crop_y+crop_height, crop_x:crop_x+crop_width] # Verify crop size (edge case: video might be smaller than expected) if cropped.shape[0] != crop_height or cropped.shape[1] != crop_width: print(f"Warning: Frame {frame_idx} crop size mismatch: " f"expected {crop_width}x{crop_height}, got {cropped.shape[1]}x{cropped.shape[0]}", file=sys.stderr) # Skip malformed frame continue # Apply caption overlay (no-op when caption_renderer is None) if caption_renderer is not None: cropped = caption_renderer.render(cropped) # Send raw frame bytes to FFmpeg stdin try: ffmpeg_proc.stdin.write(cropped.tobytes()) except BrokenPipeError: print("Warning: FFmpeg pipe closed early — check FFmpeg stderr", file=sys.stderr) break frames_processed += 1 # Explicit cleanup every 50 frames to prevent memory leaks if frame_idx % 50 == 0 and frame_idx > 0: del frame, cropped gc.collect() # Progress indicator (every 10%) if frame_idx % max(1, num_frames // 10) == 0: progress = (frame_idx / num_frames) * 100 print(f" Progress: {progress:.0f}% ({frame_idx}/{num_frames} frames)", file=sys.stderr) print(f" Progress: 100% ({frames_processed}/{num_frames} frames)", file=sys.stderr) if frames_failed > 0: print(f"Warning: {frames_failed} frames failed to process", file=sys.stderr) finally: cap.release() # Signal EOF to FFmpeg. If FFmpeg already closed its read end (normal after # receiving all frames), Python raises ValueError("flush of closed file") or # BrokenPipeError on the implicit flush inside close() — both are safe to ignore. try: ffmpeg_proc.stdin.close() except (BrokenPipeError, ValueError, OSError): pass # Prevent communicate() from trying to flush the now-closed stdin file object. ffmpeg_proc.stdin = None _, ffmpeg_stderr = ffmpeg_proc.communicate() if ffmpeg_proc.returncode != 0: print(f"FFmpeg error:\n{ffmpeg_stderr.decode()}", file=sys.stderr) raise subprocess.CalledProcessError( ffmpeg_proc.returncode, ffmpeg_cmd, None, ffmpeg_stderr.decode() ) print(f"✓ Frame-by-frame processing complete\n", file=sys.stderr) return output_path def process_segment_with_overlay( input_path: Path, output_path: Path, crop_path: CropPath, ) -> Path: """Process video segment with crop rectangle overlay (no actual cropping). Generates a debug video showing the full original frame with rectangles indicating where the crop would be applied. Uses the same interpolated positions as actual cropping for accurate visualization. Args: input_path: Path to input video file output_path: Path for output video file crop_path: CropPath with segment and keyframes Returns: Path to the generated output file Raises: RuntimeError: If video processing fails """ print(f"\n{'='*60}", file=sys.stderr) print(f"DEBUG OVERLAY: {len(crop_path.keyframes)} keyframes", file=sys.stderr) print(f"{'='*60}\n", file=sys.stderr) cap = cv2.VideoCapture(str(input_path)) if not cap.isOpened(): raise RuntimeError(f"Failed to open video: {input_path}") # Get video properties fps = cap.get(cv2.CAP_PROP_FPS) width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Correct for rotation metadata (e.g. iPhone portrait videos stored as landscape) _rotate_code = get_cv2_rotation_code(input_path) if _rotate_code in (cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE): width, height = height, width print(f"Video: {width}x{height} @ {fps:.2f} fps", file=sys.stderr) # Seek to segment start start_frame = int(crop_path.segment.start * fps) cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) # Calculate frames to process duration = crop_path.segment.end - crop_path.segment.start num_frames = int(duration * fps) print(f"Segment: {crop_path.segment.start:.2f}s - {crop_path.segment.end:.2f}s " f"({duration:.2f}s, {num_frames} frames)", file=sys.stderr) # Create interpolator (SAME as cropping function) interpolator = CropInterpolator(crop_path.keyframes, crop_path.segment.start) crop_width = interpolator.crop_width crop_height = interpolator.crop_height print(f"Crop: {crop_width}x{crop_height}", file=sys.stderr) print(f"Interpolation method: {interpolator.interpolation_method}", file=sys.stderr) # Rectangle styling rectangle_color = (0, 255, 0) # Green (BGR) rectangle_thickness = 3 text_color = (255, 255, 255) # White text_bg_color = (0, 0, 0) # Black # Pipe full-resolution annotated frames directly to FFmpeg ffmpeg_cmd = [ str(settings.ffmpeg_path), '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', f'{width}x{height}', '-r', str(fps), '-i', 'pipe:0', '-ss', str(crop_path.segment.start), '-t', str(duration), '-i', str(input_path), '-map', '0:v:0', '-map', '1:a:0?', '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', '-c:a', 'aac', '-b:a', '192k', '-y', str(output_path), ] ffmpeg_proc = subprocess.Popen( ffmpeg_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, ) print(f"\nProcessing frames with overlay...", file=sys.stderr) try: for frame_idx in range(num_frames): ret, frame = cap.read() if not ret: break if _rotate_code is not None: frame = cv2.rotate(frame, _rotate_code) current_time = crop_path.segment.start + (frame_idx / fps) # Get interpolated position (SAME calculation as cropping!) crop_x, crop_y = interpolator.get_position(current_time, width, height) # Draw rectangle instead of cropping cv2.rectangle( frame, (crop_x, crop_y), (crop_x + crop_width, crop_y + crop_height), rectangle_color, rectangle_thickness, cv2.LINE_AA ) # Add 2-line caption with black background: # Line 1 — timecode, segment type, score, crop position # Line 2 — Gemini segment description (truncated) seg = crop_path.segment font = cv2.FONT_HERSHEY_SIMPLEX line1 = ( f"t={current_time:.2f}s " f"[{seg.segment_type}] " f"score={seg.score:.2f} " f"crop=({crop_x},{crop_y})" ) desc = seg.description line2 = (desc[:120] + "...") if len(desc) > 123 else desc (w1, h1), _ = cv2.getTextSize(line1, font, 0.55, 2) (w2, h2), _ = cv2.getTextSize(line2, font, 0.45, 1) pad = 6 bg_w = max(w1, w2) + 2 * pad bg_h = h1 + h2 + 3 * pad cv2.rectangle(frame, (5, 5), (5 + bg_w, 5 + bg_h), text_bg_color, -1) cv2.putText(frame, line1, (5 + pad, 5 + pad + h1), font, 0.55, (0, 255, 0), 2) cv2.putText(frame, line2, (5 + pad, 5 + 2 * pad + h1 + h2), font, 0.45, text_color, 1) try: ffmpeg_proc.stdin.write(frame.tobytes()) except BrokenPipeError: print("Warning: FFmpeg pipe closed early", file=sys.stderr) break # Progress indicator if frame_idx % max(1, num_frames // 10) == 0: progress = (frame_idx / num_frames) * 100 print(f" Progress: {progress:.0f}% ({frame_idx}/{num_frames})", file=sys.stderr) print(f" Progress: 100%", file=sys.stderr) finally: cap.release() try: ffmpeg_proc.stdin.close() except (BrokenPipeError, ValueError, OSError): pass ffmpeg_proc.stdin = None _, ffmpeg_stderr = ffmpeg_proc.communicate() if ffmpeg_proc.returncode != 0: print(f"FFmpeg error:\n{ffmpeg_stderr.decode()}", file=sys.stderr) raise subprocess.CalledProcessError( ffmpeg_proc.returncode, ffmpeg_cmd, None, ffmpeg_stderr.decode() ) print(f"✓ Debug overlay video complete\n", file=sys.stderr) return output_path