#!/bin/bash

set -e

# source batch file to split
SOURCE_FILE=$1
if [ -z $SOURCE_FILE ]; then
    echo '$1 missing - source file'
    exit 1
fi
if [ ! -f $SOURCE_FILE ]; then
    echo "$SOURCE_FILE does not exist"
    exit 1
fi

# records already processed
OFFSET=$2
if [ -z $OFFSET ]; then
    echo '$2 missing - offset'
    exit 1
fi

# num records to include in batch
BATCH_SIZE=$3
if [ -z $BATCH_SIZE ]; then
    echo '$3 missing - batch size'
    exit 1
fi

# batch unique ID
BATCH_ID=$4
if [ -z $BATCH_ID ]; then
    echo '$4 missing - batch id'
    exit 1
fi

# parallel workers to start
WORKERS=$5
if [ -z $WORKERS ]; then
    echo '$5 missing - num workers'
    exit 1
fi

# make sure python virtual env is active
if [ -z $VIRTUAL_ENV ]; then
    echo 'virtual env not initialized'
    exit 1
fi

# make sure aws is configured
aws sts get-caller-identity

# check if batch is already in progress
WORK_DIR="batches/$BATCH_ID"
if [ -d $WORK_DIR ]; then
    read -p "Batch directory already exists, restart batch? (yes/no): " yn
    case $yn in 
        yes ) echo Restarting batch based on log files...;;
        no ) echo Exiting...;
            exit;;
        * ) echo Invalid response;
            exit 1;;
    esac
else
    mkdir -p $WORK_DIR
fi

# split source file
mkdir -p 'data'
BATCH_FILE="data/Meta_Batch_$BATCH_ID.csv"
sed -n $(($OFFSET+1)),$(($OFFSET+$BATCH_SIZE))p $SOURCE_FILE > $BATCH_FILE

# split batch file into chunks for parallel processing
gsplit --number=l/$WORKERS --additional-suffix '.csv' $BATCH_FILE $WORK_DIR/$(basename $BATCH_FILE).

# start jobs, restarting if log file exist already
for FILE in $WORK_DIR/*.csv; do
    FILE_ID=$(echo $FILE | cut -d '.' -f 3)
    LOG_FILE="$WORK_DIR/$FILE_ID.log"
    SLEEP=5
    if [ -f $LOG_FILE ]; then
        SINCE=$(tail -n 1 $LOG_FILE | cut -d ' ' -f 1)
        python -u -W ignore run.py --sleep $SLEEP --since $SINCE $FILE >> $LOG_FILE &
    else
        python -u -W ignore run.py --sleep $SLEEP $FILE >> $LOG_FILE &
    fi
done

exit 0