#!/usr/bin/perl
# fetch_orchard_report.pl -- script to download an Orchard sales report
# Arguments:
#   -c clientID
#   -a Orchard AccountID
#   -p Orchard PeriodID
#   -u RPS user_id to receive Orchard notifications.  We don't send
#      notifications from this script, but rather set it in the RPS
#      file entry so that the file import (done elsewhere) will know
#      who to send notifications to.
#   -g generate report request if no report is found (default is to
#      exit script with a report not found error).  This option will
#      cause the report request to be visible on Orchard Workstation,
#      so make sure that this is ok beforehand with the client.
#
#   -t truncate downloaded reports.  The reports can be over a million
#      lines long, and can take significant time to process.  When the
#      downloaded archive file is unpacked, we can optionally truncate
#      the sales report to 100 lines prior to uploading it into RPS.
#      This makes it much easier to import and troubleshoot file
#      ingestion issues.  THIS OPTION IS FOR DEVELOPMENT TESTING ONLY.
#
# This script will check if a text report exists for the specified
# Orchard periodID.  If found, then it will be downloaded to the
# local filesystem, unzipped and imported.  Note: we won't download the
# report if it's already on the local filesystem, nor will we try to
# unzip it if it's already been unzipped.
#
# In the case of multi-part reports -- reports that the Orchard splits
# into 1000001-line chunks -- we'll coalesce those into a single report
# for import purposes.
#
#
use strict;
use warnings;

use Data::Dumper;
use Getopt::Long;
use File::Basename;
use File::Path qw( make_path rmtree);
use WWW::Mechanize;
use JSON qw( decode_json );
use Sort::Naturally 'nsort';

use lib '/app/tools/common/lib';
use Common::Assert;
use Common::RSDB;
use Common::RSApp;
use Common::Util;
use Common::Log;

use lib '/app/tools/raptor/lib';
use Raptor::Config;

use lib '/app/tools/data_classes/lib';
use File::File;

use lib '/app/tools/rps/lib';
use RPS::File::File;
use RPS::Sale::File;
use Common::DB::Item::OrchardReport;

# Downloaded reports will be stored and unpacked in the following directory
use constant kDownloadDir => '/app/data/orchard_fetch';

# Number of times to try connecting to OWS Accounting API before giving up
use constant kMaxAttempts => 3;

my $prog = basename($0);

my %args = ();
parseCommandLine( \%args );
my $clientID  = $args{clientID};
my $accountID = $args{accountID};
my $periodID  = $args{periodID};
my $userID    = $args{userID};
my $generate  = $args{generate};
my $truncate  = $args{truncate};
my $logLevel  = $args{logLevel};

assert($clientID);
assert($accountID);
assert($periodID);
assert($userID);

my $appSingleton = Common::RSApp->new( clientID => $clientID );

Log->init( level => 'error' );
Log->setLogLevel($logLevel) if ($logLevel);

my $mech = WWW::Mechanize->new( autocheck => 0 );
$mech->max_redirect(0);    # need this in order to follow redirects

# Create the local directory to contain downloaded report
#
my $downloadDir = kDownloadDir . '/' . $clientID . '/' . $accountID . '/' . $periodID . '/';    #   .../<clientID>/<accountID>/<periodID>

if ( !-d $downloadDir ) {
    make_path $downloadDir or die "Failed to create output directory: $downloadDir";
}

# Get or create an OrchardReport object for this period
#
my $report = Common::DB::Item::OrchardReport->Lookup( client_id => $clientID, account_id => $accountID, period => $periodID );
if ( $report && $report->file_id ) {
    Log->info( "$0: reportID " . $report->report_id . " has fileID " . $report->file_id );
    exit;
}
if ( !$report ) {
    $report = Common::DB::Item::OrchardReport->Create( client_id => $clientID, account_id => $accountID, period => $periodID );
    $report->save();
}

# Check if a report exists on the Orchard server.
# Note that we're looking for a report with the following settings:
#  - Must be text format
#  - Must have numbers in US format
#  - Must include all transactions
#
my $numberFormat     = 'en_US';
my $transactionTypes = 'all';

my $url =
"https://rsgd.theorchard.io/reports?periods=$periodID&account_id=$accountID&file_type=txt&number_format=$numberFormat&account_type=vendor&transaction_types=$transactionTypes";

my $httpStatus = 0;

# Try API until successful or max number of attempts exceeded.
#
my $attempt = 1;
while( $httpStatus != 200 && $attempt <= kMaxAttempts ) {
    $mech->get($url);
    $httpStatus = $mech->status();
    $attempt++;
    sleep(1);
}

my $prefix = "[$attempt/". kMaxAttempts . "] c($clientID) a($accountID) p($periodID) $prog";
Log->info("$prefix : GET /reports returned status $httpStatus");

# Decode the response
# Note: if we don't have a 200 status, or if no JSON was returned
# then no further processing will occur.
#
my $decoded_json;

eval {
    $decoded_json = decode_json( $mech->content() );
    1;
} or do {
    my $e = $@;
    print "$e\n";
    Log->error("$prefix: FATAL ERROR: UNABLE TO DECODE JSON RESPONSE - CHECK OWS PROXY");
    exit;
};

if ( $httpStatus != 200 ) {
    my $msg  = $decoded_json->{message};
    my $code = $decoded_json->{code};
    $report->status(Common::DB::Item::OrchardReport::kFileError);
    $report->notes("/reports returned $msg : $code");
    $report->save;
    Log->error("$prefix: API GET /reports returned status $httpStatus message($msg) code($code)");
    exit;
}

my @items = @{ $decoded_json->{items} };    # all of the available reports

my $foundFile;
my $foundFileStatus;

# Check all of the available reports for the requested period.
#
Log->debug("$prog: File(s) for period $periodID :");
foreach my $item (@items) {
    Log->debug( "Item: " . Dumper($item) );

    my $s3path           = $item->{s3_path};
    my $fileType         = $item->{file_type};
    my $transactionTypes = $item->{transaction_types};
    my $numberFormat     = $item->{number_format};
    my $status           = $item->{status};

    # Is this the report that we're looking for?
    #
    if (   $fileType =~ /^txt$/i
        && $transactionTypes =~ /^all$/i
        && $numberFormat =~ /^en_US$/i ) {

        # Note: s3path is blank if a report is being generated

        $foundFile = ( $status =~ /GENERATING/i ) ? "FILE_NOT_READY" : basename($s3path);
        $foundFileStatus = $status;
    }
}

if ($foundFile) {
    Log->debug("$prog: Report found: $foundFile");

    $report->name($foundFile);
    $report->save();

    if ( $foundFileStatus && $foundFileStatus !~ /GENERATED/i ) {

        # if the report is still waiting to be generated, then we
        # we'll make a note of this and exit the script
        #
        $report->status(Common::DB::Item::OrchardReport::kFileNotReady);
        $report->save();

        Log->warn("$prog: Found file $foundFile, but it has an invalid status '$foundFileStatus'");
        exit;
    }
} else {

    # if the report isn't available, the script can optionally
    # submit a report generation request on behalf of the client.
    #
    if ($generate) {
        Log->warn("$prog: Submitting report request for client $clientID account $accountID period $periodID");

        # Setup the POST data
        #
        my $url = "https://rsgd.theorchard.io/report";
        my $req = HTTP::Request->new( 'POST', $url );
        $req->header(
            'Content-Type'       => 'application/json',
            'Orchard-User-Id'    => 'oa:1414',
            'Grass-Account-Type' => 'vendor',
            'Grass-Account-Id'   => "$accountID"
        );

        my $json =
            '{"file_type":"txt","number_format": "en_US", "transaction_types": "all", "periods": "'
          . $periodID
          . '", "request_context":{"first_name": "The","last_name": "Orchard", "contact_id": 15245}}';

        $req->content($json);

        my $status = 0;

        # Try API until successful or max number of attempts exceeded.
        #
        my $prefix;
        $attempt = 1;
        while( $status != 200 && $attempt <= kMaxAttempts ) {
            $mech->request($req);
            $prefix = "[$attempt/". kMaxAttempts . "] $prog";
            Log->debug("$prefix: Requesting report, JSON = $json");
            $status = $mech->status();
            $attempt++;
            sleep(1);
        }

        Log->debug( "$prefix: API POST /report returned status $status: " . $mech->content() );

        if ( $status == 200 ) {
            Log->debug("$prefix: report requested for client $clientID account $accountID period $periodID");
            $report->notes('');
            $report->status(Common::DB::Item::OrchardReport::kFileRequested);
        } else {
            print STDERR "API POST /report returned status $status: " . $mech->content() . "\n";
            Log->error( "$prefix: API POST /report returned status $status: " . $mech->content() );
            $report->notes( $mech->content() );
            $report->status(Common::DB::Item::OrchardReport::kFileError);
        }

        $report->save();
    } else {
        Log->warn("$prog: No report found for client $clientID period $periodID and 'generate' option not set");
        $report->notes("'generate' option not set; unable to submit report");
        $report->status(Common::DB::Item::OrchardReport::kFileNotFound);
        $report->save();
    }

    exit;
}

# Download the file if we don't have it locally
#
my $localFilepath = $downloadDir . $foundFile;

if ( !-e $localFilepath ) {
    my $url = "https://rsgd.theorchard.io/report?periods=$periodID&account_id=$accountID&file_type=txt"
      . "&number_format=$numberFormat&account_type=vendor&transaction_types=$transactionTypes";

    Log->info("$prog: download url: $url");

    my $response;
    my $status = 0;

    # Try API until we get redirect or max number of attempts exceeded.
    #
    $attempt = 1;
    while( $status != 302 && $attempt <= kMaxAttempts ) {
        $response = $mech->get($url);
        $status   = $mech->status();
        $attempt++;
        sleep(1);
    }
    my $prefix = "[$attempt/". kMaxAttempts . "] $prog";


    # if successful, we'll get a redirect to the file
    #
    if ( $status == 302 ) {
        my $location = $response->header("Location");
        Log->info("$prefix: Redirecting to $location");
        $mech->get($location);
        $mech->save_content($localFilepath);

        $report->status(Common::DB::Item::OrchardReport::kFileDownloaded);
        $report->notes('');
        $report->save();

        Log->info("$prog: File saved: $localFilepath");
    } else {
        Log->error("$prefix: Unable to download file; GET /report returned status $status");

        $report->status(Common::DB::Item::OrchardReport::kFileError);
        $report->notes( "download failed: " . $status );
        $report->save();
        exit;
    }
} else {
    Log->warn("$prog: file exists: $localFilepath");
}

#----------------------------------------------------------------------
# The downloaded report file should be a ZIP archive containing a single
# file (if <= 1000001 lines), or a file broken into multiple parts
# (each <= 1000001 lines).  If multiple parts are found, these will be
# merged into a single file that can be imported into RPS.
#----------------------------------------------------------------------

my $textFilepath = $localFilepath;    # full path to the ZIP file containing report
($textFilepath) = $1 if ( $localFilepath =~ /(.+)\.zip$/i );

my $textFilename = basename($textFilepath);    # name of the unzipped text report

if ( !-e $textFilepath ) {

    # Unzip the archive if the text report does not exist
    #
    Log->debug("$prog: unzipping '$localFilepath'");
    chdir($downloadDir);
    `unzip $localFilepath`;

    # The unzipped report may be in parts that need to be coalesced.
    # You'll know when you see files with a "_partN.txt" suffix (this
    # is in addition to the actual expected report filename).
    #
    Log->debug("$prog: looking for '$textFilename' in path $downloadDir");

    opendir my $dir_h, "$downloadDir" or die("Cannot open directory: $!");

    my $tmpFile = 'tmp_' . $$;
    my @files = nsort grep { /$textFilename(_part(\d+)\.txt)?$/ } readdir $dir_h;

    if ( @files > 1 )    # archive contains file split into multiple parts
    {
        my $first = shift @files;
        my $n     = 1;
        Log->debug("part $n: $first (tmp = $tmpFile)");
        chdir($downloadDir);
        `cp $first $tmpFile`;
        foreach my $f (@files) {
            Log->debug("part $n: $f");
            `tail -n +2 $f >> $tmpFile`;
            `rm $f`;
        }

        if ($truncate) {
            `head -100 $tmpFile > $textFilename`;    # limit the report to 100 lines for testing
            `rm $tmpFile`;
        } else {
            `mv $tmpFile $textFilename`;
        }

        closedir $dir_h;
    } elsif ( @files == 0 ) {
        Log->error("$prog: Unable to extract text file");
        $report->status(Common::DB::Item::OrchardReport::kFileError);
        $report->notes("Unable to extract text file");
        $report->save;
    } else {

        # archive contains only one file

        if ($truncate) {
            chdir($downloadDir);
            my $first = shift @files;
            `cp $first $tmpFile`;
            `head -100 $tmpFile > $textFilename`;    # limit the report to 100 lines for testing
            `rm $tmpFile`;
        }

        Log->debug("$prog: only one file, skipping file merge");
    }
    $report->status(Common::DB::Item::OrchardReport::kFileUnpacked);
    $report->save;
} else {
    Log->warn("$prog: file already unzipped: $textFilepath");
    chdir($downloadDir);
}

my $file;
my $fileID;
my $fileStatus;

# Upload the file from the local FS into RPS if we don't
# already have it in our system.
#
my $files  = RPS::File::Files->new();
my $md5sum = Common::Util::md5sum($textFilepath);

if ( $files->GetByOrigName( file_name => $textFilename ) || $files->GetByMD5( md5_sum => $md5sum ) ) {
    $file       = $files->GetNext();
    $fileID     = $file->FileID();
    $fileStatus = $file->FileStatus();
    Log->error("DUPLICATE_FILE: client $clientID fileID $fileID (status $fileStatus) exists for '$textFilename'");

    # If the file is in RPS and it looks like it's being (or has been)
    # processed, then we'll just flag the report and update the notes
    # field with the RPS file status.
    #
    if ( $fileStatus != File::File::STATUS_NEW ) {
        $report->status(Common::DB::Item::OrchardReport::kFileError);
        $report->notes( "File $fileID already exists with status " . $fileStatus );
        $report->file_id($fileID);
        $report->save;
        exit(0);
    }
} else {
    Log->info("Uploading '$textFilename' into RPS ...");
    $fileID     = upload($textFilename);
    $file       = RPS::File::File->new( file_id => $fileID, client_id => $clientID );
    $fileStatus = $file->FileStatus();

    unlink $textFilepath  or die "Unable to unlink $textFilepath: $!";     # text file
    unlink $localFilepath or die "Unable to unlink $localFilepath: $!";    # zip file
    rmtree($downloadDir)  or die "Unable to delete $downloadDir: $!";      # download directory
}

# Set the UserID to the id of client's notification email address (see FB18135).
#
$file->UserID($userID);
$file->Save();

# Update the orchard report info
#
$report->status(Common::DB::Item::OrchardReport::kFileImportReady);
$report->file_id($fileID);
$report->save();

if ( !$fileID ) {
    Log->error("$prog: #### No fileID found for '$textFilename' !!!");
} else {
    Log->info("$prog: #### File uploaded, fileID $fileID (status $fileStatus)");
}

sub upload {
    my $file = shift;
    my $fh;
    open( $fh, $file ) or die "can't open file: $!\n";
    my $md5sum = Common::Util::md5sum($fh);
    close($fh);

    return RPS::Sale::File::SimpleUploadFromShell(
        client_id => $clientID,
        filepath  => $file,
        md5_sum   => $md5sum,
    );
}

sub parseCommandLine {
    my ($a) = @_;
    my $clientID;
    my $accountID;
    my $periodID;
    my $userID;
    my $generate;
    my $truncate;
    my $logLevel;

    GetOptions(
        'c|client_id=i'  => \$clientID,
        'a|account_id=i' => \$accountID,
        'p|period_id=s'  => \$periodID,    # can be DDD or DDD,DDD,DDD
        'u|user_id=i'    => \$userID,
        'g|generate'     => \$generate,
        't|truncate'     => \$truncate,
        'l|loglevel=s'   => \$logLevel,
    );

    my $msg;
    unless ( $clientID && $clientID =~ /^\d+$/ && $clientID > 0 ) {
        my $e = "You must enter a valid clientID";
        $msg = ($msg) ? "$msg; $e" : $e;
    }
    unless ( $userID && $userID =~ /^\d+$/ && $userID > 0 ) {
        my $e = "You must enter a valid userID";
        $msg = ($msg) ? "$msg; $e" : $e;
    }
    unless ( $accountID && $accountID =~ /^\d+$/ && $accountID > 0 ) {
        my $e = "You must enter a valid accountID";
        $msg = ($msg) ? "$msg; $e" : $e;
    }
    unless (
        $periodID
        && (   $periodID =~ /^\d+$/
            || $periodID =~ /^\d+,\d+,\d+$/ )
      ) {
        my $e = "You must enter a valid periodID";
        $msg = ($msg) ? "$msg; $e" : $e;
    }

    die( usage($msg) ) if ($msg);

    $a->{clientID}  = $clientID;
    $a->{accountID} = $accountID;
    $a->{periodID}  = $periodID;
    $a->{userID}    = $userID;
    $a->{generate}  = $generate;
    $a->{truncate}  = $truncate;
    $a->{logLevel}  = $logLevel;
}

sub usage {
    my $errstr = shift;
    my $text = ($errstr) ? "ERROR: $errstr\n" : '';
    $text .= "Usage $0 -c <clientID> -a <accountID> -p <periodID> -u <userID> [-g] [-o <outputDirectory>] [-f <fileName>] [-t]";
}

###
1;    #
###