#---------------------------------------------------------------
# ____                   _ _         ____  _
#|  _ \ ___  _   _  __ _| | |_ _   _/ ___|| |__   __ _ _ __ ___
#| |_) / _ \| | | |/ _` | | __| | | \___ \| '_ \ / _` | '__/ _ \
#|  _ < (_) | |_| | (_| | | |_| |_| |___) | | | | (_| | | |  __/
#|_| \_\___/ \__, |\__,_|_|\__|\__, |____/|_| |_|\__,_|_|  \___|
#            |___/             |___/
#
# Copyright (C) 2009 RoyaltyShare, Inc.   All Rights Reserved
#---------------------------------------------------------------
package BookPub::Catalog::Import::Process::Fetcher::Base;

use strict;
use warnings;

use constant 'USE_UPLOAD' => 0;

use Net::FTP;
use Data::Dumper;
use Net::SFTP::Foreign;
use Capture::Tiny(qw/capture/);

use lib '/app/tools/common/lib';
use lib '/app/tools/bookpub/lib';
use Common::RSApp;
use Common::Util;
use Common::Assert;
use Common::Process;
use Common::Client;
use Common::DB::Item::Language;
use BookPub::DB::Item::CatalogImport;
use BookPub::DB::Item::CatalogImportItem::FTPFile;
use BookPub::Config;
use BookPub::Catalog::Import::Process::Fetcher::FileHandler::ONIX;
use BookPub::Catalog::Import::Process::Fetcher::FileHandler::RSStandard;
use BookPub::Catalog::Import::Process::Fetcher::FileHandler::ImageArchive;
use BookPub::Catalog::Import::Process::Fetcher::FileHandler::ImageFile;
use BookPub::Catalog::Import::Process::Fetcher::FileHandler::ZipArchive::ONIX;

use base 'BookPub::Catalog::Import::Process';

use constant kMaxGetAttempts     => 3;
use constant kRetrySleepInterval => 60;
use constant kMaxSortTokenLength => 14;

sub run {
    my ($self) = @_;

    $self->{_importRecord}->status(BookPub::DB::Item::CatalogImport::kImportStatusFetchRun);
    $self->{_importRecord}->save();

    $self->_report( "entering BookPub::Catalog::Import::Process::Fetcher::run", 3 );

    # Set the catalogImportID in the Application singleton, so that we'll have
    # that ID available in the table_change_log.
    #
    Common::RSApp::GetRAMCache()->{catalogImportID} = $self->_catalogImportID();

    # !!! I expect that we'll subclass this object.
    # ftp location should be abstract
    # method to _identify_ files should be abstract
    # method to rename files should be abstract

    # Connect to the ftp site.
    #
    $self->_connectToFTP();
    $self->_changeToClientDirectory();

    my @directoryListing = $self->waitingFiles();

    my @handlers;
    foreach my $filename (@directoryListing) {
        $self->_report( "  looking at file: $filename", 3 );

        # ALWAYS skip files that start with '_'...
        #
        next if ( '_' eq substr( $filename, 0, 1 ) );

        # Need to :
        # - Identify the file's "type" (image, onix, zip, etc).
        # - copy the file over if we care about it.
        # - post-process the file (unzip it, create CatalogImportFile records, etc).
        # ... and who knows what else?
        #
        # I think the most flexible strategy is to use a FileHandler object of some sort.
        #
        # However... I don't want to let the FTP connection time out!  If we run a handler, and
        # it has some lengthy post-download process to run, we might lose our connection.
        # I think the handler needs to work exclusively on _local_ files.  So, we'll download
        # _all_ files we find, and archive _all_ files, then deal with the handlers afterwards.
        #
        #
        $self->_report( "  getting handler", 3 );
        my $handler = $self->_getFileHandler($filename);

        $self->_report( "  ... received this handler:", 4 );
        $self->_report( $handler,                       4 );

        if ($handler) {
            push @handlers, $handler;
        }
    }

    # Download the files.
    #
    foreach my $handler (@handlers) {
        $self->_report(
            "invoking handler's download method: remote file=" . $handler->remoteFile() . ", local file=" . $handler->localFile() );

        # Downloading seems to occasionaly fail.
        # Due to network wackiness.
        # Need to be able to retry a few times before giving up.
        #
        $handler->download();
    }

    # Now that all files have been downloaded (presumably), process them.
    #
    foreach my $handler (@handlers) {
        $self->_report(
            "invoking handler's process method: remote file=" . $handler->remoteFile() . ", local file=" . $handler->localFile() );
        $handler->process();
    }

    return 0;
}

sub archiveFTPFiles {
    my ($self) = @_;

    Common::RSApp::GetRAMCache()->{catalogImportID} = $self->_catalogImportID();

    $self->_connectToFTP();
    $self->_changeToClientDirectory();

    my $remoteFiles = BookPub::DB::Item::CatalogImportItem::FTPFile->GetAllByCatalogImport( $self->_catalogImportID );
    while ( my $remoteFileItem = $remoteFiles->next() ) {
        my $filename = $remoteFileItem->file_name();
        $self->_ftpMoveToArchive($filename);
    }
}

sub waitingFiles {
    my ($self) = @_;

    $self->_connectToFTP();
    $self->_changeToClientDirectory();

    my $directoryListing = $self->_ftpGetDirectoryListing();

    my $filteredListing = $self->_filterNewestFiles($directoryListing);

    my @files;
    foreach my $filename (@$filteredListing) {
        $self->_report( "  looking at file: $filename", 3 );

        # ALWAYS skip files that start with '_'...
        #
        next if ( '_' eq substr( $filename, 0, 1 ) );

        push @files, $filename;
    }

    return @files;
}

sub _filterNewestFiles {
    my ( $self, $directoryListing ) = @_;

    my @sortArray;
    my %hashedFiles;
    my $smallestToken;
    foreach my $filename (@$directoryListing) {
        my $sortToken = $self->_sortTokenFromFilename($filename);

        # If we don't find _ANY_ 'token' value in the filename, ignore it.
        #
        next unless $sortToken;

        # The largest numbers we will see (at least so far) have 14 digits, so we'll pad
        # the token to that size.
        #
        my $tokenLength = length $sortToken;
        if ( $tokenLength < 14 ) {
            my $padding = 14 - $tokenLength;
            $sortToken .= '0' x $padding;
        }

        if ( !defined $smallestToken || $smallestToken > $sortToken ) {
            $smallestToken = $sortToken;
        }

        push @{ $hashedFiles{$sortToken} }, $filename;
    }

    return $hashedFiles{$smallestToken};
}

sub _sortTokenFromFilename {
    my ( $self, $filename ) = @_;

    # Default behavior is to return the first number with at least 4 digits.
    # This will work for the majority of clients (but sadly not all).
    # The rest will have to overload this method.
    #
    my $token;
    if ( $filename =~ /(\d{4,})/ ) {
        $token = $1;
    }

    return $token;
}

sub _ftpSite {
    my ($self) = @_;
    return $self->_config()->get('catalog_import_ftp_site');
}

sub _ftpUserName {
    my ($self) = @_;
    return $self->_config()->get('catalog_import_ftp_user_name');
}

sub _ftpDebug {
    my $self = shift;
    return $self->_config()->get('catalog_import_ftp_debug');
}

sub _ftpTimeout {
    my $self = shift;
    return $self->_config()->get('catalog_import_ftp_timeout');
}

sub _ftpPassword {
    my ($self) = @_;
    return $self->_config()->get('catalog_import_ftp_password');
}

sub _connectToFTP {
    my ($self) = @_;

    if (USE_UPLOAD) {
        $self->{'_ftp'} = $self->_connectToUpload;
    } else {
        $self->{'_ftp'} = $self->_connectToSFTP;
    }
}

sub _connectToSFTP {
    my $self = shift;

    my $sftpUsername = $self->_ftpUserName();
    my $sftpPassword = $self->_ftpPassword();

    use Net::SFTP::Foreign;
    my $sftp = Net::SFTP::Foreign->new(
        'transfer.royaltyshare.com',
        'user' => $sftpUsername,
        'password' => $sftpPassword,
        'queue_size' => 1, # See https://docs.aws.amazon.com/transfer/latest/userguide/transfer-file.html
        'more' => [-o => 'StrictHostKeyChecking=no', -o => 'PreferredAuthentications=password']
    );
    $sftp->error and die "ERROR - unable to connect to AWS SFTP site: " . $sftp->error;


    $sftp;

}

sub _connectToUpload {
    my $self = shift;
    $self->_report( "entering BookPub::Catalog::Import::Process::Fetcher::_connectToFTP", 3 );

    # Credentials.   Should these just live in here?
    # Seems weird to put a password in a config file.
    # But... what the hell.
    #
    my $ftpDebug    = $self->_ftpDebug();
    my $ftpTimeout  = $self->_ftpTimeout();
    my $ftpSite     = $self->_ftpSite();
    my $ftpUserName = $self->_ftpUserName();
    my $ftpPassword = $self->_ftpPassword();

    my $ftp = Net::FTP->new(
        $ftpSite,
        Debug   => $ftpDebug,
        Timeout => $ftpTimeout
    ) or die "ERROR - unable to connect to ftp site $ftpSite: $!";
    $ftp->login( $ftpUserName, $ftpPassword ) or die "ERROR - login failed: $!";

    # Set binary transfer mode immediately...
    #
    $ftp->binary() or die "ERROR - 'binary' failed!: $!";
    $ftp;

}

sub _disconnectFromFTP {
    my $self = shift;
    return $self->{_ftp}->disconnect() if $self->{_ftp};
}

sub _changeToClientDirectory {
    my ($self) = @_;

    # On the new SFTP server, the client directory name is repeated.
    my $clientDirectory = $self->_clientFTPDirectory() . "/" . $self->_clientFTPDirectory();
    $self->{_ftp}->setcwd($clientDirectory) or die "ERROR - unable to 'cwd' to ftp directory $clientDirectory : $!";
}

sub _clientFTPDirectory {
    my ($self) = @_;
    assert( 0, 'override' );
}

sub _ftpGetDirectoryListing {
    my ($self) = @_;
    return $self->{_ftp}->ls( names_only => 1, queue_size => 1 );
}

sub _getFileHandler {
    my ( $self, $filename ) = @_;

    # Many of our clients will be fine with generic default handlers.
    # We'll use a 'dispatch table' approach, which will map file extensions
    # to method names.   So we'll have two mechanisms available to change
    # how this behaves:
    # - Override the default method associated with a file type.
    #   Ex.  If you need to handle .zip files differently, just override _getFileHandlerZip().
    # - Create a new dispatch table entry to handle a new file extension.
    #   So if we are getting '.doc' file at some point, you would override the _getFileHandlerDispatchTable() method
    #   to associate a new method name (which you would also need to implement) with 'doc'.
    #

    my $dispatchTable = $self->_getFileHandlerDispatchTable();

    my $extension;

    # We need to trim off any leading/trailing spaces first.
    (my $trimmedFilename = $filename) =~ s/^\s+|\s+$//;

    # Assume 3-5 character extensions.
    #
    if ( $trimmedFilename =~ /\.(\w{3,5})$/ ) {
        $extension = $1;
    }

    my $fileHandler;
    if ($extension) {
        my $method = $dispatchTable->{$extension};
        if ($method) {
            $fileHandler = $self->$method($filename, $trimmedFilename);
        }
    }

    # Default to the 'copy and ignore' handler.
    #
    if ( !$fileHandler ) {
        $fileHandler = BookPub::Catalog::Import::Process::Fetcher::FileHandler::CopyThenIgnore->new(
            fetcher        => $self,
            remoteFile     => $filename,
            localDirectory => $self->_getBaseStagingPath(),
            localFileName  => $trimmedFilename,
            logLevel       => $self->{_logLevel},
        );
    }

    return $fileHandler;
}

sub _getFileHandlerDispatchTable {
    my ($self) = @_;

    # If you override this, make sure you call the inherited method FIRST, then add your
    # extra stuff to the hash reference that returns.
    # i.e.
    # my $table = $self->SUPER::_getFileHandlerDispatchTable();
    # $table->{foo} = '_getFileHandlerFoo';
    #

    my %dispatch = (
        'xml'  => '_getFileHandlerXML',
        'onx'  => '_getFileHandlerXML',
        'xls'  => '_getFileHandlerExcel',
        'xlsx' => '_getFileHandlerExcel',
        'zip'  => '_getFileHandlerZip',
        'jpg'  => '_getFileHandlerJPG',
    );

    return \%dispatch;
}

sub _getFileHandlerXML {
    my ( $self, $filename, $trimmedFilename ) = @_;

    return BookPub::Catalog::Import::Process::Fetcher::FileHandler::ONIX->new(
        fetcher        => $self,
        remoteFile     => $filename,
        localDirectory => $self->_getBaseStagingPath(),
        localFileName  => $trimmedFilename,
        logLevel       => $self->{_logLevel},
    );
}

sub _getFileHandlerJPG {
    my ( $self, $filename, $trimmedFilename ) = @_;

    return BookPub::Catalog::Import::Process::Fetcher::FileHandler::ImageFile->new(
        fetcher        => $self,
        remoteFile     => $filename,
        localDirectory => $self->_getBaseStagingPath(),
        localFileName  => $trimmedFilename,
        logLevel       => $self->{_logLevel},
    );
}

sub _getFileHandlerExcel {
    my ( $self, $filename, $trimmedFilename ) = @_;

    return BookPub::Catalog::Import::Process::Fetcher::FileHandler::RSStandard->new(
        fetcher        => $self,
        remoteFile     => $filename,
        localDirectory => $self->_getBaseStagingPath(),
        localFileName  => $trimmedFilename,
        logLevel       => $self->{_logLevel},
    );
}

sub _getFileHandlerZip {
    my ( $self, $filename, $trimmedFilename ) = @_;

    # So far it's pretty typical for zip files containing images to have 'covers' in the filename,
    # !!! Some have 'images' in the filename instead.
    if ( ( $filename =~ m/covers/i ) || ( $filename =~ m/images/i ) ) {
        return BookPub::Catalog::Import::Process::Fetcher::FileHandler::ImageArchive->new(
            fetcher        => $self,
            remoteFile     => $filename,
            localDirectory => $self->_getBaseStagingPath(),
            localFileName  => $trimmedFilename,
            logLevel       => $self->{_logLevel},
        );
    }

    # Otherwise we'll assume it's a zip file full of onix files.
    #
    return BookPub::Catalog::Import::Process::Fetcher::FileHandler::ZipArchive::ONIX->new(
        fetcher        => $self,
        remoteFile     => $filename,
        localDirectory => $self->_getBaseStagingPath(),
        localFileName  => $trimmedFilename,
        logLevel       => $self->{_logLevel},
    );
}

# ftp-related methods to be invoked by the filehandlers.
sub _ftpGet {
    my ( $self, $remoteFile, $destinationFile ) = @_;
    assert($remoteFile);
    assert($destinationFile);
    $self->_report( "entering BookPub::Catalog::Import::Process::Fetcher::_ftpGet", 3 );
    $self->_report( "remoteFile: $remoteFile,  destinationFile: $destinationFile",  4 );

    # We'll make several attempts to get each file, in case the network is flakey.
    #
    my $attempts = 0;
    while ( $attempts < kMaxGetAttempts ) {

        my $remoteFileSize;
        eval {
            my $stats = $self->{_ftp}->stat($remoteFile);
            $remoteFileSize = $stats->size || 0;
            $self->{_ftp}->get( $remoteFile, $destinationFile );
        };

        #print STDERR "file size is " . $self->{_ftp}->size($remoteFile) . "\n";

        # necessary checks
        my $destinationFileSize = -s $destinationFile || 0;
        if ( $destinationFileSize && $destinationFileSize == $remoteFileSize ) {
            last;
        } else {
            $self->_report( "BookPub::Catalog::Import::Process::Fetcher::_ftpGet the size of the files is not equal\n", 3 );
        }

        if ( my $error = $self->{_ftp}->error ) {
            $self->_report( "BookPub::Catalog::Import::Process::Fetcher::_ftpGet $error\n", 3 );
        }

        $self->_report( "BookPub::Catalog::Import::Process::Fetcher::_ftpGet $@\n", 3 ) if $@;
        $self->_disconnectFromFTP();

        $attempts++;
        last if ( $attempts >= kMaxGetAttempts );

        $self->_report( "BookPub::Catalog::Import::Process::Fetcher::_ftpGet  attempt $attempts FAILED: $!  : sleeping until retry", 3 );

        # Each time we retry, we'll tack on an extra minute.
        #
        sleep( kRetrySleepInterval * $attempts );

        # Chances are we'll need to reconnect.
        #
        $self->_connectToFTP();

        $self->_changeToClientDirectory();
    }

    if ( $attempts >= kMaxGetAttempts ) {
        die "ERROR - unable to get $remoteFile after $attempts attempts: $!";
    }
}

# !!! Man, deleting files is scary.
# !!! Perhaps it would be safer to move the files to a different place on the ftp server?
#
sub _ftpDelete {
    my ( $self, $remoteFile ) = @_;
}

sub _ftpMoveToArchive {
    my ( $self, $remoteFile ) = @_;
    $self->_report( "entering BookPub::Catalog::Import::Process::Fetcher::_ftpMoveToArchive", 3 );
    $self->_report( "remoteFile: $remoteFile",                                                4 );

    my $importID = $self->_catalogImportID();

    my $clientDirName    = $self->_clientFTPDirectory();
    my $archiveDirectory = "/_bookpub_archive/_bookpub_archive/$clientDirName/$importID";

    $self->{_ftp}->mkpath($archiveDirectory);

    $self->{_ftp}->rename( $remoteFile, "$archiveDirectory/$remoteFile" )
      or die "ERROR - unable to move $remoteFile to _archive directory '$archiveDirectory/$remoteFile': $!";
}

###
1;    #
###