package Support::Implementation::ExpenseTemplate;
# 18MAR10 - Added logic to force fatal error if a template column is missing
# 8/25/10: Added logic to handle duplicate payee names
# 4/11/11: Corrected payeeID lookup when looking for contracts by title
# 4/7/11: Added 'Payee' synonym for 'Payee Name'
# 5/18/11: Added '*Contract Title', '*Album Name' and '*Track Title'
#   synonyms. NOTE: the leading asterisk tells the importer that the column
#   is REQUIRED, regardless of what's in the other columns.  So, '*Track Title'
#   must not be blank.  This really should be just 'Track Title' -- the field
#   is only used if we're trying to import a track-level expense.
# 3/3/12: Filter out tabs in the input fields
# 8/30/12: Added kludge for Nettwerk to allow albumID/trackID/contractID
#   forcing of match (necessitated due to FB16432).
# 9/26/12: Updated _checkMissingColumn to be aware of 'alt' columns
# 2/18/14: Added Excel 2007 support
# 5/28/14: Added check for artist run in progress
# 12/18/14: Check for blank lines.
# 2/26/15: Clean up category name
# 5/4/15: Loosened expense category check to allow 'Expense' or 'Expenses'
# 11/30/15: Added error message for when album _and_ track are missing
# 3/8/16: Check for missing expense type
# 3/29/16: Debugging missing expense type error; added logic to fix amount formatting.
# 5/16/16: Round amounts to 4 decimal places (this matches expense.amount)
# 1/5/17: Added some debug code to detect if "album not attached" exceptions
#  are due to missing track information in the template.
# 9/6/18: Added title_version comparisons when verifying template album/track
#  titles with the RPS stored version.  Also removed Nettwerk-specific functionality.
#  Added logic to create album/track "version" titles if title_version is defined
#  for either one. See RSD-2319 for example of where template is using version
#  titles.
# 3/19/19: Removed track.title_version reference.
# 4/29/20: Limit new_artist_contract searches to active (deleted=0) contracts.
# 5/28/20: Added support for expense lines without contract info (RSD-4429)
use strict;
use warnings;

use lib '/app/tools/common/lib';
use lib '/app/tools/rps/lib';

use IO::File;
use Data::Dumper;

use Digest::MD5 qw(md5_hex);
use utf8;

use Date::Calc;
#use Storable qw(dclone);
use Clone qw(clone);

use Spreadsheet::ParseExcel;

use Common::Assert;
use Common::UTF8;
use Common::RSApp;
use Common::RSMath qw(round);
use Common::Consts;
use Common::Util qw( clean trimspaces);

use RPS::DB::Item::Expense;
use RPS::DB::Item::ExpenseName;
use RPS::DB::Item::ExpenseType;
#use RPS::DB::Item::AlbumContract;
use RPS::DB::Item::ArtistPayee;
use RPS::DB::Item::NewArtistContract;

use RPS::DB::Item::Album;
use RPS::DB::Item::Track;

use Support::Implementation::ExcelReader;
use Support::Implementation::Excel2007Reader;
use Support::Implementation::TabDelimitedReader;
use Support::Implementation::SearchUtil;

use base 'Support::Implementation::Template';

use constant kQuiet  => 0;
use constant kNormal => 1;
use constant kDebug  => 3;
my $gReportLevel = kNormal;

binmode STDOUT, ":utf8";

#-----------------------------------------------------------------------
# templateHeader maps column names to their default column number.
# The actual column number-header name map is stored in columnMap.
# Note(s):
# 1) Column names are case sensitive
# 2) Spaces are ignored; you must manually remove spaces from the names
#    stored in templateHeader
#-----------------------------------------------------------------------
my %gTemplateHeader = (
   "*Net Revenue / Expenses"  => 0,  # A
   "*Expense Type"            => 1,  # B
   "*Recoupable %"            => 2,  # C
   "*Amount %"                => 3,  # D
   "Memo"                     => 4,  # E
   "RS Contract ID"           => 5,  # F
   "Contract Title"           => 6,  # G
   "RS Album ID"              => 7,  # H
   "Catalog #"                => 8,  # I
   "Album Name"               => 9,  # J
   "RS Track ID"              => 10,  # K
   "ISRC"                     => 11,  # L
   "Track Title"              => 12,  # M
);

#--------------------------------------------------------------
# Maps column #'s to a unique key (headername).  The reverse of
# templateHeader, but with actual column #.
#--------------------------------------------------------------
my %gColumnMap = ();

#--------------------------------------------------------
# gHeaderDisplayed is a flag that we set if we've already
# displayed the template header during an exceptions dump
#--------------------------------------------------------
my $gHeaderDisplayed;

my $clientID;
my $execMode;

my $dbo;

sub new {
   my ($class, %args) = @_;
   my $self = bless {}, $class;
   return $self->_init(%args);
}


sub _init {
   my( $self, %args ) = @_;

   report("ExpenseTemplate::_init -- args = ". Dumper(\%args));

   $self->SUPER::_init(%args);

   $execMode = $self->exec_mode if ( $self->exec_mode );

   return $self;
}

sub parseHeader {
   my $self = shift;
}

#----------------------------------
# Contains map of trackIDs to ISRCs
#----------------------------------
my %gTrackISRCMap;

# Reference to SearchUtil object
my $gSearchObj;

#----------------------------------------------
# Read-in all of the template data into memory.
#----------------------------------------------
sub loadMemory {
   my $self = shift;

   $clientID = $self->client_id;
   my $app = Common::RSApp->new( clientID => $clientID );

   $dbo = Common::RSApp::GetClientDB();

   my %a = (
      clientID => $clientID
   );

   #$gSearchObj = new Support::Implementation::SearchUtil->new( %a );
   $gSearchObj = Support::Implementation::SearchUtil->new( %a );

   #-----------------------------------------------------------
   # Check if an artist run is currently queued or running.  If
   # so, abort the import.
   #-----------------------------------------------------------
   my $sql = "SELECT DISTINCT status FROM artist_royalty_run ";
   my $sth = $dbo->DoCmd($sql);
   while( my($status) = $sth->fetchrow_array() ) {
      die("ARTIST RUN IN PROGRESS - ABORTING IMPORT")       if ( $status == 0 );
      die("ARTIST RUN WAITING TO COMMIT - ABORTING IMPORT") if ( $status == 7 );
      die("ARTIST RUN IS QUEUED - ABORTING IMPORT")         if ( $status == 6 );
   }

   #------------------------
   # Fill the track-ISRC map
   #------------------------
   my $sql2 = "SELECT t.track_id, m.isrc "
      . "FROM track t JOIN master m USING(master_id) ";
   my $sth2 = $dbo->DoCmd($sql2);
   while( my($id,$isrc) = $sth2->fetchrow_array() ) {
      $gTrackISRCMap{$id} = $isrc;
   }


   my $fileName = $self->name;

   if( $self->isExcel2003( $fileName ) ) {
      print("ExpenseTemplate::loadMemory -- loading Excel2k3 $fileName into memory\n");

      # Read in the header
      my %data;
      my $reader = Support::Implementation::ExcelReader->new(
         filename => $fileName,
         data => \%data,
         header => \%gTemplateHeader,
         columnmap => \%gColumnMap
      );
      $reader->scanExcelFile;

      # Try and parse it...
      _processData(\%data);

   } elsif( $self->isExcel2007( $fileName ) ) {
      print("ExpenseTemplate::loadMemory -- loading Excel2k7 $fileName into memory\n");

      # Read in the header
      my %data;
      my $reader = Support::Implementation::Excel2007Reader->new(
         filename => $fileName,
         data => \%data,
         header => \%gTemplateHeader,
         columnmap => \%gColumnMap
      );
      $reader->scanExcelFile;

      # Try and parse it...
      _processData(\%data);


   } elsif( $self->isTabDelimited( $fileName ) ) {
      report("ExpenseTemplate::loadMemory -- processing tab-delimited file");

      my %data;
      my $reader = Support::Implementation::TabDelimitedReader->new(
         filename => $fileName,
         data => \%data,
         header => \%gTemplateHeader,
         columnmap => \%gColumnMap
      );

      $reader->scanTabbedFile;
      _processData(\%data);
   }
}

#-------------------------------------------------------------------
# _processData is where the real work is done.  It takes the generic
# information stored in the supplied array of hashes and decodes it.
# In this case, it assumes that the supplied data contains license
# data.
#-------------------------------------------------------------------
sub _processData {
   my($data) = @_;
   my $rows = $data->{rows};

   #----------------------------------
   # excount keeps track of exceptions
   #----------------------------------
   my %excount = (
   );

   #--------------------------------------
   # rowcount keeps track of line counters
   #--------------------------------------
   my %rowcount = ();

   #--------------------------------------
   # count keeps track of entities created
   #--------------------------------------
   my %count = (
      expense => 0,
      expense_type => 0,
      expense_name => 0,
   );

   use constant kColumnNetRevenueExpenses => '*Net Revenue / Expenses';
   use constant kColumnExpenseType        => '*Expense Type';
   use constant kColumnRecoupablePercent  => '*Recoupable %';
   use constant kColumnAmount             => '*Amount';
   use constant kColumnMemo               => 'Memo';
   use constant kColumnRSContractID       => 'RS Contract ID';

   use constant kColumnContractTitle      => 'Contract Title';
   use constant kColumnContractTitle_alt  => '*Contract Title';

   use constant kColumnArtistPayee        => 'Payee Name';
   use constant kColumnArtistPayee_alt    => 'Payee';
   use constant kColumnRSAlbumID          => 'RS Album ID';
   use constant kColumnCatalogNumber      => 'Catalog #';

   use constant kColumnAlbumName          => 'Album Name';
   use constant kColumnAlbumName_alt      => '*Album Name';

   use constant kColumnRSTrackID          => 'RS Track ID';
   use constant kColumnISRC               => 'ISRC';

   use constant kColumnTrackTitle         => 'Track Title';
   use constant kColumnTrackTitle_alt     => '*Track Title';

   my @columns;
   push @columns, kColumnNetRevenueExpenses;
   push @columns, kColumnExpenseType;
   push @columns, kColumnRecoupablePercent;
   push @columns, kColumnAmount;
   push @columns, kColumnMemo;
   push @columns, kColumnRSContractID;
   #push @columns, kColumnContractTitle;
   #push @columns, kColumnContractTitle_alt;
   #push @columns, kColumnArtistPayee;
   push @columns, kColumnRSAlbumID;
   push @columns, kColumnCatalogNumber;
   #push @columns, kColumnAlbumName;
   #push @columns, kColumnAlbumName_alt;
   push @columns, kColumnRSTrackID;
   push @columns, kColumnISRC;

   # 5/19/11 - Track title is only required for track-level expenses, so
   # omit these from 'columns'; this will prevent an empty track title from
   # exceptioning out. -ES
   #push @columns, kColumnTrackTitle;
   #push @columns, kColumnTrackTitle_alt;

   my $gAttachedAtTrack=0; # XXX  # of contract-not-attached-to-album-but-attached-to-track rows ...

   my @additionalRows; # if more than one contractID is being processed, then this
                       # array will hold any 'new' exception lines

   my %seenMap;

   #-----------------------------
   # Loop over each row (payee)
   #-----------------------------
   foreach my $row (@$rows) {

      # Before we do anything, go through the row and remove any embedded tabs
      #
      map { $row->{$_} =~ s/\t//g if ( $row->{$_} ) } keys %$row;

      #-----------------------------------
      # Get all of the template variables.
      #-----------------------------------
      my $rowid           = $row->{'rowid'};

      # XXX 10/9/19 - making all keys lowercase
      my $category        = $row->{ kColumnNetRevenueExpenses() };
      my $expenseType     = $row->{ kColumnExpenseType()        };
      my $recoupablePct   = $row->{ kColumnRecoupablePercent()  };
      my $amount          = $row->{ kColumnAmount()             };
      my $memo            = $row->{ kColumnMemo()               };
      my $rsContractID    = $row->{ kColumnRSContractID()       };

      my $contractTitle   = $row->{ kColumnContractTitle()      } ||
                            $row->{ kColumnContractTitle_alt()  };

      my $payeeName       = $row->{ kColumnArtistPayee()        } ||
                            $row->{ kColumnArtistPayee_alt()    };

      my $rsAlbumID       = $row->{ kColumnRSAlbumID()          };
      my $catalogNumber   = $row->{ kColumnCatalogNumber()      };

      my $albumName       = $row->{ kColumnAlbumName()          } ||
                            $row->{ kColumnAlbumName_alt()      };

      my $rsTrackID       = $row->{ kColumnRSTrackID()          };
      my $isrc            = $row->{ kColumnISRC()               };

      my $trackTitle      = $row->{ kColumnTrackTitle()         } ||
                            $row->{ kColumnTrackTitle_alt()     };

      my $digest = md5_hex( utf8::is_utf8($row) ? Encode::encode_utf8($row) : $row );

      #------------------------------------------------
      # errorCode will hold zero or more error messages
      #------------------------------------------------
      my $errorCode;

      report("#### row($rowid) ".Dumper(\%$row));
      report("  DEBUG: category($category) expenseType($expenseType) recoupablePct($recoupablePct) "
         . "amount(" . _printNull($amount) . ")");

      ++$rowcount{total};


      if( '' eq $category && '' eq $expenseType && '' eq $recoupablePct )
      {
          report("   Line $rowid is blank ... skipping");
          next;
      }

      my @expenseList; # list of expenses that get created for this row (keep track for debug purposes)

      #--------------------------------------------------------
      # Make sure all columns are in place and if the column is
      # prefixed with an asterisk, that it actually has data in
      # it.
      #--------------------------------------------------------
      my $s;
      foreach my $c (@columns) {
         $s = _checkMissingColumn( $c, $row, \%excount );
         _appendString( $errorCode, $s ) if ( $s );
      }

      #die("FATAL ERROR: One or more columns is missing!!!\n") if ( $errorCode );
      if ( $errorCode ) {
         if ( $rowid == 1 ) {
            # we're missing a column...
            die("FATAL ERROR: One or more columns is missing!!!\n");
         } else {
            report("WARNING:: Missing required data in row $rowid ...");
         }
      }


      #-----------------------
      # Remove unneeded spaces
      #-----------------------
      $rsContractID  =~ s/\s*//g if ( $rsContractID );
      $rsAlbumID     =~ s/\s*//g if ( $rsAlbumID );
      $rsTrackID     =~ s/\s*//g if ( $rsTrackID );

      $payeeName     =~ s/^\s*//g if ( $payeeName );
      $payeeName     =~ s/\s*$//g if ( $payeeName );

      $category      =~ s/^\s*//g if ( $category );
      $category      =~ s/\s*$//g if ( $category );

      $catalogNumber =~ s/^\s*//g if ( $catalogNumber );
      $catalogNumber =~ s/\s*$//g if ( $catalogNumber );

      $albumName     =~ s/^\s*//g if ( $albumName );
      $albumName     =~ s/\s*$//g if ( $albumName );

      $trackTitle    =~ s/^\s*//g if ( $trackTitle );
      $trackTitle    =~ s/\s*$//g if ( $trackTitle );

      $isrc          =~ s/^\s*//g if ( $isrc );
      $isrc          =~ s/\s*$//g if ( $isrc );

      # Clean-up
      $recoupablePct =~ s/%//;

      if ( exists $seenMap{$digest} ) {
         my $lineno = $seenMap{$digest};
         report("FYI: Row $rowid is a duplicate of line $lineno");
      }

      if ( $amount ) {
         $amount =~ s/,//;
         $amount =~ s/\$//;
         if( $amount =~ /\(/ ) {
            $amount =~ s/\(//;
            $amount =~ s/\)//;
            $amount *= -1;
         }
         $amount = trimspaces($amount);
         $amount = round($amount, 4);    # Round to 4 decimal places (this matches the DB schema for expense.amount)
         $row->{ kColumnAmount() } = $amount;
      } else {
         _appendString( $errorCode, "Missing amount" );
         ++$excount{missing_amount};
      }
 
      if ( $recoupablePct < 0 || $recoupablePct > 100 ) {
         _appendString( $errorCode, "Percent out of range" );
         ++$excount{percent_out_of_range};
      }

      if ( $amount && $amount == 0 ) {
         _appendString( $errorCode, "Zero amount" );
         ++$excount{zero_amount};
      }

      #===========================#
      #                           #
      #    Match Expense Info     #
      #                           #
      #===========================#

      #------------------------------
      # Validate the expense category
      #------------------------------
      my $preProcess;
      if ( (lc $category) eq 'net revenue' ) {
         $preProcess = 1; # 1 = net revenue deduction
      } elsif( $category =~ /^expense(s)?/i ) {
         $preProcess = 0; # 0 = expense
      } else {
         _appendString( $errorCode, "Unknown expense category '$category'" );
         ++$excount{unknown_expense_category};
      }

      #-------------------------------------------------------------------
      # Check if the expense type is valid.  Note: the name of the expense
      # is actually stored in the expense_name table, not expense_type.
      #
      # We assume that the expense name has already been setup prior to
      # trying to import expenses.
      #-------------------------------------------------------------------
      my $expenseNameID;

      if( $expenseType )
      {
         my $enObj = RPS::DB::Item::ExpenseName->Lookup( name => $expenseType );
         if ( not defined $enObj ) {
            _appendString( $errorCode, "Expense type '$expenseType' not found" );
            ++$excount{expensetype_not_found};

         } else {
            $expenseNameID = $enObj->expense_name_id;
         }
      }

      #=================================================#
      #                                                 #
      #                Match Artist Payee               #
      #                                                 #
      # Validate and match the artist payee information #
      #                                                 #
      #=================================================#
      my $artistPayeeID; # set if the payee is found
      my %artistPayeeMap;
      # 8/30/12: For Nettwerk, skip payee verification if a contractID was provided
      #if ( !($clientID == 51 && $rsContractID && ('' ne $rsContractID)) &&  $payeeName && '' ne $payeeName )
      if ( $payeeName && '' ne $payeeName )
      {
         my $sql = "SELECT artist_payee_id, name FROM artist_payee "
            . "WHERE name=" . $dbo->DBQuote($payeeName);
         my $sth = $dbo->DoCmd($sql);
         if ( $sth->rows == 0 ) {
            _appendString( $errorCode, "Payee not found" );
            ++$excount{payee_not_found};
# 11/10/09 -- Remove duplicate payee check.  We basically
# want to search for the contract using it's title.
# See Case 10456
#         } elsif( $sth->rows > 1 ) {
#            _appendString( $errorCode, "Duplicate payee" );
#            ++$excount{duplicate_payee};
         } else {

            while( my($zzPayeeID, $name) = $sth->fetchrow_array() ) {
               report("DEBUG:MATCH_ARTIST found payee($name) payeeID($zzPayeeID)");
               $artistPayeeMap{$zzPayeeID} = $zzPayeeID;
            }
         }
      } else {
         report("DEBUG:MATCH_ARTIST: no payee specified");
      }
      # MATCH ARTIST PAYEE

      #=============================================#
      #                                             #
      #               Match Contract                #
      #                                             #
      # Validate and match the contract information #
      #                                             #
      #=============================================#
      my $contractID;
      if ( $rsContractID && ('' ne $rsContractID) ) {
         # contractID specified
         my $o = RPS::DB::Item::NewArtistContract->Lookup(
            artist_contract_id => $rsContractID,
            deleted => 0,
         );
         if ( !$o ) {
            _appendString( $errorCode, "Invalid contractID" );
            ++$excount{invalid_contractid};
         } else {

            my $err;

            #if ( $artistPayeeID && $artistPayeeID != $o->artist_payee_id ) {
            #   _appendString( $errorCode, "Specified payee doesn't match payee on contract" );
            #   ++$excount{contract_payee_mismatch};
            #   $err=1;
            #}

            if ( ( (keys %artistPayeeMap) > 1 ) && (!exists $artistPayeeMap{$o->artist_payee_id}) ) { # XXX XXX XXX XXX XXX Verify - should this be > 0???
               _appendString( $errorCode, "Specified payee doesn't match payee on contract" );
               ++$excount{contract_payee_mismatch};
               $err=1;
            }

            if ( $contractTitle && $contractTitle ne $o->title ) {
               _appendString( $errorCode, "Specified contract title doesn't match title on contract" );
               ++$excount{contract_title_mismatch};
               $err=1;

               report("TITLE_MISMATCH: templateTitle($contractTitle) contractTitle(". $o->title . ")");
            }

            $contractID = $o->artist_contract_id if ( !$err );
         }

      } else {
         # contractID not specified

         if ( $contractTitle && '' ne $contractTitle ) {

            #------------------------------------------------------------
            # Look for the contract.  If an artistPayeeID is available,
            # use it to further identify the contract.  Note that you
            # may get a duplicate contract issue if the title appears
            # more than once, or if the same title/payeeID combination
            # appears more than once.  The latter condition is highly
            # unlikely, but since we don't enforce title/payee uniqueness
            # then it is something we should check for.
            #------------------------------------------------------------

            my $sql = "SELECT artist_contract_id FROM new_artist_contract "
                    . "WHERE title="
                    . $dbo->DBQuote($contractTitle)
                    . " AND deleted=0";

            #$sql .= " AND artist_payee_id=$artistPayeeID" if ($artistPayeeID);
            if ( (keys %artistPayeeMap) > 0 ) {
               $sql .= " AND artist_payee_id IN (". join(",",(keys %artistPayeeMap)) . ")";
            }

            my $sth = $dbo->DoCmd($sql);
            if ( $sth->rows == 0 ) {

               report("_EXCEPTION: contract not found '$contractTitle' :\n$sql");
               _appendString( $errorCode, "Contract not found" );
               ++$excount{contract_not_found};

            } elsif( $sth->rows > 1 ) {

               if ( $artistPayeeID ) {

                  die("FATAL ERROR: found too many contracts using title and payee name!\n$sql\n");

               } else {

                  report("_EXCEPTION: contract title not unique '$contractTitle' : sql = $sql");
                  _appendString( $errorCode, "Duplicate contract title exists - payee name required" );
                  ++$excount{duplicate_contract_title};

               }
            } else {
               ($contractID) = $sth->fetchrow_array();
            }

         } else {

            # If no contract is specified, then we'll want to attach the expense to _all_ contracts
            # tied to the album and/or track (RSD-4429).  Following is no longer an error.
            # _appendString( $errorCode, "Missing contract title" );
            # ++$excount{missing_contract_title};

         }
      }# MATCH CONTRACT


      #============================================================#
      #                                                            #
      #                       Match Album                          #
      #                                                            #
      # If an album ID was specified, make sure it's valid.        #
      # Otherwise look for the album using the title and catalog#. #
      #                                                            #
      #============================================================#
      my $albumID;

      if ( $rsAlbumID && ('' ne $rsAlbumID) ) {
         my $o = RPS::DB::Item::Album->Lookup( album_id => $rsAlbumID );
         if ( !$o ) {
            report("_EXCEPTION: albumID $rsAlbumID is invalid -- templateAlbumName($albumName)");

            _appendString( $errorCode, "Invalid albumID" );
            ++$excount{invalid_albumid};
         } else {

            my $_albumName = lc $albumName;

            my $rsAlbumVersion = '';

            if( $o->title_version ) {
                $rsAlbumVersion = lc ( $o->title . "\x{a0}(" . $o->title_version  . ")" );
            }

            if ( $albumName && '' ne $albumName &&
                 $_albumName ne (lc $o->title)  &&
                 $_albumName ne $rsAlbumVersion        ) {

               my $decorator = ($o->title_version) ? " or titleVersion($rsAlbumVersion)" : '';

               report("_EXCEPTION: templateAlbumName($albumName) doesn't match "
                  . "title(" . $o->title . ")$decorator");

               _appendString( $errorCode, "Album name mismatch" );
               ++$excount{albumname_mismatch};
            }
            if ( $catalogNumber && ('' ne $catalogNumber) && ($catalogNumber ne $o->catalog_number) ) {
               _appendString( $errorCode, "Catalog number mismatch" );
               ++$excount{catalognumber_mismatch};
            }
            $albumID = $o->album_id;
         }
      } elsif( ($albumName && '' ne $albumName) || ($catalogNumber && '' ne $catalogNumber)) {

         report("DEBUG:MATCH_ALBUM: no rsTrackID specified, searching for album...");
         #-------------------------------------------------------------------
         # albumID not specified.  Lookup the album information using the
         # title and catalog number
         #-------------------------------------------------------------------
         my ($zzAlbumID, $st) = $gSearchObj->findAlbum(
            album_name => $albumName,
            catalog_number => $catalogNumber,
         );
         if ( $st ) {

            if ( $st == Support::Implementation::SearchUtil::kAlbumNotFound ) {

               report("_EXCEPTION: album not found - title(". _printNull($albumName) .") "
                  ."cat#(". _printNull($catalogNumber) .")");
               _appendString( $errorCode, "Album not found" );
               ++$excount{album_not_found};

            } elsif ( $st && $st == Support::Implementation::SearchUtil::kNonUniqueAlbumName ) {

               report("_EXCEPTION: duplicate album($albumName) cat#(". _printNull($catalogNumber)  .")");
               _appendString( $errorCode, "Duplicate album name" );
               ++$excount{duplicate_album};

            } elsif ( $st && $st == Support::Implementation::SearchUtil::kNonUniqueCatalogNumber ) {

               report("_EXCEPTION: duplicate album($albumName) cat#($catalogNumber)");
               _appendString( $errorCode, "Duplicate catalog number" );
               ++$excount{duplicate_catno};

            } else {
               die("ERROR: Unchecked status code '$st' while calling findAlbum()\n");
            }
         } else {
            report("DEBUG:MATCH_ALBUM: found album($zzAlbumID) using title(" ._printNull($albumName) .") "
               . "cat#(". _printNull($catalogNumber) . ")");
            $albumID = $zzAlbumID;
         }

      } else {
         report("DEBUG:MATCH_ALBUM: no rsTrackID, title or cat# specified ... ");
         
      }# MATCH ALBUM

      #===========================================================#
      #                                                           #
      #                       Match Track                         #
      #                                                           #
      # If track information is not specified, then expenses will #
      # be applied at the _album level_.                          #
      #                                                           #
      # If a track ID was specified, make sure it's valid.        #
      # Otherwise look for the track using the title and ISRC.    #
      #                                                           #
      #===========================================================#
      my $trackID;

      if ( $rsTrackID && ('' ne $rsTrackID) ) {
         my $o = RPS::DB::Item::Track->Lookup( track_id => $rsTrackID );
         if ( !$o ) {
            report("_EXCEPTION: trackID $rsTrackID is invalid -- templateTrackName($trackTitle)");

            _appendString( $errorCode, "Invalid trackID" );
            ++$excount{invalid_trackid};
         } else {

            my $errors; # set if there are _any_ track errors found

            #----------------------------------------------------------------
            # Make sure the track title and ISRC match what's in the template
            #----------------------------------------------------------------

            my $_trackTitle = lc $trackTitle;

            my $rsTrackVersion = '';
# XXX N/A
#            if( $o->title_version ) {
#                $rsTrackVersion = lc ( $o->title . "\x{a0}(" . $o->title_version  . ")" );
#            }

            if ( $trackTitle && ('' ne $trackTitle) &&
                 $_trackTitle ne (lc $o->title)     &&
                 $_trackTitle ne $rsTrackVersion        ) {

               report("_EXCEPTION: templateTrackTitle($trackTitle) doesn't match trackTitle("
                  . $o->title . ")");

               _appendString( $errorCode, "Track title mismatch" );
               ++$excount{tracktitle_mismatch};
               ++$errors;
            }
            if ( $isrc && ('' ne $isrc) ) {
               if ( $isrc ne $gTrackISRCMap{$rsTrackID} ) {
                  report("_EXCEPTION: templateISRC($isrc) doesn't match trackISRC("
                     . $gTrackISRCMap{$rsTrackID} . ")");

                  _appendString( $errorCode, "Track ISRC mismatch" );
                  ++$excount{trackisrc_mismatch};
                  ++$errors;
               }
            }

            #--------------------------------------------------------------
            # Make sure the track's albumID matches any albumID information
            # derived from the album columns of the template
            #--------------------------------------------------------------
            if ( $albumID ) {
               my $zzAlbumID = $o->album_id;
               if ( $albumID != $zzAlbumID ) {

                  report("_EXCEPTION: rsTrackID($rsTrackID) is for "
                     . "albumID($zzAlbumID), but template albumID is $albumID");

                  _appendString( $errorCode, "Track albumID mismatch" );
                  ++$excount{track_albumid_mismatch};
                  ++$errors;
               }
            }


            $trackID = $rsTrackID if ( !$errors );
         }
      } else {

         #----------------------------------------------------------------
         # If we have a track title and/or isrc then try to find the track
         #----------------------------------------------------------------
         if ( ($trackTitle && '' ne $trackTitle) ||
              ($isrc && '' ne $isrc) ) {

            if ( !$albumID ) {
               report( "_EXCEPTION: found track information "
                  . "track(" . _printNull($trackTitle) . ") "
                  . "isrc(" . _printNull($isrc) . ") "
                  . "but no album specified");

               _appendString( $errorCode, "Track without album" );
               ++$excount{track_without_album};

            } else {
               my %args = (
                  album_id => $albumID,
               );
               $args{isrc}       = $isrc if ( $isrc );
               $args{track_name} = $trackTitle if ( $trackTitle );

               my($zzTrackID, $st) = $gSearchObj->findTrack( %args );

               # Did we find the track or is it exception time?
               if ( $st ) {
                  if ( $st == Support::Implementation::SearchUtil::kTrackNotFound ) {
                     report("_EXCEPTION: track(" . _printNull($trackTitle) . ") "
                        . "isrc(". _printNull($isrc) . ") not found");
                     _appendString( $errorCode, "Track not found" );
                     ++$excount{track_not_found};
                  } elsif ( $st == Support::Implementation::SearchUtil::kDuplicateTrack ) {
                     report("_EXCEPTION: duplicate track(" . _printNull($trackTitle) . ") "
                        . "isrc(". _printNull($isrc) . ")");
                     _appendString( $errorCode, "Duplicate track" );
                     ++$excount{track_not_found};
                  } else {
                     die("ERROR: Unchecked status code '$st' while calling findTrack()\n");
                  }
               } else {
                  report("DEBUG:MATCH_TRACK: found track($zzTrackID) using "
                     . "title(" ._printNull($trackTitle) .") "
                     . "isrc(". _printNull($isrc) . ")");
                  $trackID = $zzTrackID;
               }

            }
         }


      }# MATCH TRACK

      # Sanity check -- we should have an albumID and/or trackID.
      if ( !$albumID && !$trackID ) {
         _appendString( $errorCode, "No album or track found" );
         ++$excount{album_track_not_found};
         #die("row($rowid): WTF -- neither albumID or trackID is set!?\n");
         report("_EXCEPTION: row($rowid): neither albumID or trackID is set!?\n");
      }

#      # Sanity check -- we should have an albumID and/or trackID.
#      if ( !$albumID && !$trackID ) {
#         die("row($rowid): WTF -- neither albumID or trackID is set!?\n");
#      }

# XXX RSD-4429 - If we have a contractID, then we just need to find the track or album
# contract that will be linked to the expense.
# If no contract was specified in the template, then we'll be linking the expense to
# _all_ track or album contracts.
# XXX RSD-4429
#
      #--------------------------------------------------------------------------
      # If there are any errors at this point, stop processing this template line
      #--------------------------------------------------------------------------
      if ( $errorCode ) {
         $row->{'error-code'} = $errorCode;
         ++$rowcount{rows_failed};
         next;
      }


      my %contractData; # XXX RSD-4429
      # XXX For each template line, will be of the form:
      #
      #  contractData:
      #     contractID => {
      #       album_contract => [ albumContractID1, albumContractID2, ... ]
      #     },
      #     contractID => {
      #       album_contract => [ albumContractID3, albumContractID4, ... ]
      #     },
      #
      # OR
      #
      #  contractData:
      #     contractID => {
      #       track_contract => [ trackContractID1, trackContractID2, ... ]
      #     },
      #     contractID => {
      #       track_contract => [ trackContractID3, trackContractID4, ... ]
      #     },
      #
      # We'll also store the title and client_contract_id info:
      #  contractData:
      #     contractID => {
      #       title => '...',
      #       client_contract_id => '...'
      #       . . .
      #     },


      #------------------------------------------------------------
      # The contract should be attached at the album or track level
      #------------------------------------------------------------

      # The following is used if a contractID was specified in the template
      #   parentID: will be either an album_contract_id or track_contract_id
      my $parentID;

      # Keep track of the type of expenses to be created for current template line
      #   parentType:  1 = album contract, 2 = track contract
      my $parentType;

      # XXX RSD-4429: Each row in template will be for track-level _OR_ album-level expenses.
      # Hence, parentType is set once per template line (not per contract).

      if ( !$trackID ) {

         if ( $contractID ) {

            # look for album contract
            my $sql = "SELECT album_contract_id FROM album_contract "
               . "WHERE artist_contract_id=$contractID "
               . "AND album_id=$albumID";
            my $sth = $dbo->DoCmd($sql);
            if ( $sth->rows == 0 ) {
               report("_EXCEPTION: contract($contractID) is not attached to albumID($albumID)");
               _appendString( $errorCode, "Contract not attached to album" );
               ++$excount{no_album_contract};

               # XXX - let's check if the contract is attached via a track even though
               # no trackID was specified.  This is purely for reporting purposes only; the
               # template is supposed to provide track information if the contract is indeed
               # attached via a track.
               my $sql2 = "SELECT track_contract_id FROM track_contract "
                  . "WHERE artist_contract_id=$contractID "
                  . "AND track_id IN (SELECT track_id FROM track WHERE album_id=$albumID)";
               my $sth2 = $dbo->DoCmd($sql2);
               if ( $sth2->rows == 0 ) {

                  report("_EXCEPTION2: contract($contractID) is not attached to any tracks on albumID $albumID");  # this is _okay_
                  #_appendString( $errorCode, "Track not attached" );
                  #++$excount{track_not_attached};

               } elsif( $sth2->rows > 1 ) {

                  report("_WARNING: contract($contractID) is attached multiple tracks on album $albumID !!!");
                  ++$rowcount{rows_no_album_but_multi_track_matches}; # XXX
                  while( my($_trackID) = $sth2->fetchrow_array() ) {
                     report("   _WARNING: contract($contractID) not attached to album $albumID, but is attached to trackID $_trackID");
                  }

               } else {

                  my($_trackID) = $sth2->fetchrow_array();
                  report("_WARNING2: contract($contractID) not attached to album $albumID, but is attached to trackID $_trackID");
                  ++$rowcount{rows_no_album_but_track_matches}; # XXX

               }


            } elsif( $sth->rows > 1 ) {

               die("ERROR: contract($contractID) is attached multiple times to albumID($albumID)\n");

            } else {

               ($parentID) = $sth->fetchrow_array();
               $parentType = 1; # album contract

               push @{$contractData{$contractID}{album_contracts}}, $parentID;
            }

         } else {

            # No contract specified, so grab all album contracts tied to album (RSD-4429)
            #
            my $sql = "SELECT album_contract_id,artist_contract_id FROM album_contract WHERE album_id=$albumID";
            my $sth = $dbo->DoCmd($sql);
            while( my($acID, $contractID) = $sth->fetchrow_array() ) {
               push @{$contractData{$contractID}{album_contracts}}, $acID;
            }

         }

      } else {

         if ( $contractID ) {
            my $sql = "SELECT track_contract_id FROM track_contract "
               . "WHERE artist_contract_id=$contractID "
               . "AND track_id=$trackID";
            my $sth = $dbo->DoCmd($sql);
            if ( $sth->rows == 0 ) {

               report("_EXCEPTION: contract($contractID) is not attached to trackID($trackID)");
               _appendString( $errorCode, "Contract not attached to track" );
               ++$excount{no_track_contract};

            } elsif( $sth->rows > 1 ) {

               die("ERROR: contract($contractID) is attached multiple times to trackID($trackID)\n");

            } else {

               ($parentID) = $sth->fetchrow_array();
               $parentType = 2; # track contract
               push @{$contractData{$contractID}{track_contracts}}, $parentID;

            }

         } else {

            # No contract specified, so grab all track contracts tied to track (RSD-4429)
            #
            my $sql = "SELECT track_contract_id,artist_contract_id FROM track_contract WHERE track_id=$trackID";
            my $sth = $dbo->DoCmd($sql);
            while( my($tcID, $contractID) = $sth->fetchrow_array() ) {
               push @{$contractData{$contractID}{track_contracts}}, $tcID;
            }

         }
      }

print "D: contractData = ". Dumper(\%contractData) . "\n";  # XXX RSD-4429


#      die("ERROR: Multiple contract support not supported for non-RSTESTUK !!!") if ( (keys %contractData) > 1 && $clientID != 288 ); # XXX XXX XXX


      #---------------------------------------------------------------------------
      # If contract info wasn't supplied in template, and the album or track isn't
      # attached to anything, then generate an exception and stop processing line.
      #---------------------------------------------------------------------------
      if ( !$rsContractID && !$contractTitle && (keys %contractData) == 0 )  {

          report("_EXCEPTION: No contracts attached to album/track");
          _appendString( $errorCode, "No contracts attached to album/track" );
          ++$excount{album_track_not_attached};

          $row->{'error-code'} = $errorCode;
          ++$rowcount{rows_failed};
          next;
      }

      #---------------------------------------------------------------------------------
      # Error out if contract was specified but it's not attached to the specified album
      #---------------------------------------------------------------------------------
#      if ( $contractID && $albumID && !$trackID && (keys %contractData) == 0 )  {
#          report("_EXCEPTION: Contract not attached to album");
#          _appendString( $errorCode, "Contract not attached to album" );
#          ++$excount{contract_not_attached_to_album};
#
#          $row->{'error-code'} = $errorCode;
#          ++$rowcount{rows_failed};
#          next;
#      }


      #---------------------------------------------------------------------------------
      # Error out if contract was specified but it's not attached to the specified track
      #---------------------------------------------------------------------------------
#      if ( $contractID && $trackID && (keys %contractData) == 0 )  {
#          report("_EXCEPTION: Contract not attached to track");
#          _appendString( $errorCode, "Contract not attached to track" );
#          ++$excount{contract_not_attached_to_track};
#
#          $row->{'error-code'} = $errorCode;
#          ++$rowcount{rows_failed};
#          next;
#      }

      if ( $errorCode && (keys %contractData) == 0 ) {
          $row->{'error-code'} = $errorCode;
          ++$rowcount{rows_failed};
          next;
      }


      #----------------------------------------------------------------------------------
      #
      # First pass - check the contract(s) to see if there are any missing expense types.
      # Note that if a contract was specified in the template, then there will only be
      # one contract in contractData.  If no contract was specified, then contractData
      # will contain the IDs of all contracts attached to the album and/or track in the
      # template line.
      #
      #----------------------------------------------------------------------------------
      my %contractExpenseType;

      # A template line may generate multiple exceptions; if a single contract is being
      # processed (which is the case if the template has contract-specific information)
      # then any exception(s) will be output in a single exception line.  If no contract
      # was specified in the template and we're dealing with multiple contracts based
      # on how the album (or track) is attached, then any contract-specific exceptions
      # will be output on its own exception line.
      #
      my %lineError; # hash of line-level errors (cleared after each contract)

      my $lineCount=0; # of exception lines generated for the current contract; if there
                       # is only one contract then this will be one or less.  If there are
                       # multiple contracts with exceptions, then this will be more than 1.

      foreach my $contractID ( keys %contractData ) {

         #------------------------------------------------------------------------------
         # If this is a net revenue deduction, check if the contract has a default
         # term and that it's not zero percent.
         #------------------------------------------------------------------------------
         if ( $preProcess == 1 ) { # net rev
            my $sql = "SELECT artist_contract_term_id, rate "
               . "FROM new_artist_contract_term "
               . "WHERE artist_contract_id=$contractID "
               . "AND priority=0";
            my $sth = $dbo->DoCmd($sql);
            if ( $sth->rows == 0 ) {

               report("_EXCEPTION 1: netrev deduction but contract($contractID) has no default term");

               $lineError{no_default_term}{errorCode} = 'Default term cannot be 0% for net revenue expense';
               push @{$lineError{no_default_term}{contracts}}, $contractID;

            } else {

               my ( $termID, $rate ) = $sth->fetchrow_array();
               if ( 0 == $rate ) {
                  report("_EXCEPTION 2: netrev deduction but contract($contractID) has 0% default term");

                  $lineError{zero_percent_default_term}{errorCode} = 'Default term cannot be 0% for net revenue expense';
                  push @{$lineError{zero_percent_default_term}}, $contractID;
               }
            }
         }


         #-----------------------------------------------------------------
         # Find the expense_type.  Search based on the contract and expense
         # name information.  If the expense type exists, double-check the
         # percentage on the existing expense type.  If they're different
         # then it's important to make sure that this was intentional.
         #
         # 11/10/09 -- Add percent to the search criteria.
         #-----------------------------------------------------------------
         my %etArgs = (
            artist_contract_id => $contractID,
            expense_name_id    => $expenseNameID,
            inactive           => 0,
            percent            => $recoupablePct,
         );

         die("expenseNameID not set!!!") if( !$expenseNameID); # XXX
         die("expenseType not set!!!") if( !defined $expenseType); # XXX

         my $expenseTypeID;
         my $etObj = RPS::DB::Item::ExpenseType->Lookup(%etArgs);

         if ( !$etObj ) {

            print "D: ExpenseType not found: ". Dumper(\%etArgs) . "\n"; # XXX

            report(join("\t",
               "_EXCEPTION: missing expense type",
               $contractID,
               $expenseType,  # really expense_name.name
               $recoupablePct,
            ));

            $lineError{expensetype_not_setup_on_contract}{errorCode} = "Expense type not setup on contract";
            push @{$lineError{expensetype_not_setup_on_contract}{contracts}}, $contractID;

         } else {
            my $currentPercent = $etObj->percent;
            $currentPercent    = Common::RSMath::round( $currentPercent, 4 );
            $recoupablePct     = Common::RSMath::round( $recoupablePct, 4 );
            my $etID = $etObj->expense_type_id;
            if ( $currentPercent != $recoupablePct ) {

               $lineError{recoupable_pct_not_setup_on_contract}{errorCode} = 'Recoupable % is not setup on contract';
               push @{$lineError{recoupable_pct_not_setup_on_contract}{contracts}}, $contractID;

            }
            
            $expenseTypeID = $etObj->expense_type_id;
            report("   Using existing expense_type $expenseTypeID : ". Dumper(\%etArgs));

            $contractExpenseType{$contractID} = $expenseTypeID; # save this for 2nd pass
         }


         print "D[$rowid]: contractExpenseType ". Dumper(\%contractExpenseType) . "\n"; # XXX
         print "D[$rowid]: lineError: ". Dumper(\%lineError) . "\n"; # XXX

         foreach my $err (keys %lineError) {
            print "D: lineError[$rowid]: $err\n"; # XXX

            ++$excount{$err};

            my $contracts = $lineError{$err}{contracts};
            my $msg       = $lineError{$err}{errorCode};

   #         $msg .= ' (' . join(',', @$contracts) . ')'; #  if ( @$contracts > 1 );

            _appendString( $errorCode, "$msg" );

         }

         print "D: before 2nd pas - errorCode: $errorCode\n" if ( $errorCode );

         undef %lineError;


         # Ok, last chance to check for errors...
         if ( $errorCode ) {

            # If there are multiple contracts (e.g. no contract data on template line)
            # then any exception rows are appended to the template lines.  Otherwise,
            # exceptions appear on the current row.
            #
            if ( (keys %contractData) > 1 ) {

                # Fetch contract data so we can populate contract data in template
                my $c          = RPS::DB::Item::NewArtistContract->Lookup( artist_contract_id => $contractID );
                my $cTitle     = $c->title;
                my $cID        = $c->artist_contract_id;

                my $p          = RPS::DB::Item::ArtistPayee->Lookup( artist_payee_id => $c->artist_payee_id );
                my $pName      = $p->name;

                my $newrow = clone( $row );

                $newrow->{'error-code'}      = $errorCode;
                $newrow->{'RS Contract ID'}  = $cID;
                $newrow->{'*Contract Title'} = $cTitle;
                $newrow->{'Payee Name'}      = $pName;
                $newrow->{'rs-expense-id'}   = undef;
                push @additionalRows, $newrow;
                print "D: Added additional exception row\n"; # XXX

            } else {

                $row->{'error-code'} = $errorCode;

            }

            ++$rowcount{rows_failed};

            undef $errorCode;
            next;  # stop processing the current contract
         }


         # Build the list of album (or track) contracts to which the new expenses
         # will be attached

         my @parentList;
         if ( exists $contractData{$contractID}{track_contracts} ) {
            my $trackContracts = $contractData{$contractID}{track_contracts};
            foreach my $tcID (@$trackContracts) {
               print "D: contract($contractID) processing track_contract_id $tcID ...\n"; # XXX
               push @parentList, $tcID;
            }
            $parentType = 2; # track contract
         } else {
            my $albumContracts = $contractData{$contractID}{album_contracts};
            foreach my $acID (@$albumContracts) {
               print "D: contract($contractID) processing album_contract_id $acID ...\n"; # XXX
               push @parentList, $acID;
            }
            $parentType = 1; # album contract
         }


         $expenseTypeID = $contractExpenseType{$contractID};

         foreach my $parentID (@parentList) {

            # Create the expense

            my %expArgs = (
               expense_type_id => $expenseTypeID,
               parent_id       => $parentID,
               parent_type     => $parentType,
               amount          => $amount,
               percent         => $recoupablePct,
               pre_process     => $preProcess,
               processed       => 0,
            );
            $expArgs{memo} = $memo if ( $memo );

            my $expObj;
            my $expenseID;

            if ( defined $execMode ) {
               $expObj = RPS::DB::Item::Expense->Create(%expArgs);
               $expObj->save();
               $expenseID = $expObj->expense_id;
               report("   Created expense $expenseID ".Dumper(\%expArgs));
               push @expenseList, $expenseID;
               ++$count{expense};
            } else {
               report("   Non-exec mode, skipping expense creation : ".Dumper(\%expArgs));
            }

         } # parent loop


      } # contract loop (2nd pass)

      $row->{'rs-expense-id'} = join(',', @expenseList);

      ++$rowcount{rows_imported};

# XXX XXX XXX XXX XXX
#
#      last; # XXX XXX XXX XXX
#
   }#row loop



   #------------------------------------------------------------
   # Dump out the errors
   # TODO: Need to properly propagate the errors back to the user
   #------------------------------------------------------------

   foreach my $newrow (@additionalRows) {
       push @$rows, $newrow;
   }

   _showExceptions( $rows );

   #-----------------------
   # Show import statistics
   #-----------------------
   report("##### S U M M A R Y #####");
   my $totalExceptions = 0;
   my $totalRows = (scalar @$rows);
   foreach my $c (keys %excount) {
      my $v = $excount{$c};
      printf("%30s %6d\n", $c, $v);
      $totalExceptions += $v;
   }
   printf("%30s %s\n", " ", "-------" );
   printf("%30s %6d\n", "Total Exceptions", $totalExceptions );
   printf("%30s %6d\n", "Total Rows", $totalRows );
   report(" ");
   
   foreach my $c (keys %count) {
      my $v = $count{$c};
      printf("%30s %d\n", $c, $v);
   }

   report(" ");
   foreach my $c (keys %rowcount) {
      my $v = $rowcount{$c};
      printf("%30s %d\n", $c, $v);
   }

   if (!$execMode)
   {
       print STDERR ">>> Test complete.  Run 'make import' to commit changes\n";
   }
   else
   {
       print STDERR ">>> Import complete.  Attach exceptions report to FogBugz case.\n";
   }
   print STDERR ">>>\n\n";
}#_processData

#------------------------------------------------------------------
# _checkMissingColumn - checks if a column is missing or not.  Also
# ensures that a required columns (prefixed with an asterisk) has
# data in it.  A string containing error(s) is returned, otherwise
# undef is returned if column is OK.
#------------------------------------------------------------------
sub _checkMissingColumn {
   my($colName, $rows, $excount) = @_;
   my $errStr;
   if ( ! exists $rows->{$colName} ) {
      report("_checkMissingColumn -- column '$colName' is missing");
      _appendString($errStr, "Column '$colName' not found");

      my $colClean = clean($colName);
      my $key = "column_" . $colClean . "_not_found";
      ++$excount->{$key};
   }

   #------------------------------------------------------------------
   # If the column name starts with an asterisk then it can't be blank
   #------------------------------------------------------------------
   if ( $colName =~ /^\*/ ) {
      my $v = $rows->{$colName};
      # strip out leading spaces
      $v =~ s/^\s*//g if ( $v );
      if ( !$v || '' eq $v ) {
         _appendString($errStr, "Column '$colName' can't be blank");
      }
   }
   return $errStr;

}# _checkMissingColumn


#--------------------------------------------------------------------------
# _showExceptions was originally intended to _just_ show the template lines
# that exceptioned out.  It's been modified to output both exception and
# non-exception lines.  Exception message(s) will be placed in the error
# code column.  If a line is imported successfully, then the resulting
# expenseID will be stored in the import status column ("expenseID(###)"),
# otherwise this column will contain the string "__FAIL__".
#--------------------------------------------------------------------------
sub _showExceptions {
   my ( $rows ) = @_;

   #---------------------------
   # Build the exception header
   #---------------------------
   my @header;
   for( my $i = 0; $i < (keys %gColumnMap); $i++ ) {
      # Get the key at the specified column
      my $v = $gColumnMap{$i};

      #----------------------------------------------------------
      # Do _not_ push the "Error Code" or "import-status" columns
      # if they were in the original template.  We'll re-create
      # them on the fly.
      #----------------------------------------------------------
      next if ( ("Error Code" eq $v) || ("import-status") eq $v );

      push @header, $v;
   }

   report("STATUS:\t".join("\t", @header, "Error Code", "import-status"));

   #--------------------------------------
   # Now dump out the rows that had errors
   #--------------------------------------
   foreach my $row (@$rows) {
      my $rowid = $row->{rowid};
      my $errorCode = $row->{'error-code'};
      my $expenseID = $row->{'rs-expense-id'} || '---';

      #----------------------------------------
      # Output the row data in the proper order
      #----------------------------------------
      my @obuf;
      for( my $i = 0; $i < (keys %gColumnMap); $i++ ) {
         # Get the key at the specified column
         my $v = $gColumnMap{$i};
         my $val = ($row->{$v}) ? $row->{$v} : '';

         #----------------------------------------------------------
         # Do _not_ push the "Error Code" or "import-status" columns
         # if they were in the original template.
         #----------------------------------------------------------
         next if ( ("Error Code" eq $v) || ("import-status") eq $v );

         push @obuf, $val;
      }

      my $importStatus = ($errorCode) ? "__FAIL__" : "expense($expenseID)";

      my $ecString = ($errorCode) ? $errorCode : '';

      my $zzbuf = join("\t", "STATUS:", @obuf, $ecString, $importStatus);

      report($zzbuf);
   }
}#_showExceptions

#---------------------------------------------------
# _isValidDateFmt - return true if the supplied date
# is in the format YYYY-MM-DD, false otherwise
#---------------------------------------------------
sub _isValidDateFmt {
   my($dstr) = @_;
   my($yr,$mo,$dy) = split("-",$dstr);
   my $st = 1; # valid unless we detect otherwise

   $st = 0 if ( !$yr || !$mo || !$dy );
   $st = 0 if ( $mo && ( $mo !~ /^\d+$/ ));
   $st = 0 if ( $yr && ( $yr !~ /^\d+$/ ));
   $st = 0 if ( $dy && ( $dy !~ /^\d+$/ ));
   $st = 0 if ( length($yr) != 4 );
   return $st;
}

sub _appendString {
   my($str,$v) = @_;

   if ( $str ) {
      my $cur = $str;
      my $newstring = "$cur; $v";
      $_[0] = $newstring;
   } else {
      $_[0] = $v;
   }
   return;
die("str($str) v($v)") if ( !$str );

   my $cur = $str;
   my $prefix = "$cur; $v";
   $_[0] = ($str) ? $prefix : $v;
}# _appendString

sub _reportError {
   my ($name, $obj) = @_;

   report("   _reportError: checking '$name' for errors");
   if ($obj && $obj->_hasError()) {
      #my ($e, $msg) = $obj->getError();
      #print "$name has an error: $e : $msg\n";

      my %xmlParams = $obj->getXMLParams();
      my $msg = defined $xmlParams{emsg} ? $xmlParams{emsg} : "MSG_NOT_AVAILABLE";
      my $e = $xmlParams{e};
      print "$name has an error:: $e :: $msg\n";
      return 1;
   }
   return undef;
}

sub _printNull {
   my($s) = @_;
   return $s ? $s : "NULL";
}

sub report {
   my($text, $level) = @_;
   $level = kNormal unless $level;
   if ( $level <= $gReportLevel ) {
      print $text . "\n";
   }
}

1;