## THIS SHOULD REALLY BE A PART 3? 
## GL_pt2_d Append Known Totals and Ancillary.r

## create and prep DT.adjustments for rbinding to DT.merged


## Variables used for this file
{
  adjustment_storeinfo <- list(storeid = -1, store_name = "Accounting Adjustment")

  ## We will sum up stores by the following
  adjustBy      <- kCols.smry_nostore
  ## Not all groups will have a reported value. We will aggregate the adjustments by
  adjust_agg_By <- kCols.date_SC

  ## Column in DT.merged 
  colName_being_adjusted <- "GL_net_forDSOM"
}

## THESE SHOULD BE IDENTICAL
if (!all(equals(DT.mainline_group_subtotals_from_rev_summary, music_and_video_bucketcleanup_(copy(DT.mainline_group_subtotals_from_rev_summary)), na.check=TRUE))) {
  warning ("not all Equal for DT.mainline_group_subtotals_from_rev_summary")
  notifyAndEmail("2d: DT.mainline_group_subtotals_from_rev_summary NOT ALL EQUAL")
}
# something is wrong with equals() when there are NAs in
if (!all(equals(DT.merged, music_and_video_bucketcleanup_(copy(DT.merged)), na.check=TRUE))) {
  warning ("not all Equal for DT.merged")
  notifyAndEmail("2d: DT.merged NOT ALL EQUAL")
}

## Create DT.adjustments
{
  DT.adjustments <- {
         merge(
            DT.mainline_group_subtotals_from_rev_summary[ , list(reported   = sum(reported_gross)), keyby=adjustBy]
          , DT.merged[ date <= lastDateClosed    , list(calculated = sum(GL_net_forDSOM)), keyby=adjustBy]
          , all=TRUE
    )}

  ## There should be no NAs in calculated
  stopifnot(DT.adjustments[, !is.na(calculated)])
  DT.adjustments[is.na(calculated)]

  ## NOTE: there will be NAs in reported. Do NOT remove them.  
  ##       The NAs will let us know where to place the adjustments

  ## CONFIRM: There should be at least one non-NA value per each group in adjust_agg_By
  stopifnot(DT.adjustments[, length(removeNA(reported)) >= 1, by=adjust_agg_By][, V1])

  ## < see > 
  DT.adjustments[, length(removeNA(reported)) , by=adjust_agg_By]

  invisible()
}

## Calculate the Adjustment
{
  ## NOTE TO SELF:  all the code with 'unaggd_adjust_to_add' is NOT needed. They exist simply as an additional check
  ## Calculate the adjustment on a row level
  DT.adjustments[, unaggd_adjust_to_add := removeNA(reported, 0) - calculated]

  ## CONFIRM:  The sum of the unaggd_adjust_to_add should equal difference of the sums of reported and calculated
  stopifnot(DT.adjustments[, lapply(.SD, sumn), keyby=adjust_agg_By, .SDcols=c("reported", "calculated", "unaggd_adjust_to_add")][, equals(reported - calculated, unaggd_adjust_to_add)])

  DT.adjustments[, adjust_to_add := sumn(reported) - sum(calculated), by=adjust_agg_By] 

  ## the adjustment will be applied to only one row per group.  All the other rows, convert to 0.
  ## We chose the line based on the largest reported value in the group
  ##  NOTE:     DT.adjustments[, .I[which.max(reported)], by=adjust_agg_By]$V1
  ##        identifies which row of DT.adjustments contains max(reported), by group
  ##        We then check that against the original .I in DT.adjustments to get a TRUE / FALSE
  DT.adjustments[, row_will_be_adjusted := (.I %in% DT.adjustments[, .I[which.max(reported)], by=adjust_agg_By]$V1) ]

  ## The rows being kept should NOT have NA in adjust_to_add
  stopifnot(DT.adjustments[(row_will_be_adjusted), !is.na(adjust_to_add)])

  ## All rows NOT being adjusted should have adjust_to_add as 0
  DT.adjustments[!(row_will_be_adjusted), adjust_to_add := 0]

  invisible()
}

## Cleanup for rbind'ing
{

  ## NOTE To self: I am going back on forth on whether to use 
  ##           unaggd_adjust_to_add   or   adjust_to_add
  ##  I was thinking of using the former, since ultimately, we are simply adding 
  ##     rows to DT.merged and it's straightforward enough to use the unagg'd value. 
  ##
  ##  HOWEVER:  The key issue (why we use the agg'd value to begin with ) is that
  ##     there are some meta-combos which are not in the reported. While eventually 
  ##     this should be fixed, in the meantime, if we were to use the unagg'd value,
  ##     then all of these meta-combos will show up as 0 in the mgmt report, since 
  ##     they will be canceled out fully. 
  ##  THEREFORE: It makes more sense to tally up all of the adjustments and apply it
  ##     to the largest value in the group, for which the adjustment would be (in most cases)
  ##     a much smaller percentage. 
  ##
  ## See, for example, Video-Music-TRUE represents Comps sold through YouTube 
  ##         formnumb(DT.adjustments[label_sc_group == "Orchard"], round=-1)

  ## Creating a new column, allows this to easily be changed in the future, if needed.
  adjustCol_using <- "adjust_to_add"
  DT.adjustments[, c(colName_being_adjusted) := get(adjustCol_using)]

  ## Add in the store info
  DT.adjustments[, names(adjustment_storeinfo) := adjustment_storeinfo]

  ## Cleanup the columns that will not be rbound in to DT.merged
  tmp.cols_not_coming <- c("calculated", "reported", "row_will_be_adjusted"
                          ,  "unaggd_adjust_to_add", "adjust_to_add")
  suppressWarnings(DT.adjustments[, (tmp.cols_not_coming) := NULL])

  setkeyIfNot(DT.adjustments, kCols.smry, organize=TRUE, verbose=FALSE)
}