#  ## NOTE:  Since we are calculating GPU,  
#  ##    We don't need to worry as much about missing values
#  setkeyIfNot(DT.storeavg, c("store", "transac_typeid", "date"))
#  allVals <- setkey(DT.storeavg[, seq(min(date), max(date), by="month"), by=c("store", "transac_typeid")])
#  DT.MissingDates <- DT.storeavg[allVals][is.na(GPU)]

lib(forecast)


#  store - transac_typeid - Month - GPU_Stat - GPU_Est_Fore - GPU_Est_MA
#  -------------------------------------------------------------------

setkeyIfNot(DT.storeavg, c(kCols, "date"))

if (nrow(DT.storeavg[gross==0]) / nrow(DT.storeavg) < .001)
   DT.storeavg <- DT.storeavg[gross!=0]


AllDates <- DT.storeavg[, seq(min(date), max(date), by="month")]
for (D in rev(AllDates) ) {
    DT.storeavg[date <= D 
                # & !is.na(GPU)
      , `:=`( GPU_Est_Fore = if (length(GPU) < 3) NA_real_ else forecast(tail(GPU, 5), 1)[["mean"]],
              GPU_Est_MA   = mean(tail(GPU, 3), na.rm=TRUE)
            )
      , by=kCols
    ]
}

## Add a month, by taking a two-month seq and banking just the last value
DT.nextmonth <- setkey(DT.storeavg[, list(date=seq(max(date), length.out=2, by="month")[[-1L]]), keyby=kCols])
DT.storeavg2 <- setkeyv(rbind(DT.storeavg, DT.nextmonth, fill=TRUE), key(DT.storeavg))

## Confirm that the last row in each store-trans is NA
stopifnot(DT.storeavg2[, is.na(tail(GPU, 1)), by=kCols]$V1)

## Shift the ESTs down one row (ie, they are for the future, not for the present)
DT.storeavg2[, GPU_Est_MA   := c(NA, head(GPU_Est_MA,   -1)), by=kCols]
DT.storeavg2[, GPU_Est_Fore := c(NA, head(GPU_Est_Fore, -1)), by=kCols]

## Calculate the diff between estimates and stat
DT.storeavg2[, `:=`(err.fore = (GPU - GPU_Est_Fore)/GPU,
                    err.ma   = (GPU - GPU_Est_MA)  /GPU
                    )]
DT.storeavg2[, MA_better_than_Fore := abs(err.ma) <= abs(err.fore)]

DT.storeavg2[
  DT.storeavg2[date > Sys.Date() - 129
              , list(use_MA = .5 <= sum(MA_better_than_Fore, na.rm=TRUE) / sum(!is.na(GPU)) | is.na(MA_better_than_Fore))
              , keyby=kCols
              ]
, use_MA := i.use_MA
, allow.cartesian=TRUE
]

## CREATE the GPU.EST data.table.  This is what will ultimately make it into the DB
setkeyIfNot(DT.storeavg2, key(DT.nextmonth))
DT.GPU.Est <- DT.storeavg2[DT.nextmonth, list(date, store, transac_typeid, label_sc_group, GPU_estimate=ifelse(use_MA, GPU_Est_MA, GPU_Est_Fore))]

## Some values are still NA.  Take the last known value
##    ... find the NAs,  order by date, take the last value. 
DT.GPU.Fillers <- DT.storeavg2[ DT.GPU.Est[is.na(GPU_estimate), list(store, transac_typeid)]  
                              ][!is.na(GPU)
                              ][order(date)
                                , list(GPU_estimate_filler=tail(GPU, 1))
                                , keyby=kCols]

## Match keys for filling
matchKey(DT.GPU.Est, DT.GPU.Fillers, key(DT.GPU.Fillers))

## FILL by adding a column, then any NAs get filled, confirm all is good, then drop the column
DT.GPU.Est[DT.GPU.Fillers, GPU_estimate_filler := GPU_estimate_filler]

DT.GPU.Est[is.na(GPU_estimate), GPU_estimate := GPU_estimate_filler]
## Confirm there are no NAs for the estimates
stopifnot(DT.GPU.Est[, !is.na(GPU_estimate)])
## Drop the column added
DT.GPU.Est[, GPU_estimate_filler := NULL]