UPDATE:   The forecast portion of this file was copied to (what is currently 2_e) to create DT.ancillary_with_forecasts

Most of the rest of this is no longer needed. 

There is a ggplot() here that could be useful, to visualize the forecasts. 




## CREATE DT.manl_crop which is a copy of DT.manual with DT.ancillary_from_rev_summary merged in

## This file takes the reported_gross value from  DT.ancillary_from_rev_summary and 
## (1) Forecasts reported_gross for the dates not yet reported but already in OA
## (2) Overwrites the estimate_current_month value in DT.manl_crop with the reported_gross value

{
  ## Start from DT.manual (the ingestion of the manual templates), dropping un-needed columns
  {
    DT.manl_crop <- copy(DT.manual)
    DT.manl_crop[, c("item", "description", "budget_last_modified", "updated") := NULL]
    ## These next columns no longer expected in the templates, but just incase, drop them
    suppressWarnings(DT.manl_crop[, c("trueup_from_previous_month", "yago") := NULL])

    ## Instead of  grepl("^zz_", store_name)   use  !is.na(ancillaryline)
    stopifnot(DT.manl_crop[, identical(grepl("^zz_", store_name), !is.na(ancillaryline))])

    invisible()
  }

  ## Merge in the reported_gross column from DT.ancillary_from_rev_summary
  {
    matchKey(DT.ancillary_from_rev_summary, DT.manl_crop, kCols.smry)
    nrow(DT.ancillary_from_rev_summary)

    ## FOR DEV'ING: Clear 'reported_gross' column
    suppressWarnings(DT.manl_crop[, reported_gross := NULL])

    ## Merge in reported_gross column 
    DT.manl_crop[DT.ancillary_from_rev_summary, reported_gross := i.reported_gross]
    ## CONFIRM: There should NOT be any NAs in reported_gross for the zz_ stores for dates less than lastDateClosed
    stopifnot(DT.manl_crop[date <= lastDateClosed & !is.na(ancillaryline), !(is.na(reported_gross ))])
  }


  ## FORECAST future values
  {
    method_used_for_forecasting <- "all"
    forecasts_manual_from_rev_summary.ll <- emptylist(dates_not_in_GL, nm=as.character(dates_not_in_GL))

    ## We need to create forecasts FOR EACH DATE in dates_not_in_GL
    for (date.target in dates_not_in_GL) {

      date.target <- as.Date(date.target)

      ## We will test which is the better forecast
      forecasts_manual_from_rev_summary.ll[[as.character(date.target)]] <- 
                  mkForecast( DT=DT.ancillary_from_rev_summary ## GL_firstDate_forS will be NA when  !(id_is_in_GL)
                            , valueCol="reported_gross"
                            , byCols=kCols.store
                            , date_diff="month"
                            , date.target=date.target
                            , months.to.test=4
                            , method=method_used_for_forecasting
                            , verbose=FALSE
                            , fail.on.duplicates=FALSE
                            )

        ## bank the forecast method
        ## Also, Apply the same method to all future dates, after the first test
        method_used_for_forecasting <- attr(forecasts_manual_from_rev_summary.ll[[as.character(date.target)]], "method")
    } #// end for loop
  
    ## Flatten the list
    DT.forecasts_manual_from_rev_summary <- rbindlist(forecasts_manual_from_rev_summary.ll)
  }

  ## Merge the forecasted values back in
  {
    matchKey(DT.forecasts_manual_from_rev_summary, DT.manl_crop, kCols.datestore, superset.ok=TRUE)
    DT.manl_crop[DT.forecasts_manual_from_rev_summary, reported_gross := i.forecastedValue]

    # if (exists(".Pfm") && identical(.Pfm, "Darwin")){
    #   ggplot2::ggplot(data=DT.manl_crop[!is.na(ancillaryline)], ggplot2::aes(x=date, y=reported_gross, color=store_name)) + ggplot2::geom_line()
    #   dev.new()
    #   ggplot2::ggplot(data=rbind(DT.ancillary_from_rev_summary, copy(DT.forecasts_manual_from_rev_summary)[, reported_gross := forecastedValue], fill=TRUE)[!is.na(ancillaryline)], ggplot2::aes(x=date, y=reported_gross, color=store_name)) + ggplot2::geom_line()
    # }

    invisible()
  }

  ## For ancillary, REPLACE the estimate_current_month value with the reported_gross value
  {
    ## There should be no NAs for reported estimate_current_month for the unnamed stores
    stopifnot(DT.manl_crop[!is.na(ancillaryline), !is.na(reported_gross)] )

    ## have reported_gross replace the estimate_current_month, drop the reported_gross column
    DT.manl_crop[!is.na(ancillaryline), estimate_current_month := reported_gross][, reported_gross := NULL]
    invisible()
  }
}