## Calculate the percent splits, according to different groupings
## PARAMETERS
grpCols <- list(Meta = cols.to.spliton, SC = "label_sc_group", SC_Meta = kCols.SC_meta)
avgCols.store <- sprintf("avg_perc_OfStoreTotal_by%s", names(grpCols))
avgCols.datestore <- sprintf("avg_perc_OfDateStoreTotal_by%s", names(grpCols))
## ITEREATE
for (i in seq(grpCols)) {
  calcPerc_ (DT.split_percs, newCol.nm=avgCols.store[[i]],     groupingCols=grpCols[[i]], valueCol="OA_gross_forDSOM", parentGroup=kCols.store,     totalCol="TotalForStore")
  calcPerc_ (DT.split_percs, newCol.nm=avgCols.datestore[[i]], groupingCols=grpCols[[i]], valueCol="OA_gross_forDSOM", parentGroup=kCols.datestore, totalCol="TotalForDateStore")
}
if (FALSE)
  DT.split_percs[, TotalForStore := NULL]

DT.split_percs[date >= firstSCdate, unique(avg_perc_OfStoreTotal_bySC), by=kCols.main][, sum(V1), by=kCols.store] [!equals1(V1)]
DT.split_percs[date < firstSCdate,  unique(avg_perc_OfStoreTotal_bySC), by=kCols.main][, sum(V1), by=kCols.store] [!equals1(V1)]


{
  ## CONFIRM: DateStore averages sum to one
  sumsToOne(DT.split_percs[, unique(avg_perc_OfDateStoreTotal_byMeta),    by=kCols.splitgroup_noSC ], "V1", by=kCols.datestore)
  sumsToOne(DT.split_percs[, unique(avg_perc_OfDateStoreTotal_bySC),      by=kCols.main            ], "V1", by=kCols.datestore)
  sumsToOne(DT.split_percs[, unique(avg_perc_OfDateStoreTotal_bySC_Meta), by=kCols.splitgroup      ], "V1", by=kCols.datestore)
  ## CONFIRM: Store averages sum to one 
  sumsToOne(DT.split_percs[, unique(avg_perc_OfStoreTotal_byMeta),    by=kCols.store_meta        ], "V1", by=kCols.store)
  sumsToOne(DT.split_percs[, unique(avg_perc_OfStoreTotal_bySC),      by=kCols.storeSC           ], "V1", by=kCols.store)
  sumsToOne(DT.split_percs[, unique(avg_perc_OfStoreTotal_bySC_Meta), by=kCols.splitgroup_nodate ], "V1", by=kCols.store)
  ##
  ## REMINDER TO SELF
  ##  if we look at one of the 'avg_perc_OfStoreTotal_by..' columns, and we are looking at an isolated month, then it is possible that 
  ##    the percentage will NOT sum to 1.  Why?  Becuase this column represents the store's average for ALL time (or at least a larger range than one month)
  ##    Specifically, if we are looking at, say, date < firstSCdate, then we will only be seeing about 80% worth, since the SC's will not be showing up yet.

  ##### COLUMN .......................... :: ... "WHAT PERCENT OF THE STORE'S ... 
  #  avg_perc_OfDateStoreTotal_byMeta     :: MONTHLY total does [  each meta-value combination    ] represent 
  #  avg_perc_OfDateStoreTotal_bySC       :: MONTHLY total does [  each supply chain              ] represent 
  #  avg_perc_OfDateStoreTotal_bySC_Meta  :: MONTHLY total does [  each sc-meta-value combination ] represent 
  #  avg_perc_OfStoreTotal_byMeta         :: FOREVER total does [  each meta-value combination    ] represent 
  #  avg_perc_OfStoreTotal_bySC           :: FOREVER total does [  each supply chain              ] represent 
  #  avg_perc_OfStoreTotal_bySC_Meta      :: FOREVER total does [  each sc-meta-value combination ] represent 
  #
  #  The different columns will fill the NAs depending on how/where-from the NAs got into DT.merged 
}


  ## CONFIRM: NO NAs anywhere
  stopifnot(!is.na(  DT.split_percs))

  && TODO:  Move this to the check after 
  if (FALSE)  {
    DT.split_percs[, month_is_zero_in_OA := {sum(OA_gross_forDSOM) == 0}, by=kCols.datestore]
    ## CONFIRM: When the month is none-zero, perc and subtotal are either BOTH zero, or BOTH non-zero
    stopifnot(DT.split_percs[!(month_is_zero_in_OA), {perc_of_store_total_bySCMeta == 0} == {OA_gross_forDSOM == 0}])
    ## CONFIRM: Similarly, if subtotal is zero but not the whole month is zero, then perc MUST be zero as well.
    stopifnot(DT.split_percs[OA_gross_forDSOM == 0 & !month_is_zero_in_OA, equals0(perc_of_store_total_bySCMeta)])


  }
  setkeyIfNot(DT.split_percs, kCols.main, verbose=FALSE)
}


"&&:"     ## apply this somewhere
"&&:"     
"&&:"     setkeyv(DT.split_percs.unexpanded, kCols.main)
"&&:"     DT.split_percs.unexpanded[.(Aug, 580)]
"&&:"     DT.split_percs[.(Aug, 580)]
"&&:"     DT[, (avgCol.nm) := lapply(.SD, perc_func), .SDcols=valueCol, by=byCols]
"&&:"     .us()
"&&:"     Q
"&&:"     
"&&:"     (DT <- DT.merged[storeid%in% c(1, 571)])
"&&:"     (DT <- DT.split_percs[storeid%in% c(1, 571)])


  ## This extra column is not needed, but allows for changing the perc_split being used, without much code manipulation
  DT.split_percs[, perc_split := avg_perc_OfStoreTotal_bySC_Meta ]


  ## Ideally, every store that is in OA should also be in the GL. 
  ## However, this may not always be true -- generally the reason for this not being true
  ##  can be that a store is going by a different name, or some other purpose. 
  ## We will eventually add another mapping. 
  ## In the interim, we identify these stores, output it to a file that someone can then investigate
  DT.split_percs[, id_is_in_GL := storeid %in% DT.mgmtr_summary_tall$storeid ]

  if (any(DT.split_percs[, !id_is_in_GL])) {
        cat("There are", DT.split_percs[(!id_is_in_GL), lunique(storeid)], "OA IDs missing from the GL\n")

        f.missing_from_GL <- out.p("present_in_OA_but_missing_in_GL", ext="csv")
        write.csv(DT.split_percs[(!id_is_in_GL), list(Total_Gross = sum(OA_gross_forDSOM), first_appear=min(date), last_appear=max(date)), keyby=list(storeid, store_name)]
                , f.missing_from_GL, row.names=FALSE)

        message("Missing OA IDs have been outputed to \n\t  ", f.missing_from_GL, "\n")  
  }

#...     
#...     ## There Will be NAs in the number columns from OA, correspoding to months were the row's combination of meta values did not naturally appear in OA
#...     ## The NAs in these value cols will be filled by key (ie kCols.main) 
#...     ## The fillers will be chosen in this order, moving on to each subsequent if the NA persists
#...     ## ties within a choice are broken by highest perc_split
#...     First, any other instance in key +- 3 months, 
#...     Second, for allegro 
#...     
#...      
#...     
#...     DT <- DT.merged[storeid==571 & date==May]
#...     DT <- DT.merged[storeid%in% c(1, 571)]
#...     DT[, sum(perc_split), keyby=kCols.datestore][, all(equals1(V1))]
#...     DT[is.na(perc_split)]
#...     
#...     valueCol="OA_gross_forDSOM"
#...     temp_DT.NA_perc_split_fill.DS  <- DT[, lapply(.SD, sumn), keyby=kCols.splitgroup_nodate, .SD=valueCol][, perc_split_when_NA := get(valueCol) / sum(get(valueCol)), keyby=kCols.store][, (valueCol) := NULL] 
#...     temp_DT.NA_perc_split_fill.DSO <- DT[, lapply(.SD, sumn), keyby=kCols.splitgroup_nodate, .SD=valueCol][, perc_split_when_NA := get(valueCol) / sum(get(valueCol)), keyby=kCols.main_nodate][, (valueCol) := NULL] 
#...     formnumb(copy(temp_DT.NA_perc_split_fill.DS)[, perc_split_when_NA := fwp(perc_split_when_NA)][], round=-3)
#...     formnumb(copy(temp_DT.NA_perc_split_fill.DSO)[, perc_split_when_NA := fwp(perc_split_when_NA)][], round=-3)
#...     
#...     jsdklfjasdkkdsfkl;ajdsfsdkljfklsdajfkl
#...     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.s(storeid==571 & date==May)
{

  ## Match the keys for merging
  matchKey(DT.mgmtr_summary_tall, DT.split_percs, key=kCols.main, superset.ok=FALSE, verbose=FALSE)

  cols.not.merging <- c("ASR_percof_DS_forDSO", "margin")
  ## Allow cartesian specifically for those stores with dates not in the GL (either too early or too late)
  DT.merged <- merge(DT.mgmtr_summary_tall[, !cols.not.merging, with=FALSE], DT.split_percs, all=TRUE, allow=TRUE) [date >= minDateToKeep]
  ## create column of proper type
  DT.merged[, perc_split := NA_real_]

  message("Beginning permeates ..   Can take about 20~30 seconds.")
  na_count_before <- colSums(is.na(DT.merged))

  {

    ## List of columns that will need permeating and how the groups are grouped by
    colsToPermeate.ll <- list(
        'kCols.store'     = list(by=kCols.store,      cols=c("id_is_in_GL", "id_is_in_OA", "music_vs_video_by_dmv", "storeid_char", "GL_firstDate_forS"))
      , 'kCols.datestore' = list(by=kCols.datestore,  cols=c("GL_gross_forDS",  "gross_last_modified"))
      , 'kCols.main'      = list(by=kCols.main,       cols=c("total_netsc_by_DSO", "GL_gross_forDSO"))
      , 'kCols.storeSC'   = list(by=kCols.storeSC,    cols=c("avg_perc_OfStoreTotal_bySC"))
      , 'kCols.splitgroup_nodate'   = list(by=kCols.splitgroup_nodate,    cols=c("avg_perc_OfStoreTotal_bySC_Meta"))
      # , list(by=kCols.date_SC,    cols=c("perc_split_as_reported", "margin_for_perc_split"))
    )

    cat("Permeating group ")
    for (i in names(colsToPermeate.ll)) {
      cat(i, "....  ")
      for (col in colsToPermeate.ll[[i]][["cols"]] )
        permeate_ (DT.merged, col, by=colsToPermeate.ll[[i]][["by"]], na.rm=TRUE)
    }
    cat(" done.\n")
  }

~~~~~~ The problem is that the month does NOT exist in OA ~~~~~~~~
NA types:
(1) the entire month is missing for a given store:   .s(date == "2013-09-01" & storeid == 497)
    fill the meta with default values
    fill the perc_split

## CONFIRM: When the whole month's meta is NA, there are exactly four rows, and they are the four OSC factor levels
stopifnot(DT.merged[, if (all(is.na(music_vs_video))) {.N == 4} && all(oscFactorLevels %chin% label_sc_group) else TRUE, by=kCols.datestore][, V1])

## Since the above is confirmed, we can designate which have the whole month NA and need to be filled with defaults
if ("whole_DS_is_NA" %ni% names(DT.merged)) ## for deving
  DT.merged[, whole_DS_is_NA := all(is.na(music_vs_video)), by=kCols.datestore]
  
## Fill default meta values
DT.merged[(whole_DS_is_NA), names(default.metavalues) := default.metavalues]

DT.merged[(whole_DS_is_NA)]
DT.merged[is.na(music_vs_video)]
sumsToOne(DT.merged[(whole_DS_is_NA)], "avg_perc_OfStoreTotal_bySC",      by=kCols.datestore, treat.entire.group.NA.as=FALSE)
sumsToOne(DT.merged[(whole_DS_is_NA)], "avg_perc_OfStoreTotal_bySC_Meta", by=kCols.datestore, treat.entire.group.NA.as=FALSE)


DT.merged[(whole_DS_is_NA)][!is.na(avg_perc_OfStoreTotal_bySC)]

DT.merged[(whole_DS_is_NA), perc_split]

DT.merged[is.na(GL_firstDate_forS)]
DT.merged[(storeid == 497)][, table(id_is_in_GL, date, useNA="always")]
DT.merged[.(May,  497)]
.s(date == "2013-09-01" & storeid == 497)

  ## CONFIRM: perc_splits sum to 1
  sumsToOne(DT.merged, "perc_split", by=kCols.datestore)

  setcolorderpt(DT.merged, kCols.splitgroup, end=c("id_is_in_GL", "id_is_in_OA"))

  ## CONFIRM: perc_split should be NA for non-Orchard SC's prior to firstSCdate
  stopifnot(DT.merged[date < firstSCdate & label_sc_group != "Orchard", is.na(perc_split)])

  ## Make the perc_split $0 for SC's prior to firstSCdate
  DT.merged[date < firstSCdate & label_sc_group != "Orchard", perc_split := 0]


  temp_DT_stores_to_set_to_one <- DT.merged[, list(.N, Zeros=sumn(perc_split == 0), NAs=sum(is.na(perc_split))), keyby=kCols.datestore] [ (NAs == 1) & Zeros + NAs == N]
  # DT.merged[temp_DT_stores_to_set_to_one, perc_split := ifelse(label_sc_group == "Orchard" & is.na(perc_split), 1, perc_split)]

  ## add in a flag to indicate which date-stores might be just orchard, and should be set to 1. This extra step (column) is necessary, because cartesian join does not line up when filtering
  DT.merged[temp_DT_stores_to_set_to_one, justOrchard_possibly := TRUE]

  DT.merged[justOrchard_possibly & label_sc_group=="Orchard", perc_split := 1]

  stopifnot(DT.merged[, unique(perc_split), keyby=kCols.datestore][, equals1(sum(V1)), by=kCols.datestore][, V1])
  stopifnot(DT.merged[, unique(perc_split), keyby=kCols.main][, equals1(sum(V1)), by=kCols.datestore][, V1])

DT.merged[.(as.Date("2013-02-01"), 1)]

sumcolumns(DT.merged[.(as.Date("2013-02-01"), 1)])
   storeid GL_gross_forDS budget GL_gross_forDSO total_netsc_by_DSO OA_gross_forDSOM avg_perc_OfStoreTotal_byMeta avg_perc_OfStoreTotal_bySC avg_perc_OfStoreTotal_bySC_Meta perc_split perc_split2
1:      11        53,610,008      0         38,989,096         38,989,096       10,667,542                            1                          6                               1          1           0

DT.merged[.(May, 1)]
sumcolumns(DT.merged[.(May, 1)])

.SUMS_noNA <- DT.merged[] [, unique(avg_perc_OfStoreTotal_bySC_Meta), keyby=kCols.datestore][, equals1(sumn(V1)), by=kCols.datestore]
.SUMS_NA <- DT.merged[] [, unique(avg_perc_OfStoreTotal_bySC_Meta), keyby=kCols.datestore][, equals1(sum(V1)), by=kCols.datestore]

identical(.SUMS_NA[, is.na(V1)],  .SUMS_noNA[, !(V1)] )
.SUMS_noNA[!(V1)]
.SUMS_NA[is.na(V1)]


.SUMS_NA[is.na(V1)] [!.SUMS_noNA[!(V1)]]
.SUMS_noNA[!(V1)] [!.SUMS_NA[is.na(V1)]]



DT.merged[, equals1(sum(perc_split)), by=kCols.datestore][!(V1)]
.Us   <- DT.merged[, unique(perc_split), keyby=kCols.splitgroup]
.SUMS <- .Us[, equals1(sum(V1), 1e-3), by=kCols.datestore]
.SUMS[!(V1)]
.Us[.(as.Date("2013-02-01"), 1)]
DT.merged[.(as.Date("2013-02-01"), 1)][, sum(perc_split)]

.Us[.(as.Date("2013-02-01"), 1)][, (sum(V1))]
.Us[.(Jan13, 1)][, 1==(sum(V1))]
equals
DT.merged[.(Jan13, 1)]

sumcolumns(DT.merged[tail(.SUMS[!(V1)], 1)])


DT.merged[.(as.Date("2013-02-01"), 1)] 
sumcolumns(DT.merged[.(as.Date("2013-02-01"), 1)] )
DT.merged[, unique(perc_split), keyby=kCols.datestore][, equals1(sum(V1), 1e-3), by=kCols.datestore][!(V1)]

  DT.merged[(perc_split == 1 & label_sc_group == "Orchard")][!(GL_gross_forDS == GL_gross_forDSO)]
  DT.merged[(perc_split == 1 & label_sc_group == "Orchard")][!(GL_gross_forDS == GL_gross_forDSO)]
  DT.merged[.(as.Date("2013-06-01"), 240)][(perc_split == 1 & label_sc_group == "Orchard")]
  DT.merged[.(as.Date("2013-06-01"), 240)]

  DT.merged[is.na(perc_split)]

DT <- copy(DT.merged[storeid == 500 & date >'2013-03-01' & date < '2013-07-01'])
temp_DT_stores_to_set_to_one[DT, (label_sc_group == "Orchard" & is.na(perc_split))]
  DT[temp_DT_stores_to_set_to_one, perc_split2 := (label_sc_group == "Orchard" & is.na(perc_split))]
  DT[temp_DT_stores_to_set_to_one, (label_sc_group == "Orchard" & is.na(perc_split))] []
DT[temp_DT_stores_to_set_to_one]
DT.merged[temp_DT_stores_to_set_to_one][is.na(perc_split) & label_sc_group=="Orchard"]
DT.merged[temp_DT_stores_to_set_to_one][1==(perc_split) & label_sc_group=="Orchard"]
DT.merged[temp_DT_stores_to_set_to_one][(label_sc_group == "Orchard" & is.na(perc_split))]

  matchKey(DT.merged, temp_DT.splitCols_expanded, kCols.store)
  default_values.nm <- setdiff(names(temp_DT.splitCols_expanded), kCols.store)

  DT <- setkeyIfNot(copy(DT.merged[storeid == 1 & date < "2013-03-01"]), kCols.splitgroup, organize=TRUE)
  default_values.nm
  DT[temp_DT.splitCols_expanded, ]

  ## ------------------------------------------------------------ ##
  ## NOTE: There are stores where SC labels have sales through those stores only on occasion. (eg, see MediaNet in 2014-02-01 to 2014-04-01 )
  ##       At some point in the future, we can get more preceise.  For now, disregard, and just permeate the necessary values
  ##       We can identify these rows using the fact (i think) that they will have NAs in the meta cols -- we would need to account for NAs in DT.split_percs 
  ## ------------------------------------------------------------ ##

  {

    ## What NAs are there? 
    { 
      na_count_after <- colSums(is.na(DT.merged))

      nas_expected <- {
                      c( "--" = 0
                        ## Inside and Outisde does NOT include 'Rows of stores not in GL'
                       , "InsideDateRange"     = nrow(DT.merged[(date <= lastDateClosed) & (date >= GL_firstDate_forS)])
                       , "OutsideDateRange"    = nrow(DT.merged[(date >  lastDateClosed) | (date <  GL_firstDate_forS)])
                       , "All_Rows NOT in GL"  = nrow(DT.merged[!DT.mgmtr_summary_tall, allow=TRUE])
                       , "All_Rows NOT in OA"  = nrow(DT.merged[!DT.split_percs, allow=TRUE])
                       , "All_Rows"            = nrow(DT.merged)     
                       , "Rows of stores not in GL" = nrow(DT.merged[!(id_is_in_GL)]) 
                       , "Rows of stores not in OA" = nrow(DT.merged[!(id_is_in_OA)]) 
                       )
                    }

      nas_count <- cbind(col=names(na_count_before), na_count_before, na_count_after, nas_removed=na_count_before - na_count_after)
      for (nas.nm in names(nas_expected))
        nas_count[nas_count == nas_expected[[nas.nm]]] <- nas.nm

      nas_count <- as.data.table(nas_count)
      nms.GL <- names(DT.mgmtr_summary_tall)
      nms.OA <- names(DT.split_percs)
      nas_count[, source := ifelse (col %in% nms.GL & col %in% nms.OA, "BOTH", ifelse(col %in% nms.GL, "GL", ifelse(col %in% nms.OA, "OA", NA_character_)))]
      setkeyIfNot(nas_count, source, na_count_after, organize=FALSE, verbose=FALSE)
      setcolorderpt(nas_count, c("col", "source"))
      nas_count
    }

    ## The NAs in id_is_in_GL correspond to stores that are in GL but not in OA for that given month 
    ## CONFIRM: 
    stopifnot(DT.merged[is.na(id_is_in_GL), all(storeid %in% DT.mgmtr_summary_tall$storeid)])
    DT.merged[is.na(id_is_in_GL), id_is_in_GL := TRUE]
    
    ## The NAs in id_is_in_OA correspond to stores that are in OA but in GL 
    ## CONFIRM: 
    stopifnot(DT.merged[is.na(id_is_in_OA), all(storeid %in% DT.split_percs$storeid)])
    stopifnot(DT.merged[is.na(id_is_in_OA), !id_is_in_GL])
    DT.merged[is.na(id_is_in_OA), id_is_in_OA := TRUE]

  }
}

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
jsdklfjasdkkdsfkl;ajdsfsdkljfklsdajfkl