#   DT.merged[date > "2013-01-01", 1L, by=c(kCols.datestore, "OA_ZeroMonth_forDS", "GL_lastDate_forS")][, lunique(OA_ZeroMonth_forDS), by=c(kCols.store, "GL_lastDate_forS")][V1 != 1]$storeid -> ids
#   DT.merged[storeid %in% ids][date > "2014-06-02" & !OA_ZeroMonth_forDS, unique(storeid)] -> ids
#   DT.merged[storeid %in% ids, 1L, keyby=c(kCols.datestore, "OA_ZeroMonth_forDS", "GL_lastDate_forS")][(OA_ZeroMonth_forDS)] [date == "2014-06-01"]
# ------

if (FALSE) 
{
  # deving : 
  # deving :   kCols.parent=kCols.datestore; kCols.subgroup=kCols.splitgroup;
  # deving :   newCol = "OA_percof_DS_forDSOM"; measureCol = "OA_gross_forDSOM"; dateCol = "date";  units="month"; ma_range=4;
  # deving : 
  # deving :   DT <- copy(DT.merged)[(storeid %in% c(c(206, 426), 1, 7, 187, 526, 506, 555) | store_name %chin% c("Liberator Music", "DMX")) & date >= "2014-03-01"]
  # deving :   DT <- copy(DT.merged)[(storeid %in% c(206, 426, 1, 7, 578) | store_name %chin% c("Liberator Music", "DMX")) & date >= "2014-03-01"]
  # deving :   if (newCol %chin% names(DT)) {
  # deving :     DT[, old_percs := get(newCol)]
  # deving :     DT[, (newCol) := NULL]
  # deving :   }
  # deving : 
  # deving : 
  # deving :   calculateGroupPerc(DT=DT, kCols.parent=kCols.datestore, kCols.subgroup=kCols.splitgroup, newCol = "OA_percof_DS_forDSOM", measureCol = "OA_gross_forDSOM", dateCol = "date", ma_range=4, units="month")
}



calculateGroupPerc_ <- function(DT, kCols.parent, kCols.subgroup, newCol, measureCol, dateCol="date", ma_range=4, units="month") 
{

  key.bak <- key(DT)
  nms.dt <- names(DT)

  ## ensure no dateCol
  kCols.parent   <- setdiff(kCols.parent, dateCol)
  kCols.subgroup <- setdiff(kCols.subgroup, dateCol)

  if (any (kCols.parent %ni% kCols.subgroup) ||  length(kCols.subgroup) == length(kCols.parent))  
    stop ("kCols.parent is NOT a strict subset of kCols.subgroup")
  
  ## order kCols.parent according to kCols.subgroup and make sure all of the first cols of subgroup allign to kCols.parent-- needed for step 2
  kCols.parent   <- intersect(kCols.subgroup, kCols.parent)
  kCols.subgroup <- c(kCols.parent, setdiff(kCols.subgroup, kCols.parent))

  ## create copies WITH date
  kCols.date_parent   <- c(dateCol, kCols.parent)
  kCols.date_subgroup <- c(dateCol, kCols.subgroup)

  ## for part 3
  kCols.noParent <- setdiff(kCols.subgroup, kCols.parent)
  kCols.date_noParent <- c(dateCol, kCols.noParent)

  ## the kCOls for parentgroup should be a strict subgroup -- since the kCols indicate how the data is chopped up
  if (!length(kCols.parent) || !length(kCols.subgroup))
    stop ("kCols.parent & kCols.subgroup must contain at least one column that is not dateCol")
  ## check for missing cols
  if (any(kCols.date_subgroup %ni% nms.dt))
    stop (warningCols("some elements of kCols.parent are missing from DT", kCols.date_parent[kCols.date_parent %ni% nms.dt]))
  if (newCol %in% nms.dt)
    stop ("The expected newCol ('", newCol, "') already exists in DT")
  if (measureCol %ni% nms.dt)
    stop ("The measureCol ('", measureCol, "') does not exists in DT")

  ## THERE SHOULD BE NO NAs in measureCol
  if (any(DT[, any(is.na(measureCol))]))
    stop ("There are NAs in measureCol.\n If these should be treated as 0, please convert prior to calling this function")


  date_range <- function(date) {
    sort(c(
        seq.Date(as.Date(date, origin=.origin), length.out=ma_range_btm, by=paste0("-1 ", units))
      , seq.Date(as.Date(date, origin=.origin), length.out=ma_range_top+1, by=units)
    ))[1:ma_range]
  }


  ## Fill NAs in this order
  # 1. Take the direct percentage 
  # 2. Take an average for the parentgroup, across the different dates in the ma_range    [ie, store trending avg:  for each store, take an average across dates (avg for each row)]
  # 3. Take an average for the subgroup,    across the different parentgroups in the given date [ie, company avg for the month: ]
  # 4. Take an average for the subgroup,    across the different dates in the ma_range
  # 5. Take an average for the subgroup,    across the entire dt
  #
  # Note on 3: It is tempting to have 3 be a weighted average between wider ranges for the group and averages for the whole store
  #            especially since a groups percentages might differe greatly from that of a store
  #            However, keep in mind that in order to reach 3, there has to be a rather wide gap, else it would have been caught in 2
  #


  ## change names for easier 
  dict.colnames <- c("mezz"=measureCol, "newzz"=newCol)
  if (any(names(dict.colnames) %in% nms.dt))
    stop ("it's a Tuesday. Pick new column names please for DT. ", pasteQand(names(dict.colnames)), " are reserved names")
  setnames(DT, dict.colnames[dict.colnames != newCol], names(dict.colnames[dict.colnames != newCol]))



  confirmTotalPercentage <- function(checkpoint=NULL, tol=1e-3) {
    DT.checksum <- DT[, equals0or1(sumn(newzz), tol=tol), keyby=kCols.date_parent]
    if (all(DT.checksum$V1))
      return(invisible(TRUE))
    ## ELSE
    print(DT.checksum[!(V1)])
    stop ("The percentage no longer sums to 0 or 1 (by parent gorp) for the above offending group(s)", if (!is.null(checkpoint)) paste(" ---  Checkpoint is:  ", checkpoint))
  }

  ### -------------------------------------------------------------------------------- ###
  ## 1  
  ### -------------------------------------------------------------------------------- ###
  setkeyIfNot(DT, kCols.date_parent, superset.ok=TRUE, verbose=FALSE)
  DT[DT[, sumn(mezz), keyby=kCols.date_parent], newzz := removeNA(mezz, repl=0) / i.V1]

  ## NOTE that if there is one NA, then the whole month should be NA (ie, division by 0)
  stopifnot(DT[, {NAs <- is.na(newzz); all(NAs) || !any(NAs)}, keyby=kCols.date_parent][, V1])

  ## CONFIRM, Total percentage is either 0 or q
  confirmTotalPercentage(checkpoint="after 1")

  ## stop if no NAs left
  if (!any(is.na(DT[, newzz]))) {
    setnames(DT, names(dict.colnames), dict.colnames)
    return(invisible(DT))
  }

  ### -------------------------------------------------------------------------------- ###

  ### -------------------------------------------------------------------------------- ###
  ## 2
  ### -------------------------------------------------------------------------------- ###
  ## we will try a few ranges, namely the given range, a little les than twice it, and then the 'full' range
  ma_ranges <- sort(unique(ceiling(c(ma_range, ma_range * 1.75, (1+findLengthOfDateRange(DT[[dateCol]], units=units))/2))))
  for (.ma_range in ma_ranges) {
  ## note that if gaps remain after the first pass, the gap must be quite wide, hence factor of 1.75

    if (!any(is.na(DT[, newzz])))
      break


    ma_range_btm <- floor(.ma_range/2)
    ma_range_top <- ceiling(.ma_range/2)

    ## Note that we are filtering DT to only rows where newzz is NA
    NAs_2 <- DT[ is.na(newzz)
              , list( # indx.assign=list(setkey(setnames(as.data.table(.BY), kCols.date_parent))),  ## not sure if this is needed, since same as key
                      indx.avg=list(setkey(setnames(as.data.table(c(list(date_range(.BY[[1]])), .BY[-1])), kCols.date_parent))))
              , keyby=kCols.date_parent]

    ## we set the key to subgroup, which is a strict superset of kCols.parent
    setkeyIfNot(DT, kCols.date_subgroup, organize=TRUE, verbose=FALSE)
    for (i in seq(nrow(NAs_2))) {
      AvgBy <- DT[NAs_2[i, indx.avg], sumn(mezz), by=kCols.subgroup, nomatch=0
                ][, .grp.avg. := V1 / sum(V1)
                ][, (dateCol) := NAs_2[i, get(dateCol)]]

      setkeyIfNot(AvgBy, kCols.date_subgroup, organize=TRUE, verbose=FALSE)
      DT[AvgBy, newzz := {
        ## First confirm that it is all NA
        if (!all(is.na(newzz))) {
          print(sprintf("ERROR is in the for loop in part 2 of calculateGroupPerc(),  i = %2i", i))
          stop ("Not all of 'newzz' is NA -- that should not be the case. Investigate")
        }
        ## Note that .grp.avg. will itself be NaN if there were no sales for the whole ma_range
        .grp.avg.
      }]
    } ## // inner for loop

    ## CONFIRM, Total percentage is either 0 or q
    confirmTotalPercentage(checkpoint=paste0("after 2    .ma_range = ", .ma_range))
  
    ## stop if no NAs left
    if (!any(is.na(DT[, newzz]))) {
      setnames(DT, names(dict.colnames), dict.colnames)
      return(invisible(DT))
    }

  } ## // outter for loop
  ### -------------------------------------------------------------------------------- ###



  ## TODO: All of part 3 is the same as part 5.  The only difference is the averaging DT and the keys
  ##    .. and the NAs to 0 in part 5


  ### -------------------------------------------------------------------------------- ###
  ## 3
  ### -------------------------------------------------------------------------------- ###
  ## Unlike in part1, in part3 we only want to modify those rows 
  ##   where the entire date-parent group is NA
  DT[, .tmp.3.fill.with.avg. := all(is.na(newzz)), keyby=kCols.date_parent]

  MonthlyAvgsPerSubGroup <- DT[, list(monthly_total_mezz = sumn(mezz)), keyby=kCols.date_noParent][, avg.perc := monthly_total_mezz / sumn(monthly_total_mezz), by=dateCol]
  ## CONFIRM:  avg.perc sums to one for each date
  stopifnot(MonthlyAvgsPerSubGroup[, equals1(sum(avg.perc)), by=dateCol][, V1])

  matchKey(DT, MonthlyAvgsPerSubGroup, key=kCols.date_noParent, superset.ok=TRUE, organize=TRUE)
  DT[MonthlyAvgsPerSubGroup, newzz3 := i.avg.perc]
  ## Note, newzz3 will NOT sum to 1 by kCols.date_parent as we would expect
  ##   The reason being that certain subgroup combination may not be present in every date-parent group
  ##   (eg, a smaller store may never sell a given SC)
  ## Thus we have to scale before assigning
  DT[, newzz3 := newzz3 / sum(newzz3), by=kCols.date_parent]
  ## CONFIRM:  Now we can expect newzz3 to sum to one
  sumsToOne(DT, "newzz3", by=kCols.date_parent)
  ## CONFIRM: a date-parent group is either ALL fill or not. We are not filling "some but not others" subgroup rows
  stopifnot(DT[, all(.tmp.3.fill.with.avg.) || !any(.tmp.3.fill.with.avg.), by=kCols.date_parent][, V1])

  ## Fill in the missing values
  DT[(.tmp.3.fill.with.avg.), newzz := newzz3]

  ## cleanup
  DT[,  .tmp.3.fill.with.avg. := NULL]
  DT[,  newzz3 := NULL]

  ## CONFIRM, Total percentage is either 0 or q
  confirmTotalPercentage(checkpoint="after 3")

  ## stop if no NAs left
  if (!any(is.na(DT[, newzz]))) {
    setnames(DT, names(dict.colnames), dict.colnames)
    return(invisible(DT))
  }

  ### -------------------------------------------------------------------------------- ###



  ### -------------------------------------------------------------------------------- ###
  ##  4
  ### -------------------------------------------------------------------------------- ###
  "SKIPPING PART 4"
  ### -------------------------------------------------------------------------------- ###


  ### -------------------------------------------------------------------------------- ###
  ##  5
  ### -------------------------------------------------------------------------------- ###
  ## Unlike in part1, we only want to modify those rows 
  ##   where the entire date-parent group is NA
  DT[, .tmp.5.fill.with.avg. := all(is.na(newzz)), keyby=kCols.date_parent]
  
  OverallAvgsPerSubGroup <- 
          DT[, sumn(mezz), keyby=kCols.date_noParent
           ][, sumn(V1), keyby=kCols.noParent
           ][, avg.perc :=V1 / sumn(V1)
           ]
  ## CONFIRM: 
  sumsToOne(OverallAvgsPerSubGroup, "avg.perc", by=NULL)

  ## Consider
  ## Think about the NAs here, what are they telling us? Essentially, that the whole group does not exist!
  ## Question: How do the NAs break down
  ## SEE:
  OverallAvgsPerSubGroup[, list(NA.perc=howManyNAs(avg.perc, perc=TRUE), NA.count=sum(is.na(avg.perc))), by=kCols.noParent]
  stopifnot(OverallAvgsPerSubGroup[, {NAs <- is.na(avg.perc); all(NAs) || !any(NAs)}, by=kCols.noParent][, V1])
  # Notice that groups are either all NA or have no NAs
  # Thus, it is a safe bet, that these groups should be 0. 
  #  eg,  Allegro-Music-Comps  (There are no comps except for Orchard)
  #
  # Therefore, we will make the NAs zero, here
  OverallAvgsPerSubGroup[is.na(avg.perc), avg.perc := 0]


  matchKey(DT, OverallAvgsPerSubGroup, key=kCols.noParent, superset.ok=TRUE, organize=TRUE)
  DT[OverallAvgsPerSubGroup, newzz5 := i.avg.perc]
  ## Note, newzz5 will NOT sum to 1 by kCols.date_parent as we would expect
  ##   The reason being that certain subgroup combination may not be present in every date-parent group
  ##   (eg, a smaller store may never sell a given SC)
  ## Thus we have to scale before assigning
  DT[, newzz5 := newzz5 / sum(newzz5), by=kCols.date_parent]
  ## CONFIRM:  Now we can expect newzz5 to sum to one
  sumsToOne(DT, "newzz5", by=kCols.date_parent)
  ## CONFIRM: a date-parent group is either ALL fill or not. We are not filling "some but not others" subgroup rows
  stopifnot(DT[, all(.tmp.5.fill.with.avg.) || !any(.tmp.5.fill.with.avg.), by=kCols.date_parent][, V1])

  ## Fill in the missing values
  DT[(.tmp.5.fill.with.avg.), newzz := newzz5]

  ## cleanup
  DT[,  .tmp.5.fill.with.avg. := NULL]
  DT[,  newzz5 := NULL]

  ## CONFIRM, Total percentage is either 0 or q
  confirmTotalPercentage(checkpoint="after 3")

  ## stop if no NAs left
  if (!any(is.na(DT[, newzz]))) {
    setnames(DT, names(dict.colnames), dict.colnames)
    return(invisible(DT))
  }


  ### -------------------------------------------------------------------------------- ###

  ## Fill NAs in this order
  # 1. Take the direct percentage 
  # 2. Take an average for the parentgroup, across the different dates in the ma_range
  # 3. Take an average for the subgroup,    across the different parentgroups in the given date
  # 4. Take an average for the subgroup,    across the different dates in the ma_range
  # 5. Take an average for the subgroup,    across the entire dt
  #
  # Note on 3: It is tempting to have 3 be a weighted average between wider ranges for the group and averages for the whole store
  #            especially since a groups percentages might differe greatly from that of a store
  #            However, keep in mind that in order to reach 3, there has to be a rather wide gap, else it would have been caught in 2
  #

  ### -------------------------------------------------------------------------------- ###

  ## If any remains, that is strange
  if (!any(is.na(DT[, newzz])))
    warning ("Could not fill ALL of the percentage column. Some NAs remain")

  setnames(DT, names(dict.colnames), dict.colnames)
  return(invisible(DT))


}