message("Aggregating using aggFunc = ", aggFunc, "")
## In case there is any issue with data.table reference assignment
if (!truelength(DT.EU_analytics))
  DT.EU_analytics <- copy(setDT(DT.EU_analytics))
if (!truelength(DT.upc_has_override))
  DT.upc_has_override <- copy(setDT(DT.upc_has_override))


countCols <- c("units", "revenue")
dateCols <- c("TestWk", "TestWk.dates")
## UPDATE:   GET RID OFF daily_avg_album_units_30preTest
byCols <- c("store_name", "transac_type_abbr", "country_code", "category", "TreatmentWk1", "upc", "is_iTunesDA", "TestWk", "TestWk.dates") # daily_avg_album_units_30preTest
byCols.noDates <- setdiff(byCols, dateCols)


tmp.using_mean <- aggFunc %in% c("mean", "meann")
if (tmp.using_mean)
  aggFunc <- "sumn"
DT.EU_aggd <- aggregateDT(DT.EU_analytics[!is.na(TestWk)]
                        , colsToAgg=countCols
                        , by=byCols
                        , aggFunc=aggFunc)

## In case there is any issue with data.table reference assignment
if (!truelength(DT.EU_aggd))
  DT.EU_aggd <- copy(setDT(DT.EU_aggd))

if (tmp.using_mean) {
  days_diff    <- 1 + as.numeric(diff(dateBoundaries[[1]]))
  DT.EU_aggd[TestWk != "Wk2", (countCols) := lapply(.SD, function(x) x / days_diff),    .SDcols = countCols]

  if ("Wk2" %in% names(DT.EU_analytics) && all(is.na(DT.EU_analytics[["Wk2"]]))) {
      days_diffWk2 <- 1 + as.numeric(diff(DT.EU_analytics[TestWk == "Wk2", rangen(date)]))
      DT.EU_aggd[TestWk == "Wk2", (countCols) := lapply(.SD, function(x) x / days_diffWk2), .SDcols = countCols]
  }
}


## Note: the large numeric values of upc causes some issue in two different upc's being interpreted by R as the same value
## namely:   "8424295051103"  "8424295051066"  (only differ in the last three digits)
upcToID_(DT.EU_aggd)


## make sure each UPC is present in each TestWk
GroupsNeededForEachUPC <- unique(DT.EU_aggd[, dateCols, with=FALSE], by=NULL)
## CONFIRM: No NAs in countCols before hand
stopifnot(0 == howManyNAs(DT.EU_aggd[, countCols, with=FALSE], hide.full=FALSE))
tmp_DT.all_groups <- DT.EU_aggd[, as.list(GroupsNeededForEachUPC), by=byCols.noDates]
matchKey(DT.EU_aggd, tmp_DT.all_groups, key=byCols)
DT.EU_aggd <- DT.EU_aggd[tmp_DT.all_groups]
## NAs to 0
DT.EU_aggd[, (countCols) := lapply(.SD, removeNA, repl=0), .SDcols = countCols]
rm(tmp_DT.all_groups)


## Clean up:  Spotify has no revenue
DT.EU_aggd[store_name == "Spotify", revenue := NA_real_]

## Add log of countCols
DT.EU_aggd[, sprintf("log_1p_%s", countCols) := lapply(.SD, function(x) log10(x+1)), .SDcols = countCols]

## Pad the string by adding a few space to the end
levels(DT.EU_aggd[["TestWk.dates"]]) <- paste0(levels(DT.EU_aggd[["TestWk.dates"]]), "  ")
setDT(DT.EU_aggd)


## We want to know whether or not an album was purchased during either the test or pretest phase
DT.EU_aggd[, album_was_purchased_incountry_during_experiment := FALSE]
DT.EU_aggd[is_iTunesDA & units > 0 & !is.na(TestWk), album_was_purchased_incountry_during_experiment := TRUE]
DT.EU_aggd[, table(album_was_purchased_incountry_during_experiment, useNA="always")]
DT.EU_aggd[, album_was_purchased_incountry_during_experiment := any(album_was_purchased_incountry_during_experiment), by=byCols.upc_country]

DT.EU_aggd[upc == DT.EU_aggd[album_was_purchased_incountry_during_experiment != album_was_purchased_incountry_during_experiment][1, upc]]

# DT.EU_aggd[album_was_purchased_incountry_during_experiment & !album_was_purchased_incountry_during_experiment & store_name == "iTunes"]
# DT.EU_aggd[upc == "886788102675"]
stopifnot(DT.EU_aggd[, lunique(album_was_purchased_incountry_during_experiment), by=byCols.upc_country][, V1 == 1])

## CROP
# DT.EU_aggd <- DT.EU_aggd[(is_iTunesDA & !is.na(TestWk))]

## Add Group for above and below 5 units
## ------------------ ##
# DT.EU_aggd[, iTunesDA_units_preTest_above5 := NULL]
# DT.EU_aggd[, iTunesDA_units_preTest_above5 := is_iTunesDA & units > 5 & TestWk == "preTest"]
# DT.EU_aggd[, iTunesDA_units_preTest_above5 := any(iTunesDA_units_preTest_above5), by=byCols.upc_country]
# DT.EU_aggd[, iTunesDA_units_preTest_above5 := factor(iTunesDA_units_preTest_above5
#                                                     , levels = c(TRUE, FALSE)
#                                                     , labels = sprintf("Album sold %s 5 units week before test", c("≥", "<"))
#                                                     )]
# stopifnot(DT.EU_aggd[, lunique(iTunesDA_units_preTest_above5), by=byCols.upc_country][, V1 == 1])

# ## Indicate whether UPC increased or decreased from PreToWk1
# DT.EU_aggd[, deltaunits.Wk1_minus_preTest := (units[TestWk == "Wk1"]  - units[TestWk == "preTest"]), by=byCols.noDates]
# DT.EU_aggd[, increased_Wk1 := factor(ifelse(deltaunits.Wk1_minus_preTest < 0, "Decreased", ifelse(deltaunits.Wk1_minus_preTest > 0, "Increased", "No Change")), levels = c("Decreased", "No Change", "Increased"))]
# stopifnot(DT.EU_aggd[, lunique(deltaunits.Wk1_minus_preTest), by=setdiff(byCols.noDates, "TreatmentWk1")][, V1 == 1])
# stopifnot(DT.EU_aggd[, lunique(increased_Wk1), by=setdiff(byCols.noDates, "TreatmentWk1")][, V1 == 1])

# ----------------------------------
addColsFrom_(DT.EU_aggd, DT.upc_has_override, joinCols=intersect(c("upc", "country_code"), names(DT.upc_has_override)), colsToBring=upc_addl_metaCols)
## put the key back
setkeyIfNot(DT.EU_aggd, c(byCols, upc_addl_metaCols), organize=TRUE, verbose=FALSE)
# ----------------------------------

if (tmp.using_mean) {
  aggFunc <- "meann"
  rm(tmp.using_mean)
}


### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ##
  addlByCols    <- c("album_was_purchased_incountry_during_experiment") # , "increased_Wk1", "iTunesDA_units_preTest_above5"
  addlCountCols <- c(sprintf("log_1p_%s", countCols)) # , "deltaunits.Wk1_minus_preTest"
  if (length(wh.forgottencols <- setdiff(names(DT.EU_aggd), c(byCols, countCols, addlCountCols, addlByCols))))
    warning (pasteC(c("You are forgetting a column for DT.EU_aggd_reshaped (if you are even using it): ", wh.forgottencols), C="\n\t"), call.=FALSE)

  if (aggFunc != "sumn") 
    warning ("results for DT.EU_aggd_reshaped may be inaccurate for aggFunc = ", aggFunc)

  colsFormula <- setdiff(names(DT.EU_aggd), c(dateCols, countCols))
  colsFormula <- setdiff(c(byCols, addlByCols), dateCols)

  tmp.formula <- makeFormula(colsFormula, "TestWk")
  DT.EU_aggd_reshaped <-  
      merge(
          dcast.data.table(DT.EU_aggd, formula=tmp.formula, value.var=c("units"), fun.aggregate=sumn)
        , dcast.data.table(DT.EU_aggd, formula=tmp.formula, value.var=c("revenue"), fun.aggregate=sumn)[store_name == "Spotify", (names(dateBoundaries)) := NA]
        , suffix = c(".units", ".revenue")
        , all = TRUE
        )
  rm(tmp.formula)
  if ("Wk2" %in% names(DT.EU_aggd_reshaped) && all(is.na(DT.EU_aggd_reshaped[["Wk2"]])))
    DT.EU_aggd_reshaped[, Wk2 := NULL]


  DT.EU_aggd_reshaped[, deltaunits.pre_minus_prePre := preTest.units - pre_preTest.units]
  DT.EU_aggd_reshaped[, deltaunits.wk1_minus_pre    := Wk1.units - preTest.units]
  # DT.EU_aggd_reshaped[, deltaunits.wk2_minus_pre    := Wk2.units - preTest.units]
  # DT.EU_aggd_reshaped[, deltaunits.wk2_minus_wk1    := Wk2.units - Wk1.units]

  DT.EU_aggd_reshaped[, deltarevenue.pre_minus_prePre := preTest.revenue - pre_preTest.revenue]
  DT.EU_aggd_reshaped[, deltarevenue.wk1_minus_pre    := Wk1.revenue - preTest.revenue]
  # DT.EU_aggd_reshaped[, deltarevenue.wk2_minus_pre    := Wk2.revenue - preTest.revenue]
  # DT.EU_aggd_reshaped[, deltarevenue.wk2_minus_wk1    := Wk2.revenue - Wk1.revenue]


  ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ##


  # library(reshape2)
  # as.data.table(dcast(DT.EU_aggd[(is_iTunesDA & !is.na(TestWk)) ], upc + TreatmentWk1 + country_code ~ TestWk, value.var=c("units"), ) )
  tmp.formula.no_country <- "upc + category + TreatmentWk1 ~ TestWk"
  DT.EU_aggd_reshaped.no_country <-  
    merge(
        dcast.data.table(DT.EU_analytics[(is_iTunesDA & country_code %ni% "IE")], formula=tmp.formula.no_country, value.var=c("units"), fun.aggregate=sumn)
      , dcast.data.table(DT.EU_analytics[(is_iTunesDA & country_code %ni% "IE")], formula=tmp.formula.no_country, value.var=c("revenue"), fun.aggregate=sumn)
      , suffix = c(".units", ".revenue")
      , all = TRUE
      )
  rm(tmp.formula.no_country)