# aggregate_and_clean.r

aggregate_and_clean <- function(DT=get("DT.users", envir=parent.frame()), products_to_include=NULL, drop_other_products=FALSE, ...) {
  ## Basic repetitive cleaning taks on a DT before and after calling aggregateDT
  ## ... :: passed to aggregateDT

  prodCols <- c("product", "product_description")
  ctryCols <- c("country", "region_group")

  DT <- copy(DT)

  new.info <- paste0("Aggregting "
                    , if (!is.null(products_to_include)) sprintf(", restricted to\nproduct %%in%% c%s", pasteQ(products_to_include))
                    )
  info <- appendInfo(DT=DT, new.info=new.info, showWarnings=FALSE)

  if (!is.null(products_to_include))
      DT[product %ni% ProductsUsing, (prodCols) := list("ZZ", "All Other Products")]

  ## Drop any existing global rows, as this will aggregate poorly
  if ("country" %in% names(DT) && "global" %in% DT[["country"]])
    DT <- DT[country != "global"]

  ret <- aggregateDT(DT, addInfo=FALSE, showWarnings.info=FALSE, ...)

  if ("country" %in% names(ret)) {
    key.bak <- key(ret)
    ret <- rbind(ret, aggregateDT(ret, exclude=ctryCols, addInfo=FALSE)[, (ctryCols) := list("global", "Global Aggregate")], use.names=TRUE)
    setkeyIfNot(ret, key.bak, organize=TRUE, verbose=FALSE)
  }

  setSpotifyProductToFactor(ret)
  computeRatioCols_(ret)
  setInfo(ret, info=info)
  return(ret)
}