{
            ### RELOAD
            loadFromJesus("DB.using", overwrite=TRUE)
            cat(" Release  Date range is: ", paste(as.character(DB.using[, range(releaseDate)]), collapse=" ~ "), "\n",
                 "Download Date range is: ", paste(as.character(DB.using[, range(download_date)]), collapse=" ~ "), "\n")

            suppressWarnings(DB.using[,  c("SpotifyAddRan", "XLS.included", "impr1k", "ctr", "formatOfAddProd") := NULL])
## ------------------------------ ##
##    Add XLS Meta Dat            ##
## ------------------------------ ##

# SpotifyAddRan Column comes from the SpotifyAdds DB.  If this is unavailable, use one of the SpotifyMeta tables
matchKey(DB.using, SpotifyAdds, "upc")


#DB.using[, SpotifyAddRan := FALSE]
DB.using[, c("SpotifyAddRan", "XLS.included") := FALSE]
DB.using[SpotifyAdds[, list(upc, impressions, ctr, RqstdImpressions, formatOfAddProd, SpotifyAddRan)]
       , `:=`(XLS.included=TRUE, SpotifyAddRan=i.SpotifyAddRan, impr1k=impressions/1000, ctr=i.ctr, formatOfAddProd=as.character(i.formatOfAddProd) )]
DB.using[, formatOfAddProd := factor(formatOfAddProd, levels=formatOfAddProd.levels)]
## We may not have received data for some 
# unique(DB.using[SpotifyAddRan & is.na(impressions)], by=c("upc", "releaseDate"))[order(releaseDate)]
# unique(DB.using[SpotifyAddRan & !is.na(impressions)], by=c("upc", "releaseDate"))[order(releaseDate)]
# DB.using[SpotifyAddRan & is.na(impressions)]


# Move "A or B" to B
DB.using[mkt_priority == "A or B", mkt_priority := "B"]
DB.using[is.na(mkt_priority),      mkt_priority := "C"]


W.min <- c(0L, 0L, 2L)
W.max <- c(2L, 6L, 6L)
W.nms <- c("during", "total", "after")

aggCounts <- function(DT, Wmin, Wmax, Wnm) {
  DB.tmp <- copy(DT)
  setkeyIfNot(DB.tmp, "upc")
    
         ## filter to only the values within the date range
  DB.tmp[download_date >= releaseDate + (Wmin*7) & download_date < releaseDate + (7*Wmax)
          ##  Add a "week number", number of rows, total count
          ##  All other columns remain present.
        , `:=`(weeks=Wnm, N=.N, total.count.by.upc=if(is.na(count[[1]])) 0L else as.integer(sum(count)))
          ## per each upc
        , by=list(upc) ]
  
  ## Some UPCs may not have any downloads in the subset of weeks. 
  ##  We still need to mark these as "0" otherwise, we will dilute the natural skew in the data
  uncounted.upcs <- DB.tmp[, sum(!is.na(weeks)), by=upc][V1==0, upc]
  DB.tmp[uncounted.upcs, mult="first", `:=`(weeks=Wnm, N=0L, total.count.by.upc=0L)]

  return(DB.tmp[!is.na(weeks)])
}

## Aggregate
setkeyIfNot(DB.using, "upc")
DB.using.agg <-  rbindlist(lapply(seq(W.min), function(i)
                        aggCounts(DB.using, W.min[[i]], W.max[[i]], W.nms[[i]])  ))

## Double check the aggregation
if (! (DB.using.agg[, total.count.by.upc[weeks=="total"][[1]] == total.count.by.upc[weeks=="during"][[1]] + total.count.by.upc[weeks=="after"][[1]], by=upc][, all(V1)]))
  stop("The aggregation is not correct. Please doublecheck DB.using.agg")

## individual "day" & "count" columns no longer needed, since we aggregated over them.
ColsToDrop <- c("download_date", "count")
DB.using.agg[, (ColsToDrop) := NULL]
## After we drop these columns, there will be duplicate rows. So remove all duplicates
DB.using.agg <- unique(DB.using.agg, by=NULL)  # [order(releaseDate)]  <~~ not needed since setting key next

## Convert to factor for lm
factorCols <- c("mkt_priority", "genre", "weeks")
DB.using.agg[, (factorCols) := lapply(.SD, function(x) as.factor(x)), .SDcols=factorCols]

# reset key
setkey(DB.using.agg, "upc")

}

if (FALSE)
      jesusForData(DB.using.agg)

#     # ~~~~~~~~~~~~~~~~~~~
#     
#     response <- "total.count.by.upc"
#     predictors <- c(0, "mkt_priority", "genre", "impr1k", "SpotifyAddRan", "ctr")
#     unused <- c("upc", "download_date", "count", "user_country", "artist", "release", "releaseDate")
#     setdiff(names(DB.using.agg), c(response, predictors, unused))
#     
#     fmla <- as.formula(paste(response, "~", paste0(predictors, collapse="+")))
#     models <- list()
#     e <- environment()
#     DB.using.agg[, {assign(pasteC(c("models", as.character(W)), C="_") , lm(fmla, data=.SD), envir=e); .BY}, by=list(W=weeks)]
#     lsos(patt="models")
#     summary(models_during)
#     summary(models_after)
#     summary(models_total)
#     
#     
#     ----------------------
#     
#     unique(DB.using.agg[(SpotifyAddRan)], by="upc")[order(mkt_priority, genre)]
#     unique(DB.using.agg[genre %ni% MainGenres, list(mkt_priority, SpotifyAddRan, genre), by=upc], by=NULL)[, table(mkt_priority, SpotifyAddRan, genre, useNA="ifany")]
#     unique(DB.using.agg[, list(mkt_priority, SpotifyAddRan, genre), by=upc], by=NULL)[, table(mkt_priority, SpotifyAddRan)]
#     
#     MainGenres <- c("Rock", "Electronic", "Metal")
#     
#      "Folk", "Spoken Word", 
#     "Hip-hop/Rap", "Soundtracks", "New Age", "Christian", "Blues", 
#     "Reggae", "R&B", "Country", "Holiday", "Audio Books", "Educational", 
#     "Video")
#     Rock, Metal, Electronic, Rap / Hip-Hop
#     
#     
#                 SpotifyAddRan
#     mkt_priority FALSE TRUE
#           A         53   16
#           A or B    10    2
#           B        187   14
#     
#     ls -alh '/mnt/data/home/rsaporta/git/orch/data/SpotifyAdds/DB.using-!98024x10!-20131119_2208.RDS'
#     
#     scp ubuntu@107.20.192.2:'/mnt/data/home/rsaporta/git/orch/data/SpotifyAdds/DB.using-!98024x10!-20131119_2208.RDS' '~/git/orch/data/SpotifyAdds/DB.using-!98024x10!-20131119_2208.RDS'
#     
#     DB.using <-  setnames(readRDS('~/git/orch/data/SpotifyAdds/DB.using-!98024x10!-20131119_2208.RDS'), "SpotifyRan", "SpotifyAddRan")
#     
#     
#     
#      idcol      :   upc,   product_type_id
#     
#      character  :   release,          mkt_priority,   artist,           label,
#                     label_priority,   genre,          artist_country,   label_country
#     
#      Date       :   releaseDate
#     
#     
#     
#