## Cleanup the loaded DTs

## Converting column names from what was given in the Spotify report to more interpratable name
tmp_dict.new_names_colnames_users <- {
        c(
              #                 New Name  = Old Name
                           total_streams  = "total_tracks"
          ,      orchard_and_osc_streams  = "rightholders_tracks"
          ,      spotify_net_revenue_usd  = "net_revenue_usd"
          ,  orchard_and_osc_revenue_usd  = "payable_usd"
          ,                         date  = "month"
          ,                      country  = "country_code"
        )}

assignWithInfo("DT.users", copy(DT.revshare), envir=globalenv())

## Drop these unused columns from DT.users
tmp_colsNotUsing <- {
  c("net_revenue"        # only need net USD
  , "gross_revenue"      # only need net USD
  , "gross_revenue_usd"  # only need net USD (since it is not clear, "net of what?")
  , "pro_rata_share"     # This is the rounded reported value. Using instead the exact calculated
  , "pro_rata_exact"     # Equivalent to MarketShare_byStreams
  , "EUR_to_USD"         # Rick added this to DT.revshare  Not needed ere
  , "currency"           # Everything in DT.users is in USD, thus no need for currency
  , "payable_eur"        # only need payable_usd
  , "payable"            # only need payable_usd
  , "new_product_users"  # These are '0' across the board
  , "subscriber_number"  # These are '0' across the board
  )}


DT.users[, (tmp_colsNotUsing) := NULL]

setNamesDict(DT.users, tmp_dict.new_names_colnames_users, showWarnings=FALSE)


## Crop below minDate.global  -- specifically, the accounting was different prior to 2013
if (exists("minDate.global"))
  assignWithInfo(DT.users, DT.users[date >= minDate.global], info=sprintf("Cropped to date >= %s", minDate.global))

## make revenue_share into a factor, since generally, it will be utreated as such
DT.users[, revenue_share := setFactorOrder(revenue_share, c(60, 55, 80, 0))]
# DT.users[, table(revenue_share)]

## Temporary fill
DT.users[, region_group := "Ungrouped"]

## Change "Student Discount" to simply "Discount" 
DT.users[product_description == "Premium Student Discount", product_description := "Premium Discount"]

## Keys & organize
kCols.DTusers <- c("date", "revenue_share", "product", "product_description", "region_group", "country")
setkeyIfNot(DT.users, kCols.DTusers, verbose=FALSE)
setcolorderpt(DT.users, unique(c(kCols.DTusers, unlist(sapply(c("users", "streams", "revenue"), extract,  DT.users, sort=TRUE)))))

## FOR SOME REASON, there are several date-countries with 2 lines of data for PD
## Aggregate those
DT.users <- aggregateDT(DT.users, by=kCols.DTusers)


## TODO: Divide streams and number of users by 1K or 1M
# DT.users <- DT.users[, lapply(.SD, "/", 1e6), keyby=key(DT.users)]

## Add "global" rows and region_group
{
  if ("global" %in% DT.users[["country"]])
    DT.users <- DT.users[country != "global"]

  ## Add "global"
  tmp_DT_all_countries <- aggregateDT(DT.users, exclude=c("country"), addInfo="none")[, country := "global"]
  assignWithInfo("DT.users", rbind(DT.users, tmp_DT_all_countries), info="Adding in 'global' country, ie aggregate of DT.users by month and product")

  ## Add region_group
  get_dim_country(refresh=FALSE, verbose=FALSE)
  suppressWarnings(DT.users[, region_group := NULL])
  addColsFrom_(DT.users, DT.country, colsToBring="region_group", joinCols.rec=c("country"), joinCols.g=c("country_code"), nms="region_group")
  DT.users[country == "global", region_group := "Global Aggregate"]
  DT.users[region_group %ni% c("Global Aggregate", "North America", "Scandinavia", "Western Europe", "GSA", "Latin America"), region_group := "All Others"]

  rm (tmp_DT_all_countries)
}

## Clean up the info
splat.info <- strsplit(getInfo(DT.users, quiet=TRUE), "\\n")[[1]]
setInfo(DT.users, pasteC(splat.info[!grepl("Previous Info for DT.users", splat.info)], "\n"))

setkeyIfNot(DT.users, kCols.DTusers, organize=TRUE, verbose=FALSE)


## ~~~~~~~~ NO AGGREGATIONS BEYOND THIS POINT ~~~~~~~~~ ##
setSpotifyProductToFactor(DT.users)
computeRatioCols_(DT.users)