DT.upc_price_override   <- fread(ingest.p("Release_Price_Override.csv"), colClass="character")
DT.track_price_override <- fread(ingest.p("Track_Price_Override.csv"), colClass="character")
DT.track_and_CD_counts  <- fread(ingest.p("Tracks_and_cds_per_upc.csv"), colClass="character")

## Some strange duplicates.  eg: 
upcs_with_multi_values <- DT.track_price_override[, .N, keyby=list(upc, unique_track_id, country_code)][(N != 1), unique(upc)]
if (length(upcs_with_multi_values)) {
  cat("\n", pasteR("~=", n=25), "\nAsk Jian about the strange duplicates in Track_Price_Override.csv\n")
  print(unique(DT.track_price_override[upc %in% upcs_with_multi_values][, list(.N, price_code), keyby=list(unique_track_id, upc, country_code)][N != 1], by=c("upc", "unique_track_id", "price_code")), nrow=300)
  cat(pasteR("~=", n=25), "\n\n")
  ## remove them
  DT.track_price_override[, price_code := tail(price_code, 1), by=list(unique_track_id, upc, cd, track_no, country_code)]
  DT.track_price_override <- unique(DT.track_price_override, by=c("unique_track_id", "upc", "cd", "track_no", "country_code", "price_code"))
  DT.track_price_override[unique_track_id == 6075018]

  ## Confirm no more duplicates
  stopifnot(DT.track_price_override[, .N, keyby=list(upc, unique_track_id, country_code)][, N == 1])
}


## Default price code in GB / EU is  3 for Albums,  99 for tracks
DT.track_price_override <- DT.track_price_override[price_code != 99]
upcs_with_up_and_down <- DT.track_price_override[, lunique(price_code), by=upc][(V1 != 1), unique(upc)]
if (length(wh.up_and_down <- setdiff(upcs_with_up_and_down, RockyHorror.upc))) {
  warning (warningCols("There are new UPCs with up and down overrides besides just Rocky Horror", wh.up_and_down))
}

## Clean up  UPC PRICE OVERRIDE
upcToID_(DT.upc_price_override)
DT.upc_price_override[, c("currency_code", "custom_price") := NULL]
convertNumberCols_(DT.upc_price_override, verbose=FALSE)
setkeyIfNot(DT.upc_price_override,    "upc", organize=TRUE, verbose=FALSE)

## Clean up  TRACK AND CD COUNTS
upcToID_(DT.track_and_CD_counts)
convertNumberCols_(DT.track_and_CD_counts, verbose=FALSE)
DT.track_and_CD_counts <- DT.track_and_CD_counts[, list(total_cds = lunique(cd), total_tracks = sum(track_count)), keyby=upc]
DT.track_and_CD_counts[, avg_tracks_per_cd := total_tracks / total_cds]
setkeyIfNot(DT.track_and_CD_counts,          "upc", organize=TRUE, verbose=FALSE)

## Clean up  TRACK PRICE OVERRIDE
upcToID_(DT.track_price_override)
DT.track_price_override[, unique_track_id := as.idcolfactor(unique_track_id)]
DT.track_price_override[, price_code := factor(price_code)]
DT.track_price_override[, c("currency_code", "custom_price") := NULL]
## country_code is nearly irrelevant.
DT.track_price_override[, country_code := NULL]
DT.track_price_override <- unique(DT.track_price_override, by=c("upc", "unique_track_id"))
convertNumberCols_(DT.track_price_override, verbose=FALSE)
setkeyIfNot(DT.track_price_override,  "upc", organize=TRUE, verbose=FALSE)

# DT.upc_has_override <- DT.track_price_override[, list(numb_of_track_overrides = lunique(unique_track_id)), keyby=list(upc, country_code, cd)]
cat("Calculating DT.upc_has_override which will take about 5~10 seconds\n")
DT.upc_has_override <- DT.track_price_override[, list(numb_of_track_overrides = lunique(unique_track_id), relative_to_default = ifelse(mean(as.num.as.char(price_code)) > 99, "higher", "lower")), keyby=list(upc, cd)]
DT.upc_has_override <- merge(DT.upc_has_override, DT.track_and_CD_counts, allow=TRUE, all=TRUE)
upcToID_(DT.upc_has_override)
DT.upc_has_override[upc == RockyHorror.upc, ]
DT.upc_has_override[, perc.tracks_with_override := numb_of_track_overrides / total_tracks]
DT.upc_has_override[is.na(numb_of_track_overrides), `:=`(numb_of_track_overrides=0L, perc.tracks_with_override=0, relative_to_default="Zero Overrides")]

## Discretize the Override Percentage
suppressWarnings(DT.upc_has_override[ , OverridePerc := NULL])
DT.upc_has_override[perc.tracks_with_override == 1, OverridePerc := "100 %"]
DT.upc_has_override[perc.tracks_with_override == 0, OverridePerc := "  0 %"]
DT.upc_has_override[perc.tracks_with_override != 1 & numb_of_track_overrides == 1, OverridePerc := "Single Track Override"]
DT.upc_has_override[perc.tracks_with_override != 1 & numb_of_track_overrides > 1, OverridePerc := "Multi Track Override"]
DT.upc_has_override[, OverridePerc := factor(OverridePerc, levels = c("  0 %", "Single Track Override", "Multi Track Override", "100 %"))]
## CONFIRM: No NAs in OverridePerc.  (Neither because not caught when assigning, nor when making into factor)
stopifnot(DT.upc_has_override[, !is.na(OverridePerc)])

## There are very few triple-cds or above.  Create a binary field. However, keep the total cd count for averages
DT.upc_has_override[, multi_cd := ifelse(total_cds > 1, "YES", "NO")]
DT.upc_has_override[OverridePerc == "Multi Track Override"][, total_tracks - numb_of_track_overrides]
## Discretized tracks
track_count_bounds <- c(0, 9, 15, 22, 50, Inf)
track_count_bounds <- c(0, 6, 10, 12, 18, 22, 50, Inf)
track_count_bounds <- c(0, 10.5, Inf)
DT.upc_has_override[, discretized_tracks := cut(total_tracks, track_count_bounds)]

summary(DT.upc_has_override)