## `dateMin.SpotifyAdds` should have been set in 01.A or again in 01.C.  
##    However if not run, may be missing, so we will recreate just in case
if (!exists("dateMin.SpotifyAdds"))  {
	if (exists("SpotifyAdds"))
		dateMin.SpotifyAdds <- min(SpotifyAdds[!is.na(upc)][["startDate"]]) - 100
	else if (exists("DB.SpotifyMeta"))
		dateMin.SpotifyAdds <- min(DB.SpotifyMeta[!is.na(upc)][["startDate"]]) - 100
	else if (exists("DB.SpotifyMeta.Long"))
		dateMin.SpotifyAdds <- min(DB.SpotifyMeta.Long[!is.na(upc)][["startDate"]]) - 100
	else
		stop("No SpotifyAdds exists to set `dateMin`")
}

## The mkt_priority column should be a factor
if (!is.factor(DB.all.countmeta[["mkt_priority"]]))
	DB.all.countmeta[, mkt_priority := factor(mkt_priority)]

## Create a DB, which is a subset of DB.all, where only A's & B's priorities are kept
##     and the releaseDate is on or after `dateMin.Spotifyadds`
DB.all.AnB <- DB.all.countmeta[!is.na(mkt_priority)][releaseDate >= dateMin.SpotifyAdds]

## Grab the unique UPCs (and a few other infos) from the above DB
AnB.upcs <- unique(DB.all.AnB[,  list(mkt_priority, releaseDate, artist), keyby=upc], by=NULL)

# no UPC should have more than one priority
stopifnot(0==nrow(AnB.upcs[, .N, by=upc][N>1]))

## Grab the subset of UPCs that are not in Spotify
AnB.upcs.notInSpotify <- AnB.upcs[!unique(DB.SpotifyMeta[["upc"]])][order(releaseDate, decreasing=TRUE)]

#  ## Only keep those since Oct 2012
#  AnB.upcs.notInSpotify <- AnB.upcs.notInSpotify[releaseDate > "2012-10-01"]


## There should be no intersection between the UPCs from DB.raw (which represents the UPCs in SpotifyAdds) and those from AnB.upcs...
stopifnot(0 == length(intersect(DB.raw[["upc"]], AnB.upcs.notInSpotify[["upc"]])))

## Save to File
if (tolower(.Pfm) == "linux")  {
	file.AnBs <- as.path(outDir, "AnBs_notUsed.csv")
	write.table(AnB.upcs.notInSpotify, file=file.AnBs, sep=",", row.names=FALSE, col.names=TRUE, fileEncoding="UTF-8")
	## Show copy statement 
	AWS <- "root@107.20.201.0"
	paste0("scp ", AWS, ":/",file.AnBs, " ", gsub(path.expand(wrkDir), wrkDir, file.AnBs))
}

#   ----------------------------------------------------------------------------
#   ----------------------------------------------------------------------------
#   
#   subProj <- "AvsB_Agg"
#   ## CREATING THRESHOLDS
#   
#   ## Across alllll UPCs
#   DB.all.using
#   
#   ## alllll As & Bs
#   DB.all.using[!is.na(mkt_priority)]
#   
#   ## unused As & Bs
#   matchKey(DB.all.using, AnB.upcs.notInSpotify, "upc")
#   DB.all.using[AnB.upcs.notInSpotify]
#   
#   ## Spotify As & Bs
#   DB.raw # counting 
#                t.MonthlyCount.byupc,   u.MonthlyCount.byupc
#   by  
#   c("measuring", "month", "user_country", "genre", "store")
#   ----------------------------------------------------------------------------

#   ----------------------------------------------------------------------------
DB.raw.monthly <-
 unique(DB.raw[, list(
	upc
	, t.MonthlyCount.byupc_Genre_UserCountry
	, u.MonthlyCount.byupc_Genre_UserCountry
	, month=month.abb[dl_month], user_country, genre, mkt_priority
	, store="Spotify"
	)]
 , by=NULL)

othcols <- setdiff(names(DB.raw.monthly), c("t.MonthlyCount.byupc_Genre_UserCountry", "u.MonthlyCount.byupc_Genre_UserCountry"))

DB.raw.monthly <- 
rbind(
    DB.raw.monthly[, list(measuring="total", count=t.MonthlyCount.byupc_Genre_UserCountry), by=othcols]
  , DB.raw.monthly[, list(measuring="unique", count=u.MonthlyCount.byupc_Genre_UserCountry), by=othcols]
)
#   ----------------------------------------------------------------------------

## MAKE SURE NO NA's IN THE count OF THE DT'S BEING USED
stopifnot(0 == is.na(DB.all.countmeta[["count"]]))
stopifnot(0 == is.na(DB.raw.monthly[["count"]]))

## Replace this
DB.all.using <- DB.all.countmeta[releaseDate >= dateMin.SpotifyAdds] [!is.na(count)]

FullSummary <- function(vec) {
	c(summary(vec), SD=sd(vec), MAD=mad(vec), N=length(vec))
}

byCols.agg <- c("measuring", "month", "user_country", "genre", "store")

AggdCounts.all            <- DB.all.using[, as.list(FullSummary(count)), by=byCols.agg]
AggdCounts.AsAndBs_All    <- DB.all.using[!is.na(mkt_priority), as.list(FullSummary(count)), by=byCols.agg]
AggdCounts.AsAndBs_UnProm <- DB.all.using[AnB.upcs.notInSpotify, as.list(FullSummary(count)), by=byCols.agg]
AggdCounts.Spotify        <- DB.raw.monthly[, as.list(FullSummary(count)), by=byCols.agg]

ByMkt.AggdCounts.all            <- DB.all.using[, as.list(FullSummary(count)), by=c("mkt_priority", byCols.agg)]
ByMkt.AggdCounts.AsAndBs_All    <- DB.all.using[!is.na(mkt_priority), as.list(FullSummary(count)), by=c("mkt_priority", byCols.agg)]
ByMkt.AggdCounts.AsAndBs_UnProm <- DB.all.using[AnB.upcs.notInSpotify, as.list(FullSummary(count)), by=c("mkt_priority", byCols.agg)]
ByMkt.AggdCounts.Spotify        <- DB.raw.monthly[, as.list(FullSummary(count)), by=c("mkt_priority", byCols.agg)]


-----------------------------------------------------

	f.out <- as.path(dataDir, "Aggd_DTs.Rda")
	save(
		AggdCounts.all
	, AggdCounts.AsAndBs_All
	, AggdCounts.AsAndBs_UnProm
	, AggdCounts.Spotify
	, ByMkt.AggdCounts.all
	, ByMkt.AggdCounts.AsAndBs_All
	, ByMkt.AggdCounts.AsAndBs_UnProm
	, ByMkt.AggdCounts.Spotify
	, file=f.out)

	AWS <- "root@107.20.201.0"
	paste0("scp ", AWS, ":/",f.out, " ", gsub(path.expand(wrkDir), wrkDir, f.out))


-----------------------------------------------------

LOAD THIS FILE LOCALLY: 

load("/Users/rsaporta/gitData/ImageSaves/SpotifyAdds_ImageSave_20131109_0124.Rda")

## THIS ONE REMOTELY:
SpotifyAdds_COMPLETED.01.02.03.04_20131108_2007.Rda


> lsos(DT)

SAMPLES:
                       Name      Size       Type     Rows Columns         Key
1:         DB.all.countmeta   3.12 GB data.table 25783394      16         upc
2:    SAMP.DB.all.countmeta   3.12 GB data.table 25783394      16         upc
3:                   DB.raw 230.23 MB data.table   499992      63         upc
4:              SAMP.DB.raw 230.23 MB data.table   499992      63         upc
5:      DB.SpotifyMeta.Long  47.88 KB data.table       81      40   upc, Week
6: SAMP.DB.SpotifyMeta.Long  47.88 KB data.table       81      40   upc, Week
7:           DB.SpotifyMeta  34.48 KB data.table       27      42         upc
8:      SAMP.DB.SpotifyMeta  34.48 KB data.table       27      42         upc