lsosdt()

DT.spotify_product_dict

## These are the reports delivered individually
DT.breakage
DT.downloads
DT.revshare

## This is the "additional data" sent by Spotify
DT.Activity_and_usage
DT.Ad_impressions
DT.Breakage_Rate
DT.Conversion
DT.Streams_per_platform
DT.Total_usage
DT.Trial_uptake


DT.Activity_and_usage
DT.Trial_uptake
DT.Conversion
DT.Ad_impressions
DT.Total_usage
DT.Breakage_Rate
DT.Streams_per_platform


DT.Trial_uptake
DT.Total_usage
DT.Breakage_Rate
DT.Streams_per_platform

lapply(list(
   DT.Activity_and_usage
, DT.Trial_uptake
, DT.Conversion
, DT.Ad_impressions
, DT.Total_usage
, DT.Breakage_Rate
, DT.Streams_per_platform
), names)


------------------------------------------------
DT.EU <- get_DT.EU()
stopifnot(DT.EU$country_code %in% DT.Trial_uptake$country)
# ------------------------------------------------

DT.plot <- copy(DT.Trial_uptake)
addColsFrom_(DT.plot, DT.EU, joinCols.g="country_code", joinCols.r="country")
setnames(DT.plot, "region_group", "cgroup")
DT.plot <- DT.plot[!is.na(cgroup)]

y.var = "No._of_trials_starting_during_the_month"
## Dont keep country name, just country group
byCols <- c(setdiff(key(DT.plot), extract("country", DT.plot)), "cgroup")
DT.plot[, extract("country", DT.plot) := NULL]


## Aggregate
DT.plot.agg <- DT.plot[, lapply(.SD, sumn), keyby=byCols]
## Drop any type that only has one or two data points
DT.plot.agg <- DT.plot.agg[type_of_trial %in% DT.plot.agg[, .N, by=type_of_trial][N > 2, type_of_trial]]

lib(ggplot2)
ggplot(data = DT.plot.agg, aes_string(x="month", y=y.var, color="cgroup")) + geom_line(size=1.4) + facet_grid(type_of_trial ~ ., scales="free_y") + thousands.y() + ylab(paste0(gsub("_", " ", y.var), " (in thousands)"))
----------------------------


cls(10)
DT.Breakage_Rate [date_file == "2014-07-01"]
DT.breakage           [date_file == "2014-07-01"]

DT.Breakage_Rate [date_file == "2014-08-01"]
DT.breakage           [date_file == "2014-08-01"]

                            name   rows cols                                                    keys
 1:                  DT.revshare 10,101   19
 2:                       counts  4,606    3                             folder, activity_month_date
 3:         SpotifyArchive.Files  4,606    8
 4:   DT.Activity_and_usage  2,739   11
 5:                 DT.downloads  1,487   12
 6:           DT.Conversion  1,057   10
 7:         DT.Trial_uptake    978    7
 8:       DT.Ad_impressions    798    7
 9:                    DT.stores    525   10                                     storeid, store_name
10:                      folders    426    2
11:          DT.Total_usage    399   13
12:                  DT.breakage    395   11 Partner, Product, Country, Currency, Pro rata share, ..
13: DT.Streams_per_platform    281    8
14:        DT.Breakage_Rate    133   11
15:           DT.RowCounts.byday    122    2                                            activitydate
16:       SpotifyArchive.Folders    122    4
17:       JulyFolders.subfolders    113    6
18:                   SubFolders    111    5
19:       DT.spotify_product_dict     83    2                        Product tag, Product description
20:               DT.spotify_gpu     48    4
21:    DT.spotify_fact_analytics      6    3                                          activity_month
22:                           DT      5    1                                                      ID
23:                 DT.RowCounts      4    2                     Activity Month, Total Monthly Units
24:         DT.MISSING_DATA      1   12
--------------


dates.revshare <- (setkeyIfNot(rbindlist(lapply(selfname_(names(DT.revshare), showWarnings=FALSE), function(col) DT.revshare[is.na(get(col)), if (.N) list(col=col, min=min(date), max=max(date))])), min, max, organize=TRUE))
dates.revshare[!(setkey(DT.revshare[, as.list(range(date))])) ]
DT.revshare[is.na(`Report start date`)]

DT.revshare[date == "2011-09-01"][is.na(`Report start date`)]
dates.revshareFiles <- dateFromSpotifyFile(revshareFiles)
dupdates <- unique(dates.revshareFiles[duplicated(dates.revshareFiles)])
wh.dups <- dates.revshareFiles %in% dupdates
cbind(names(revshareFiles)[wh.dups][order(dates.revshareFiles[wh.dups])])
revshareFiles[which( == "2011-09-01")]


range(dateFromSpotifyFile(extract("download", fullFilePaths)))


reproduce(extract("2014", fullFilePaths), whole=TRUE)
DT.breakagefiles <- as.data.table(listFlatten(strsplit(namesOfFiles.txt, "-|\\.") ))

c(namesnamesOfFiles.txt, namesOfFiles.xls)

# --------------------------------- #

TODO:  Keep a running list of what I''ve brought
Update new files regularly. 

# --------------------------------- #