## rdio marketshare - process files.r

## FIlES ARE On the O DRivE: 
##    cd /ODrive/Orchard\ Accounting/Digital\ Accounting\ Folder/DMS\ Transactional\ Files/Statement\ Archive/RD.IO/rd.io\ DONE/2015-08\ \&\ 09/


# screen -xRR rdio


setScience(proj="marketshare", subProj="RDIO", create=TRUE)

check_countries <- FALSE

if (check_countries) {
  wh <- "looker_wh_large"
  dbname <- "prod"
  setSnowflake(wh=wh, dbname="prod")
}


## Folder were files are located
folder <- "/ODrive/Streaming DMS Revenue Summaries/Rdio/2015/2015-09/rdio_orchard_statements-201509"

## Confrim is a dir
stopifnot(file.exists(folder), isdir(folder))

files <- extractFilesFromFolder(folder, ext="csv", full=TRUE)
files <- files[!grepl("summary", files)]

stopifnot(length(files) > 0)

rdio_extractCountryFromFileName <- function(nm) {
  nm %>% removeText("rdio_", .) %>% strsplit("_") %>% sapply(head, 1)
}

countries <- files %>% names %>% rdio_extractCountryFromFileName

if (check_countries) {
  DT.country <- get_dim_country()
  stopifnot(countries %in% DT.country$country_code)
}

ll_DTs <- emptylist(files)
verboseMsg(verbose, "Will process", length(files), "files")
for (nm in names(files)) {
  verboseMsg(verbose, "Processing '", nm, "'", sep="")
  f <- files[[nm]]
  raw <- readLines(f) %>% pasteC(C="\n")
  DT <- strsplit(raw, "\\n\\n")[[c(1, 2)]] %>% fread(skip=1, header=TRUE)
  DT[, country_code := rdio_extractCountryFromFileName(nm)]
  ll_DTs[[nm]] <- copy(DT)
}

DT.rdio <- rbindlist(ll_DTs, fill=TRUE) %>% cleanColNamesForSQL_

## -------------------------------------------- ##
## BR and Mexico are fucked up -- clean them
## -------------------------------------------- ##
cols_to_fix <- c("user_count", "revenue", "taxes")
DT.rdio[country_code %in% c("BR", "MX"), (cols_to_fix) := lapply(.SD, removeText, pattern="\\s+.+"), .SDcols=cols_to_fix]
DT.rdio[, (cols_to_fix) := lapply(.SD, as.numeric), .SDcols=cols_to_fix]
DT.rdio_ww <- aggregateDT(DT.rdio, by="product", exclude=c("country_code", "taxes"), showWarnings.info=TRUE)
## -------------------------------------------- ##


## CALCULATE MARKETSHARE
{
  catheader("By Product", endl=0)
  DT.rdio_ww_marketshare_by_product <- DT.rdio[, list(orchard_streams=sum(partner_plays), total_rdio_streams=sum(total_plays)), by=product][, marketshare := orchard_streams / total_rdio_streams] []
  print(DT.rdio_ww_marketshare_by_product)

  catheader("Overall", endl=0)
  DT.rdio_ww_marketshare <- DT.rdio[, list(orchard_streams=sum(partner_plays), total_rdio_streams=sum(total_plays))][, marketshare := orchard_streams / total_rdio_streams] []
  print(DT.rdio_ww_marketshare)
}

writeDT(DT.rdio_ww_marketshare)
writeDT(DT.rdio_ww_marketshare_by_product)