## STARTING WITH DT.toptracks
if (!exists("DT.toptracks")) {
  warning ("DT.toptracks should exist. Did you forget to run 01 Scrape..r ?\nWill load from Jesus. Might be outdated")
  loadFromJesus("DT.toptracks", over=TRUE)
}

kCols         <- c("country", "date")
kCols_streams <- c("country", "date", "top_rank")
DT.toptracks[, date := as.Date(date)]
DT.toptracks[, top_rank := rank(-num_streams), by=kCols]
setkeyIfNot(DT.toptracks, kCols_streams)
setcolorderpt(DT.toptracks, c(kCols_streams, "num_streams", "artist_name", "album_name", "track_name"))
## < SEE > 
if (FALSE) {
  desc(DT.toptracks)
  DT.toptracks
}


# DT.toptracks[DT.toptracks[, {browser(expr=any(is.na(.I[1:10]))); .I[1:10]}, keyby=kCols]$V1]
DT.Agg_lists <- list()
for (N in c(5, 10, 20, 50, 100, 200)) {
 DT.Agg_lists[[as.character(N)]] <-  setnames(DT.toptracks[, if (.N >= N) sum(num_streams[1:N]) else NA_real_, keyby=kCols], "V1", sprintf("top%02i", N))
}
DT.Agg <- Reduce(merge, DT.Agg_lists)
DT.Agg <- melt(DT.Agg, id.vars=kCols, variable.name="topx", value.name="total_streams")

DT.Agg[, fromdate := date - 6]
DT.Agg[, todate   := date]
DT.Agg[, avg_daily_streams := total_streams / 7]


DT.daily <- DT.Agg[, list(day=seq.Date(from=fromdate, to=todate, by="day"), avg_daily_streams), keyby=c(kCols, "topx")]
DT.daily[, date := NULL]
DT.daily[, month := lubridate::floor_date(day, "month")]
DT.monthly <- DT.daily[, list(avg_monthly_streams = round(sum(avg_daily_streams))), keyby=c("country", "month", "topx")]


## SAVE
{
  cat ("Saving ... this will take about 30 seconds", fill=TRUE)
  jesusForData(DT.monthly, DT.daily, DT.Agg, DT.toptracks)
  saveImageTo()
}



## TODO: 
" &&& "
## Pull in total streams from spotify numbers
## Calculate as a percentage of Total Spotify streams

loadFromJesus("DT.revshare",         srcDir=srcOther("Spotify_Accounting_ETL"), over=TRUE)
loadFromJesus("DT.data.Total_usage", srcDir=srcOther("Spotify_Accounting_ETL"), over=TRUE)


## Cleanup the loaded DTs
DT.data.Total_usage[, month := as.Date(month)]
DT.revshare[, month := as.Date(month)]
DT.revshare[, currency_rate_per_USD_calculated := payable_usd / payable]
DT.revshare[, net_revenue_usd   := net_revenue   * currency_rate_per_USD_calculated]
DT.revshare[, gross_revenue_usd := gross_revenue * currency_rate_per_USD_calculated]

setkeyIfNot(DT.data.Total_usage, "country_code", "month")


DT.reported_totals <- DT.revshare[, list(spotify_streams=sum(as.numeric(total_tracks)), orchard_streams=sum(as.numeric(rightholders_tracks))), keyby=list(country_code, month)]
DT.reported_totals <- rbind(DT.reported_totals, DT.reported_totals[, list(country_code = "global", spotify_streams = sum(spotify_streams), orchard_streams=sum(orchard_streams)), keyby="month"], use.names=TRUE)
setkeyIfNot(DT.reported_totals, country_code, month, verbose=FALSE)

DT.monthly[DT.reported_totals
            , `:=`(
                  "Orchard_Total_Streams" = orchard_streams
                , "Spotify_Total_Streams" = spotify_streams
            ), allow=TRUE]

DT.monthly[, monthly_as_perc_of_orchard := avg_monthly_streams / Orchard_Total_Streams]
DT.monthly[, monthly_as_perc_of_spotify := avg_monthly_streams / Spotify_Total_Streams]

## < SEE >
DT.monthly[(month == "2014-10-01")]
# DT.monthly[DT.data.Total_usage
#             , `:=`(
#                   "Orchard_Total_Streams" = orchard_streams_above_30_sec
#                 , "Spotify_Total_Streams" = all_streams_above_30_sec
#             )]


## --- Plotting ----- 


### --- weekly global absolute ----- ###
DT.plot_weekly <- melt(Reduce(merge, DT.Agg_lists)[.("global")], id.vars=kCols, variable.name="topx", value.name="total_streams")

#---------------------- #
### --- weekly USA absolute ----- ###
DT.plot_weekly <- melt(Reduce(merge, DT.Agg_lists), id.vars=kCols, variable.name="topx", value.name="total_streams")
setkeyIfNot(DT.plot_weekly, country, date)
## Add labels at local maximums
DT.plot_weekly[topx == "top50", label := ifelse(is_local_max(total_streams), as.character(date), NA_character_), by=country]
## Avoid labels that are on top of each other.  Make sure they are at least three weeks apart
DT.plot_weekly[topx == "top50" & !is.na(label), label := ifelse(diffNA(date, padTop=FALSE, fill=1000)/7 < 3, NA, label), by=country]


P.ll.weekly_absolute_bycountry <- emptylist(unique(c("global", "US", "GB", "SE", DT.plot_weekly$country)))

for (nm in names(P.ll.weekly_absolute_bycountry))
  P.ll.weekly_absolute_bycountry[[nm]] <- {
    if (DT.plot_weekly[.(nm)][topx == "top10", sum(is.na(total_streams)) / .N > .7])
      NULL
    else 
    {
      ggLinegraph(DT = DT.plot_weekly[.(nm)][!is.na(total_streams)], x = "date", y="total_streams"
      , color="topx"
      , dotsize=1
      , yscale="millions"
      , label = "label"
      , labelcolor = list("#FFFFFF", "#444444")
      , labelsize = 2.5
      , labelface = "bold"
      , labelalpha= c(.85, 1)
      , label_y_up = 1/15
      ## These uncommented lines are for when using printToPDF on the results
      , title = DT.country[.(nm)][, ifelse(is.na(country_name), nm, country_name)]
      , xlab  = NULL
      # , title= sprintf("Spotify [%s]", DT.country[.(nm)][, ifelse(is.na(country_name), nm, country_name)])
      # , title_append = "\nTotal number of streams attributed to Top 'x#' tracks each week"
      # , xlab_append = "Blue line is introduction of 'Top 200' list"
      , vline = ("2014-10-27")
      , legend = if (which(nm == names(P.ll.weekly_absolute_bycountry)) %% 6 == 1)  "top" else "off"
      )
    }
  }

printToPDF(P.ll.weekly_absolute_bycountry, f.name="weekly_absolute_bycountry", main = "Spotify", height.per=6, foot=orchardFootNote(), sub="Total number of streams attributed to the weekly Top 'x#' tracks", showWarnings.sub=FALSE)


## ~~~ Graba standalone graph for global ~~~ #
P.global_weekly_absolute <- 
  ggLinegraph(DT = DT.plot_weekly[.("global")][!is.na(total_streams)], x = "date", y="total_streams"
  , color="topx"
  , dotsize=1
  , yscale="millions"
  , label = "label"
  , labelcolor = list("#FFFFFF", "#444444")
  , labelsize = 2.5
  , labelface = "bold"
  , labelalpha= c(.85, 1)
  , label_y_up = 1/15
  # ## These uncommented lines are for when using printToPDF on the results
  # , title = DT.country[.(nm)][, ifelse(is.na(country_name), nm, country_name)]
  # , xlab  = NULL
  , title= "Spotify Global Across All Countries"
  , title_append = "Total number of streams attributed to Top 'x#' tracks each week"
  , xlab_append = "Blue line is introduction of 'Top 200' list"
  , vline = ("2014-10-27")
  , legend = "top"
  )

ggsave.out(P.global_weekly_absolute)
#---------------------- #

#---------------------- #
dates_using <- as.Date(c("2014-01-12", "2014-10-12", "2014-11-09"))
countries_using_all <- c("US", "GB", "SE", "DE", "MX", "NL", "ES", "FR", "PH", "CL", "AU", "NO")
countries_using <- c("US", "GB", "SE", "DE", "MX", "NL", "ES", "PH", "CL", "AU", "NO")
#---------------------- #


#---------------------- #
{
  ggLinegraph(DT = DT.Agg_lists[["50"]][date %in% dates_using & country %in% countries_using], x = "date", y="top50", color="country", yscale="millions"
    , title="Spotify"
    , title_append="Total number of streams attributed to Top 'x' tracks each week"
    )
}

#---------------------- #
"Total number of streams attributed to Top 'x' tracks each week"
## ~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=-~=~ ##
DT.top50  <- copy(DT.Agg_lists[["50"]])
setnames(DT.top50, "top50", "total_streams")
DT.top50[, streams_as_perc_of_max     := total_streams / max(total_streams), by=country]
DT.top50[, streams_as_perc_of_max_MA3 := forecast::ma(streams_as_perc_of_max, 3), by=country]

DT.top50[, streams_scaled     := scale(total_streams), by=country]
DT.top50[, streams_scaled_MA3 := forecast::ma(streams_scaled, 3), by=country]

DT.top50[(date < "2014-10-12"), streams_scaled_pre     := scale(total_streams), by=country]
DT.top50[, streams_scaled_pre_MA3 := forecast::ma(streams_scaled_pre, 3), by=country]

{
  P.streams_as_perc_of_max_MA3 <- 
  ggLinegraph(DT = DT.top50[ country %in% countries_using][!is.na(streams_as_perc_of_max_MA3)]
    , x = "date", y="streams_as_perc_of_max_MA3", color="country"
    , yscale="percent"
    , size = .5, dots=0
    , title="Spotify"
    , title_append="Streams for Top-50 by region (scaled to regional max)"
    , ylab =  "Regional Total Streams of Top 50\nas percent of max streams for top 50 in the time period for the region (3 wk MA)"
    )
}
{
  P.streams_scaled_pre_MA3 <- 
  ggLinegraph(DT = DT.top50[ country %in% countries_using][!is.na(streams_scaled_pre_MA3)]
    , x = "date", y="streams_scaled_pre_MA3", color="country"
    , yscale="percent"
    # , y_lim = c(-101, 250)/100
    , size = .5, dots=0
    , title="Spotify"
    , title_append="Streams for Top-50 by region (scaled relative to regional mean)"
    , ylab =  "Regional Total Streams of Top 50\nas percent of max streams for top 50 in the time period for the region (3 wk MA)"
    )
}

## Why all the missing values? 
ggsave.out(P.streams_as_perc_of_max_MA3)
ggsave.out(P.streams_scaled_pre_MA3)



ggplot(DT.plot_weekly[topx != "top200"], aes(x=date, color=topx, y=total_streams)) + geom_line() + geom_point() + millions.y() + ggtitle("Spotify\nTotal number of streams attributed to Top 'x' tracks each week globally")
ggsave.out(P.global_weekly_absolute)
### --- weekly global absolute ----- ###

### --- weekly monthly relative ----- ###
DT.plot_monthly.molt <- melt(DT.monthly, id.vars=c("country", "month", "topx"), measure.vars=c("monthly_as_perc_of_orchard", "monthly_as_perc_of_spotify"), value.name="topx_streams_percent_of___")
setnames(DT.plot_monthly.molt, "variable", "of")
DT.plot_monthly.molt[, of := gsub(".*_", "", of)]
DT.plot_monthly.molt <- DT.plot_monthly.molt[!is.na(topx_streams_percent_of___)]
setkeyIfNot(DT.plot_monthly.molt, "country")


# P.global_monthly_relative <- 
  ggplot(DT.plot_monthly.molt[.("global")][topx=="top50"], aes(x=month, color=of, y=topx_streams_percent_of___)) + geom_line() + geom_point() + percent.y() + ggtitle("Spotify\nTotal number of streams attributed to Top 'x' tracks each month globally")
# ggsave.out(P.global_monthly_relative)

### --- weekly monthly relative ----- ###


## missing data at the end.... 
P.global_monthly_absolute <- ggplot(DT.monthly[.("global")], aes(x=month, color=topx, y=avg_monthly_streams)) + geom_line() + geom_point() + millions.y() + ggtitle("Spotify\nTotal number of streams attributed to Top 'x' tracks each month globally")
ggsave.out(P.global_monthly_absolute)

ggplot(DT.monthly[.("global")], aes(x=month, color=topx, y=avg_monthly_streams)) + geom_line() + geom_point() + millions.y()



ggplot(DT.monthly[.("global")] [!is.na(monthly_as_perc_of_spotify)] [topx == "top50"], aes(x=month)) + geom_line(aes(y=monthly_as_perc_of_orchard), color="orange") + geom_line(aes(y=monthly_as_perc_of_spotify), color="green")