setScience(projName="Spotify_Accounting_ETL", subProj="Overview")
library(forecast)
library(animation)

setGitBranchToSystem()
.g()

DT.spotify_raw <- sfQry(sprintf("SELECT tmstamp::date as activity_date, user_country as country_code, %s, count(*) as streams from production.staging_raw_spotify_v2 group by 1, 2, 3 order by 1, 2, 3", getSpotifyCaseStmnt()))
jesusForData(DT.spotify_raw, projName="Spotify_Accounting_ETL")

DT.applemusic <- sfQry("SELECT datestamp as activity_date, storefront_name as country_code, count(*) as streams from applemusic.applemusicraw group by 1, 2 order by 1, 2")
DT.applemusic <- DT.applemusic[activity_date <= "2015-08-23"]
jesusForData(DT.applemusic, projName="Spotify_Accounting_ETL")

DT.applemusic
streams <- DT.spotify_raw$streams

## DAILY DATA ONLY
spotify_streams_ts <- ts(DT.spotify_raw, start=1, frequency=7, class="ts")

is(spotify_streams_ts)
fit.spotify_streamsold
fit.spotify_streams <-
 forecast::auto.arima(spotify_streams_ts)
forecast(fit.spotify_streams, )
DT.spotify_raw




--------- Time Series Example ---------------------------
dict.colors.forecast <- c("actual streams" = "#07130D", "forecasted streams"="#555555")

DT.spotify_raw[, value_type := "actual streams"]

holidates <- seq.Date(as.Date("2014-12-24"), as.Date("2015-01-11"), by="day") %>% c(as.Date("2014-10-31"))
holidates
DT.spotify_ts <- DT.spotify_raw[activity_date <= "2015-05-25"]
DT.spotify_ts[, streams := ifelse(activity_date %in% holidates, NA, streams)]
ggLinegraph(DT.spotify_ts, y="streams")

ts.spot <- ts(DT.spotify_ts$streams, frequency=7)
fitted.spotify <- auto.arima(ts.spot)

dates.forecasted  <- seq.Date(max(DT.spotify_ts$activity_date)+1, to=max(DT.spotify_raw$activity_date)+1+14, by="day")
values.forecasted <- forecast(fitted.spotify, length(dates.forecasted))$mean

DT.spotify_forecasted <- data.table(
        activity_date = dates.forecasted
      , streams = as.numeric(values.forecasted)
      , value_type = "forecasted streams"
      )

matchKey(DT.spotify_raw, DT.spotify_forecasted, key="activity_date")
DT.expected <- rbind(DT.spotify_forecasted, DT.spotify_raw[!DT.spotify_forecasted]) %>% setkey(activity_date)
DT.actual_and_expected <- rbind(DT.spotify_forecasted, DT.spotify_raw) %>% setkey(activity_date)
DT.actual_and_expected[, has_forecasted_value := "forecasted streams" %in% value_type, by=activity_date]
DT.actuals <- DT.actual_and_expected[(activity_date >= min(activity_date[has_forecasted_value])-1 ) & value_type == "actual streams"]



keyDates <- c("Apple Music\nAnnouncement\nat WWDC" = "2015-06-08", "Apple Music\nLaunches\n" = "2015-07-01", "Halloween" = "2014-10-31", "Christmas  " = "2014-12-24", "  New Years" = "2015-01-02") %>% {setNames(nm=names(.), obj=as.Date(.))}

setkeyIfNot(DT.expected, activity_date)
DT.keyDates <- DT.expected[.(keyDates)]
## Text is the naes of keyDates
DT.keyDates[, text := names(keyDates)]
## New Years specifically is low, crank itup
DT.keyDates[trim(text) == "New Years", streams := max(streams, 50000000)]
DT.keyDates[trim(text) == "Apple Music\nAnnouncement\nat WWDC", streams := streams + 2000000]
## Pump up the strems, so text appears higher
DT.keyDates[, streams := streams * 1.1]
setkeyIfNot(DT.keyDates, activity_date)