# Spotify_Streams_Per_Top_Artist setScience("Spotify_Streams_Per_Top_Artist", create=TRUE, subl=FALSE, load=FALSE) lib(rjson) lib(RCurl) lib(reshape2) lib(ggplot) lib(bit64) sourceSupportFns() countries.avail <- c("global", DT.spotify_countries[, country_code]) # dates.avail <- head(as.character(seq.Date(as.Date("2013-04-28"), Sys.Date()+1, by="7 days")), -1) ## I thought that there would be a rounding issue with the date, but I amy have been wrong dates.avail <- as.character(seq.Date(as.Date("2013-04-28"), Sys.Date(), by="7 days")) ## Threshold for number of items that must be returned to be considered valid for the territory nrowThresh <- 46 { if (exists("DT.nms")) { DT.nms.bak <- copy(DT.nms) assign(timeStamp("DT.nms", seconds=TRUE), value=DT.nms) } DT.nms <- CJ(country=countries.avail, date=dates.avail) DT.nms[, nms := sprintf("%s.%s", country, date)] DT.nms[, url := sprintf("http://charts.spotify.com/api/tracks/most_streamed/%s/weekly/%s", country, date)] setkeyIfNot(DT.nms, "nms", organize=TRUE, warnForColNameInEnv=FALSE, verbose=FALSE) if (exists("DT.nms.bak")) { setkeyIfNot(DT.nms.bak, "nms", organize=TRUE, warnForColNameInEnv=FALSE, verbose=FALSE) DT.nms[DT.nms.bak, ret_json := i.ret_json] rm(DT.nms.bak) } } ## Organize the names, because I believe there are API limits ## first the global, then anything with 2014 nms.ordered <- c( DT.nms[country == "global" & date >= "2014-02-01", nms], DT.nms[country %in% c("US", "SE") & date >= "2014-02-01", nms], DT.nms[country %ni% c("global", "US", "SE") & date >= "2014-02-01", nms], DT.nms[country %in% c("global", "US", "SE") & date < "2014-02-01", nms], DT.nms[country %ni% c("global", "US", "SE") & date < "2014-02-01", nms] ) stopifnot(identical(sort(nms.ordered), sort(DT.nms$nms))) for (nm in nms.ordered) { ## Recovery after error # for (nm in nms.ordered[(which(nms.ordered == nm) + 1) : length(nms.ordered)]) { cat("Processing : ", nm, "\n") ## don't pull again if already pulled if (exists("DTs.list") && isTRUE(nrow( DTs.list[[nm]] ) >= nrowThresh)) { "do nothing" } else { ret <- try(getURL(DT.nms[.(nm)]$url)) if (isErr(ret)) { warning (sprintf("ERROR: '%s', '%s'", DT.nms[.(nm)]$country, DT.nms[.(nm)]$date)) } else { if (!grepl("^.\"tracks\"", substr(ret, 1, 100))) warning (sprintf("POSSIBLE ERROR: '%s', '%s'", DT.nms[.(nm)]$country, DT.nms[.(nm)]$date)) DT.nms[.(nm), ret_json := ret] } } if (nm %in% nms.ordered[unique(1 + seq_along(nms.ordered) %/% 100) * 100 ]) jesusForData(DT.nms, info="mid step save") } jesusForData(DT.nms, info="with raw data") f.lastSaved <- saveImageTo() # for (date in dates.avail) { # cat("Processing date: ", date, "\n") # tracks[[date]] <- emptylist(countries.avail) # for (country in countries.avail) { # U <- sprintf("http://charts.spotify.com/api/tracks/most_streamed/%s/weekly/%s", country, date) # urls <- c(urls, U) # if (!is.null(tracks [[date]] [[country]])) { # cat ("skipping : ", country, "\n") # next # } else # cat ("processing : ", country, "\n") # ## don't pull again if already pulled # if (exists("DTs.list") && isTRUE(nrow( DTs.list[[sprintf("%s.%s", date, tolower(country))]] ) >= nrowThresh)) { # "do nothing" # } else { # ret <- try(getURL(U)) # if (isErr(ret)) { # warning (sprintf("ERROR: '%s', '%s'", country, date)) # } else { # if (!grepl("^.\"tracks\"", substr(ret, 1, 100))) # warning (sprintf("POSSIBLE ERROR: '%s', '%s'", country, date)) # tracks [[date]] [[country]] <- ret # } # } # } # // end inner for-loop # } # // end outter for-loop tracks.flat <- unlist(DT.nms[date != Sys.Date() & substr(ret_json, 1, 27) != "\n= 1001 # DTs.list[rows < nrowThresh] <- NULL cat("These would other wise be dropped") print(rbindlist(lapply(which(rows < 10 & !sapply(DTs.list, is.null)), function(i) cbind(DTs.list[[i]] [1, list(country, date)], rows=rows[[i]] ) )) [, list(dates_per_country = .N), by=country][order(dates_per_country, decreasing=TRUE)]) ## SEE rows[rows < 50] rows[rows == 1] stopifnot(all(sapply(DTs.list, ncol) == ncol(DTs.list[[1]]))) DT.toptracks <- rbindlist(DTs.list) kCols <- c("country", "date") kCols_streams <- c("country", "date", "num_streams") DT.toptracks[, date := as.Date(date)] setkeyIfNot(DT.toptracks, kCols_streams) setcolorderpt(DT.toptracks, c(kCols_streams, "artist_name", "album_name", "track_name")) DT.toptracks # DT.toptracks[DT.toptracks[, {browser(expr=any(is.na(.I[1:10]))); .I[1:10]}, keyby=kCols]$V1] DT.Agg_lists <- list() for (N in c(10, 25, 40, 50, 200)) { DT.Agg_lists[[as.character(N)]] <- setnames(DT.toptracks[, if (.N >= N) sum(num_streams[1:N]) else NA_real_, keyby=kCols], "V1", sprintf("top%02i", N)) } DT.Agg <- Reduce(merge, DT.Agg_lists) DT.Agg <- melt(DT.Agg, id.vars=kCols, variable.name="topx", value.name="total_streams") DT.Agg[, fromdate := date - 6] DT.Agg[, todate := date] DT.Agg[, avg_daily_streams := total_streams / 7] DT.daily <- DT.Agg[, list(day=seq.Date(from=fromdate, to=todate, by="day"), avg_daily_streams), keyby=c(kCols, "topx")] DT.daily[, date := NULL] DT.daily[, month := lubridate::floor_date(day, "month")] DT.monthly <- DT.daily[, list(avg_monthly_streams = round(sum(avg_daily_streams))), keyby=c("country", "month", "topx")] ## TODO: " &&& " ## Pull in total streams from spotify numbers ## Calculate as a percentage of Total Spotify streams loadFromJesus("DT.revshare", srcDir=srcOther("Spotify_Accounting_ETL"), over=TRUE) loadFromJesus("DT.data.Total_usage", srcDir=srcOther("Spotify_Accounting_ETL"), over=TRUE) DT.data.Total_usage[, month := as.Date(month)] setkeyIfNot(DT.data.Total_usage, "country_code", "month") DT.revshare[, month := as.Date(month)] DT.reported_totals <- DT.revshare[, list(spotify_streams=sum(as.numeric(total_tracks)), orchard_streams=sum(as.numeric(rightholders_tracks))), keyby=list(country_code, month)] DT.reported_totals <- rbind(DT.reported_totals, DT.reported_totals[, list(country_code = "global", spotify_streams = sum(spotify_streams), orchard_streams=sum(orchard_streams)), keyby="month"], use.names=TRUE) setkeyIfNot(DT.reported_totals, country_code, month, verbose=FALSE) DT.monthly[DT.reported_totals , `:=`( "Orchard_Total_Streams" = orchard_streams , "Spotify_Total_Streams" = spotify_streams )] DT.monthly[, monthly_as_perc_of_orchard := avg_monthly_streams / Orchard_Total_Streams] DT.monthly[, monthly_as_perc_of_spotify := avg_monthly_streams / Spotify_Total_Streams] # DT.monthly[DT.data.Total_usage # , `:=`( # "Orchard_Total_Streams" = orchard_streams_above_30_sec # , "Spotify_Total_Streams" = all_streams_above_30_sec # )] ## --- Plotting ----- ### --- weekly ----- ### DT.plot_weekly <- melt(Reduce(merge, DT.Agg_lists)[.("global")], id.vars=kCols, variable.name="topx", value.name="total_streams") P.global_weekly_absolute <- ggplot(DT.plot_weekly[topx != "top200"], aes(x=date, color=topx, y=total_streams)) + geom_line() + geom_point() + millions.y() + ggtitle("Spotify\nTotal number of streams attributed to Top 'x' tracks each week globally") + geom_vline(xintercept=as.numeric(as.Date("2014-10-27")), color="blue", linetype=2, size=1.2, alpha=.5) + xlab("Date\nBlue line is release date of Taylor Swifts' \"1989\"") ggsave.out(P.global_weekly_absolute) ### --- weekly ----- ### P.global_monthly_absolute <- ggplot(DT.monthly[.("global")], aes(x=month, color=topx, y=avg_monthly_streams)) + geom_line() + geom_point() + millions.y() + ggtitle("Spotify\nTotal number of streams attributed to Top 'x' tracks each month globally") ggsave.out(P.global_monthly_absolute) ggplot(DT.monthly[.("global")], aes(x=month, color=topx, y=avg_monthly_streams)) + geom_line() + geom_point() + millions.y() ggplot(DT.monthly[.("global")] [!is.na(monthly_as_perc_of_spotify)] [topx == "top50"], aes(x=month)) + geom_line(aes(y=monthly_as_perc_of_orchard), color="orange") + geom_line(aes(y=monthly_as_perc_of_spotify), color="green")