# Spotify_Streams_Per_Top_Artist source ("~rsaporta/git/orch/src/Spotify_Streams_Per_Top_Artist/00 Setup.r") "-------------------------------------------------" # -------------------------------------------------------------------------------- { if (exists("DT.nms")) { DT.nms.bak <- copy(DT.nms) assign(timeStamp("DT.nms", seconds=TRUE), value=DT.nms) } DT.nms <- CJ(country=countries.avail, date=dates.avail) DT.nms[, nms := sprintf("%s.%s", country, date)] DT.nms[, url := sprintf("http://charts.spotify.com/api/tracks/most_streamed/%s/weekly/%s", country, date)] setkeyIfNot(DT.nms, "nms", organize=TRUE, warnForColNameInEnv=FALSE, verbose=FALSE) if (exists("DT.nms.bak")) { setkeyIfNot(DT.nms.bak, "nms", organize=TRUE, warnForColNameInEnv=FALSE, verbose=FALSE) DT.nms[DT.nms.bak, ret_json := i.ret_json] rm(DT.nms.bak) } } ## Organize the names, because I believe there are API limits ## first the global, then anything with 2014 nms.ordered <- { c( DT.nms[country == "global" & date >= "2014-02-01", nms], DT.nms[country %in% c("US", "SE") & date >= "2014-02-01", nms], DT.nms[country %ni% c("global", "US", "SE") & date >= "2014-02-01", nms], DT.nms[country %in% c("global", "US", "SE") & date < "2014-02-01", nms], DT.nms[country %ni% c("global", "US", "SE") & date < "2014-02-01", nms] ) } stopifnot(identical(sort(nms.ordered), sort(DT.nms$nms))) ## TODO: Create a DT of scrape_success. ## Where results can be either (already present, success, error) for (nm in nms.ordered) { ## Recovery after error # for (nm in nms.ordered[(which(nms.ordered == nm) + 1) : length(nms.ordered)]) { cat("Processing : ", nm, "\n") ## don't pull again if already pulled if (exists("DTs.list") && isTRUE(nrow( DTs.list[[nm]] ) >= nrowThresh)) { "do nothing" } else { ret <- try(getURL(DT.nms[.(nm)]$url)) if (isErr(ret)) { warning (sprintf("ERROR: '%s', '%s'", DT.nms[.(nm)]$country, DT.nms[.(nm)]$date)) } else { if (!grepl("^.\"tracks\"", substr(ret, 1, 100))) warning (sprintf("POSSIBLE ERROR: '%s', '%s'", DT.nms[.(nm)]$country, DT.nms[.(nm)]$date)) DT.nms[.(nm), ret_json := ret] } } if (nm %in% nms.ordered[unique(1 + seq_along(nms.ordered) %/% 100) * 100 ]) jesusForData(DT.nms, info="mid step save") } jesusForData(DT.nms, info="with raw data") f.lastSaved <- saveImageTo() ### ------------------- THIS SHOULD SPLIT INTO SECOND FILE ------------------ ### ## ## ## EVERYTHING THAT FOLLOWS ## ## EVERYTHING THAT FOLLOWS ## ## EVERYTHING THAT FOLLOWS ## ## ## ### ------------------- THIS SHOULD SPLIT INTO SECOND FILE ------------------ ### ### ------------------- THIS SHOULD SPLIT INTO SECOND FILE ------------------ ### ### ------------------- THIS SHOULD SPLIT INTO SECOND FILE ------------------ ### ### ------------------- THIS SHOULD SPLIT INTO SECOND FILE ------------------ ### { tracks.flat <- unlist(DT.nms[date != Sys.Date() & substr(ret_json, 1, 27) != "\n= 1001 # DTs.list[rows < nrowThresh] <- NULL cat("These would other wise be dropped") print(rbindlist(lapply(which(rows < 10 & !sapply(DTs.list, is.null)), function(i) cbind(DTs.list[[i]] [1, list(country, date)], rows=rows[[i]] ) )) [, list(dates_per_country = .N), by=country][order(dates_per_country, decreasing=TRUE)]) ## SEE rows[rows < 50] rows[rows == 1] stopifnot(all(sapply(DTs.list, ncol) == ncol(DTs.list[[1]]))) DT.toptracks <- rbindlist(DTs.list) setInfo(DT.toptracks, "rbindlist of DTs.list, which itself is a collection of data.tables of the JSON parse of the track info from the Spotify Web Scrape, by country and week") jesusForData(tracks.flat) jesusForData(DT.nms) jesusForData(DT.toptracks) jesusForData(DTs.list) # .g() # loadFromJesus("tracks.flat", over=TRUE) # loadFromJesus("DT.nms", over=TRUE) # loadFromJesus("DT.toptracks", over=TRUE) # loadFromJesus("DTs.list", over=TRUE)