library(ProjectTemplate)
library(data.table)
library(rjson)


rm(list=setdiff(ls(), "utilSource"))

utilSource()
setProject("~/git/orch/", project="Intro", subl=FALSE)
.eWorking <- environment()

setwd("streamdl")

SpotifyTrack  <- read.table("data/SpotifyTrack.txt",  sep="\t", quote = "\"", comment.char="", header=TRUE)
SpotifyStream <- read.table("data/SpotifyStream.txt", sep="\t", quote = "\"", comment.char="", header=TRUE)
SpotifyUser   <- read.table("data/SpotifyUser.txt",   sep="\t", quote = "\"", comment.char="", header=TRUE)
Deezer        <- read.table("data/Deezer.txt",   sep="\t", quote = "\"", comment.char="", header=TRUE)
iTunes        <- read.table("data/iTunes.tsv",   sep="\t", quote = "\"", comment.char="", header=TRUE)


load.project()


## convert dataframes to data.tables
#  , but only if there are data.frame objects
if(!all(is.na(lsos(type="data.frame")))) {
  DataSets <- lsos(type="data.frame")[, sort(Name)]

  ## We will make all datasets lowercase, but only if this does not produce any duplicates. 
  ##   If duplicates would be produced, instead we use the identity function to leave them as is
  lowerFunc <-  if(!anyDuplicated(tolower(DataSets))) tolower else identity  

	invisible(sapply(DataSets, function(m) 
		{
      assign(lowerFunc(m), as.data.table(get(m)), envir=.eWorking)
      if(m!=lowerFunc(m)) 
          rm(list=m, envir=.eWorking)
    }
  )) 
}

## Clean up Spotify


parseJSONtoColumns <- function(DT, jsonCol, expectedCols=NULL, removeOriginal=TRUE) {
  if(!is.character(jsonCol))
    stop("`jsonCol` should be a character.")
  if (!(jsonCol %in% names(DT))) {
      warning(" `", jsonCol , "` is not a name of ", substitute(DT), ". Could it already have been processed?")
      return(DT)
  }

  # pare using `simpleFromJSON`
  jsonparsed <- DT[, simpleFromJSON(as.character( get(jsonCol) ), expectedCols=expectedCols)]
  # add the parsed as columns
  DT[, c(names(jsonparsed)) := jsonparsed]
  # remove original column
  if (removeOriginal)
    DT[, c(jsonCol) := NULL]
}

expectedCols.spotStream <- c("mobile", "timestamp")
expectedCols.spotUser   <- c("country", "product", "birth_year", "zipcode", "gender")
expectedCols.spotTrack  <- c("isrc", "album_name", "track_name", "album_code", "album_artist")

parseJSONtoColumns(spotifystream, "jsonString")
parseJSONtoColumns(spotifyuser,   "jsonString", expectedCols=expectedCols.spotUser)
parseJSONtoColumns(spotifytrack,  "jsonString", expectedCols=expectedCols.spotTrack)


itunes
spotifytrack
spotifyuser
spotifystream