library(XLConnect)
library(bit)
library(bit64)
library(data.table)

.us()
setScience(proj="Spotify Data", subl=FALSE)


## File where to save results
f.out <- as.path(outDir, "Spotify_New_Data_Fields.xlsx")
f.out.data <- as.path(outDir, "Spotify_New_Data.xlsx")


## Folder containing the input TSV files
folder.in <- as.path(dataDir, "New Spotify Data")

## Load them in
files <- dir(folder.in, full=TRUE)
setattr(files, "names", basename(files))
#  raw <- lapply(files, readLines)

raw   <- vector(mode="list", length=length(files))
rinfo <- vector(mode="list", length=length(files))

for (i in seq(files))  {
	rr <- readLines(files[[i]])

	rr <- gsub("\\#", "Numb", rr)
	rr <- gsub("\\%", "Perc", rr)

	## Grab the starting points for each section
	dps <- grep("^Data point", rr)

	##  The ending point is 1-less from each starting point, plus the last line
	ends <- c(dps[-1] -1, length(rr))

	## The few lines before the first start point is the header info
	info.inds <- seq(1, dps[[1]] - 1)
	# rinfo[[i]] <- rr[info.inds]

	info <- as.data.table(read.csv(text=rr[info.inds], sep="\t", header=TRUE, stringsAsFactors=FALSE))
	start.date <- info[, as.Date(Start.date)]
	end.date   <- info[, as.Date(End.date)]

	raw[[i]] <- 
		mapply(function(s, e) {
			dat <- as.data.table(read.table(text=rr[s:e], sep="\t", header=TRUE, stringsAsFactors=FALSE))
			setnames(dat, strsplit(rr[[s]], "\t")[[1]])
			dat[, c("Start Date", "End Date") := list(start.date, end.date)]
		}, dps, ends)


	setattr(raw[[i]], "names", sapply(raw[[i]], function(D) D[["Data point"]][[1]]) )
}


rinfo2 <- lapply(rinfo, function(x) as.data.table(read.csv(text=x, sep="\t", header=TRUE)))