# Data Growth.r

source("~rsaporta/git/misc/rscripts/utils/notify/notify.r")

if (!exists("utilSource"))
	source("~rsaporta/git/misc/rscripts/utilsRS.r")

lib(ggplot2, 	  quietly=TRUE)
lib(scales, 	  quietly=TRUE)
lib(forecast, 	quietly=TRUE)
# lib(lubridate, 	quietly=TRUE)

utilSource()

f.out <- "~rsaporta/git/orch/src/!Misc/DataGrowth.png"

if (!exists(DAT)) { ## To avoid re-running the query, just in case
	Q <- "SELECT count(*) AS Rows, EXTRACT('year' FROM download_activity_date) AS Year, EXTRACT('month' FROM download_activity_date) AS Month from fact_analytics GROUP BY Year, Month ORDER BY Year, Month"
	analytics()
	DAT <- runQry(Q)
}

## Convert year & month to a single date, and drop the two superfluous columns
DAT[, date := as.Date(sprintf("%4s-%02s-01", year, month))]
DAT[, c("year", "month") := NULL]

## Set the key to the new date col
setkey(DAT, date)

## Drop "this month" since it is incomplete and will skew results
DAT <- DAT[!.(as.Date(format(Sys.Date(), "%Y-%m-01")))]

## Sum up the total rows to date
DAT[, totalRows := cumsum(rows)]


## ----------- FORECASTING ---------------- ##
## Number of Months Back to use for Forecasting
MthsBack <- 15
## Number of Months forward in time to forecast
MthsOut  <- 25

## Grab the last 'MthsBack' data points
TotRows.current <- DAT[, tail(totalRows, MthsBack)]

## Forecast `MthsOut` many points
TotRows.forecasted <- forecast(TotRows.current, MthsOut)


## find the last date, then creat a sequence from there
lastDate <- DAT[, date[which.max(date)]]
dts <- seq.Date(from=lastDate, length.out=MthsOut+1, by="month")[-1L]  ## the "+1" then "[-1L]" is because we want to start from the month AFTER the last month

## create the forecast into a data.frame and clean the column names
fore <- cbind(as.data.frame(TotRows.forecasted), x=dts)
setnames(fore, c("totalRows", "lo80", "hi80", "lo95", "hi95", "date"))

## add in the lastpoint of the actual data, for a continous line
lastPt <- tail(DAT, 1)[, data.table(rbind(rep(totalRows, length(fore)-1)), date) ] 
setnames(lastPt, names(fore))
fore   <- rbind(fore, lastPt, fill=TRUE)
setkeyv(fore, key(DAT))


## Additional Info to plot onto the plot
	thisMonth <- lubridate::floor_date(Sys.Date() - 30 , "month")
	Info <- unique(DD[.(lubridate::`%m+%`(thisMonth, lubridate::years(0:2))), list(date, totalRows)])
	Info[, label := sprintf("%s our\ndata %s\n%.1f Billion Rows\n", c("Today", "Next Year", "In Two Years"), c("exceeds", "is expected to exceed", "is expected to exceed"), totalRows/1e9)]
	Info[, xstart := date - 1015 - (900/.I^2)]
	Info[, xend   := date - 42 - (10/.I)]


## Main DATA, with forecast and actual
	DD <- rbind(cbind(DAT, actual=TRUE), cbind(fore, actual=FALSE), fill=TRUE)
	setkeyv(DD, key(DAT))


	P <- 
	{

		max.y.actual <- max(DD[, sapply(DD, is.numeric), with=FALSE], na.rm=TRUE)
		max.y.limit  <- max(DD$totalRows) * 2.2
		DD[hi80 > max.y.limit,  hi80 := max.y.limit]
		DD[hi95 > max.y.limit,  hi95 := max.y.limit]

		ggplot(data=DD, aes(x=date, y=totalRows)) +

			geom_ribbon(aes(ymin=lo95, ymax=hi95), alpha=1, fill="white") + 
			geom_ribbon(aes(ymin=lo80, ymax=hi80), alpha=.18, fill="cyan") + 
			geom_line(aes(y=lo95), alpha=.5, color="#E5F400", size=.7, linetype=6) + 
			geom_line(aes(y=hi95), alpha=.5, color="#E5F400", size=.7, linetype=6) + 

		  geom_line(size=1.15, aes(linetype=!actual)) + 


		  labs(x="Year", y="Billion Rows of Data", title="Data Growth in 'Analytics'\nwith 80% and 95% Prediction Intervals") + 
		  # billions.y(limits=ylims) + 
		  billions.y() + 
		  relativetext(x=1, y=1.2) + 
		  nolegend() + 

			## Additional Info
			geom_segment(data=Info, color="red", aes(x=xstart, xend=xend, y=totalRows, yend=totalRows)) + 
			geom_text(data=Info, aes(label=label, x=xstart-430, y=totalRows+5e9), size=3) + 

		NULL
	}		 


dir.create(dirname(f.out), showWarnings=FALSE)
ggsave(file=f.out, plot=P)

try( if (.Pfm == "Darwin") .o(f.out),  silent=TRUE)

cat("\nfile is\n     ", f.out, "\n\n\n")

invisible(f.out)