## Accounting_vs_Analytics_2015 (Plotitng OA Anal vs OA Acc).r
fileSelfDescription('
This file plots accounting vs analytics, comparing paidunits at matching activity_month544

Primary Table :  DT.activty_totals_compare
                 merge and aggregated from 
                  *   DT.bi_anal_totalpaidunits
                  *   DT.bi_acc_totalpaidunits.activity_month
Purpose       :  Compares paidunits from bi.analytics to bi.accounting via activity_month544
')



## ------------------------------------------------------------------- ##

.us()
lib(lubridate, quietly=TRUE)
lib(data.table, quietly=TRUE)
setScience("Acc_vs_Anal_2015", load=TRUE)
subProj <- "Anal_vs_Accounting"

## Cols to pull from both tables
aggCols.acc   <- c(revenue="gross"        ,  paidunits="units")
aggCols.anal  <- c(revenue="royaltydollar",  paidunits="paidunits")
addlCols <- c("storeid", "transac_type_abbr", "label_sc_group", "country_code") ## c("transac_type_abbr", setdiff(kCols.splitgroup, "date"))
minDate <- as.Date("2013-01-01")
kCols.main <- c("date", addlCols)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

## COMPARE ANALYTICS TO ACTIVITY DATE
loadIfNotExists("DT.bi_anal_totalpaidunits")
loadIfNotExists("DT.bi_acc_totalpaidunits.activity_month")


desc(DT.bi_anal_totalpaidunits)
desc(DT.bi_acc_totalpaidunits.activity_month)

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


## Create DT.activity_total_compare
BackUpOrRestore("DT.activty_totals_compare", clear=TRUE)
DT.activty_totals_compare <- { 
    stores_not_in_anal <- c(27, 36)
    maxDate_for_comparing <- monthFloor(today() %m-% months(4))
    minDate_for_comparing <- as.Date("2013-11-01")

    merge(
       DT.bi_acc_totalpaidunits.activity_month[, list(paidunits = sumn(paidunits)), keyby=list(date=activity_month, storeid, transac_type_abbr)]
     , DT.bi_anal_totalpaidunits[, list(paidunits = sumn(paidunits)), keyby=list(date=activity_month544, storeid, transac_type_abbr)]
     , all=TRUE
     # , by = c("date", "storeid", "transac_type_abbr")
     , suffix=c(".acc", ".anal")
     , allow=FALSE
     )[storeid %ni% stores_not_in_anal
     ][date <= maxDate_for_comparing & date >= minDate_for_comparing
     ]
}
setInfo(DT.activty_totals_compare, appendInfoFromMany(DT.bi_acc_totalpaidunits.activity_month, DT.bi_anal_totalpaidunits, more="aggregating each table by date, store, trans") )


## Add in .OVERALL. s
DT.activty_totals_compare <- rbind(DT.activty_totals_compare
                                , DT.activty_totals_compare[!is.na(paidunits.acc) & !is.na(paidunits.anal), list(paidunits.acc = sumn(paidunits.acc), paidunits.anal=sumn(paidunits.anal), transac_type_abbr=".OVERALL."), keyby=list(date, storeid)] 
                                # , DT.activty_totals_compare[storeid == 1 & transac_type_abbr %ni% c("NR"), list(paidunits.acc = sumn(paidunits.acc), paidunits.anal=sumn(paidunits.anal), transac_type_abbr=".OVERALL except NR."), keyby=list(date, storeid)] 
                                )
setInfo(DT.activty_totals_compare, appendInfo(new.info="rbind of self adding rows for .OVERALL", DT=DT.activty_totals_compare) )

## Calculate Percentage
DT.activty_totals_compare[, anal_percOf_acc := as.perc(paidunits.anal / paidunits.acc)]

## Add store info
addStoreName.storeid_(DT.activty_totals_compare, factor=TRUE, overwrite=TRUE, showWarnings=FALSE)

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

## Reset-point for DT.activty_totals_compare - Taking a backup before modifying
BackUpOrRestore("DT.activty_totals_compare")
{
  cuts <- c(.005, .1, 2.5, 20) / 100
  deltaCut_(
      DT = DT.activty_totals_compare
    , colToCut = "anal_percOf_acc"
    , cuts = cuts
    , distanceFrom = 1
    , bottom = -Inf
    , top = Inf
    , newCol = "accuracy"
    , deltaSymbol = "Δ"
    , cropMax = TRUE
    , overwrite=TRUE
    )
}


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

## Determine which combinations will NOT get graphed
dates_required <- seq(maxDate_for_comparing, length.out=4, by="-1 month")
DT.activty_totals_compare[, .include_in_graph := all(dates_required %in% date), by=list(store_name, transac_type_abbr)]
DT.activty_totals_compare[(!.include_in_graph), .N, by=list(store_name, transac_type_abbr)]

## Grab the levels and make colors for graph
levs.accuracy <- levels(DT.activty_totals_compare$accuracy)
colors.accuracy <- rev(RColorBrewer::brewer.pal(length(levs.accuracy) + 1, name="RdYlGn"))[-2]

## For manual debugging
stores.using <- "iTunes"
stores.using <- "Spotify"

PLOTS.analytics_compare <- list()
main_stores <- c("iTunes", "Spotify")
ylims <- 1 + c(-1, 1) * max(cuts)
for(stores.using in c(main_stores, OtherStores=list(setdiff(DT.activty_totals_compare$store_name, main_stores)))) {
  stitle <- if(length(stores.using)==1) stores.using else "All Other Stores"
  title <- sprintf("Comparing OA Analytics to Accounting for %s", stitle)
  DT.plot <- DT.activty_totals_compare[(.include_in_graph)][store_name %in% stores.using][!is.na(anal_percOf_acc)]
  PLOTS.analytics_compare[[stitle]] <- 
    ggLinegraph(DT.plot, x="date", y="anal_percOf_acc"
          , facet_formula="store_name + transac_type_abbr ~."
          , hline=1, hline_alpha=.3
          , ylims=ylims
          , dot_color="accuracy"
          , ylab="OA Analytics as percentage of OA Accounting"
          , xlab=""
          , xlab_append="Values with Δ > 20% were cropped to fit on graph"
          , title=title
          , title_append="Values with Δ > 20% were cropped to fit on graph"
          , size=.9
          , dotsize=2) + 
    color_by_dict(values=levs.accuracy, colors=colors.accuracy, drop=FALSE) + 
    years.x(format="%b %Y") + 
    geom_point(data = DT.plot[.col_was_cropped & anal_percOf_acc < 1], shape=25, size=2.5) + 
    geom_point(data = DT.plot[.col_was_cropped & anal_percOf_acc > 1], shape=24, size=2.5) 
  if (stitle == "All Other Stores")
    PLOTS.analytics_compare[[stitle]] <- PLOTS.analytics_compare[[stitle]] + relativefacettext(x=.6)
}
printToPDF(PLOTS.analytics_compare, paginate=TRUE, pdfFunc="cairo_pdf", family="Helvetica-Narrow")

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~