if (FALSE) 
  source("~rsaporta/git/orch/src/misc/Gina Catalog work.R")

setScience("misc")
setGitBranchToSystem(); .g(); 
.us()
lib(ggplot)

f.in_small <- ingest.p("ginas_catalog_data", "ginas_catalog_data_20160626_sample", ext="csv")
f.in_large <- ingest.p("ginas_catalog_data", "ginas_catalog_data_20160626",        ext="csv")
stopifnot(file.exists(c(f.in_small, f.in_large)))

sampling <- TRUE
sampling <- FALSE

## CHOSE WHICH FILE TO USE
f.in <- f.in_small
f.in <- f.in_large


f_sufx <- function(f.name) {
## For the plot files;  
##  the file name should indicate whether we are using a sampling of data or the whole data set
  using_sample <- sampling || (f.in == f.in_small)
  paste0(f.name, ifelse(using_sample, "__SAMPLEOFDATA", "_FULLDATA"))
}

cut_years <- function(Y, na.name=NA) {
  years_back <- c(-5, 0, 2, 5, 15, 30, 45, 100, Inf)
  rel_years_cuts <- sort(year(today()) - years_back) + 1

  Hmisc::cut2(Y, cuts=rel_years_cuts, oneval=FALSE, levels.mean=FALSE) %>% {
    if (is.character(na.name)) {
      levs <- levels(.) %>% c(na.name)
      removeNA(., replace=na.name, modify_levels_when_x_is_factor=TRUE, NA_first_or_last_level="first", showWarnings=FALSE)
    } else
      .
  }
}

# DT2 <- data.table(rel_yr=Y, yrs=.)
#  [as.num.as.char(rel_yr) < 1970, list(rel_yr, as.num.as.char(rel_yr), yrs)]
# DT2[, Z := removeNA(., replace=na.name, modify_levels_when_x_is_factor=TRUE, NA_first_or_last_level="first")]
# DT2 [as.num.as.char(rel_yr) < 1970, ]


## Read in the file
minDate <- as.Date("2015-04-01")
maxDate <- as.Date("2016-03-01")
upcs_of_interest <- c(889176574870, 889176505898, 889176620997, 889176621000)
DT <- fread(f.in) %>% setnamestolower

## SAMPLING
if (isTRUE(sampling)) {
  indexs <- {set.seed(1); sample(nrow(DT), 200000)}  %>% c(., which(DT$upc %in% upcs_of_interest)) %>% sunique
  message("Taking only a sampling of ", formnumb(length(indexs), round=FALSE), " rows of data")
  DT <- DT[indexs]
}

## Clean up
DT[, upc := as.character(upc)]
DT[, stream_download := factor(stream_download)]

## DROP ODDBALL YEARS
DT[rel_yr > 2016, rel_yr := NA]
DT[rel_yr < 1917, rel_yr := NA]

# ## Coonvert bin to factor
# bin_levs <- DT[, shiftLeft(unique(bin[rev(order(rel_yr))]), roll=TRUE)]
# DT[, bin := factor(bin, levels=bin_levs)]

## AGGREGATE GROSS USD
DT.aggd <- DT[activity_month >= minDate & activity_month <= maxDate, list(gross_usd_fy16=sum(gross_usd)), keyby=list(labelid=label_id, upc, rel_yr, stream_download)
    ] [, list(streaming_gross_usd_fy16 = sum(gross_usd_fy16[stream_download == "Stream"]), total_gross_usd_fy16 = sum(gross_usd_fy16)), keyby=list(labelid, upc, rel_yr)]

DT.aggd[, perc_streaming := streaming_gross_usd_fy16 / total_gross_usd_fy16]


## Cut the years into yearly bins
DT.aggd[, yrs := cut_years(rel_yr, na.name="BAD METADATA")]

## Convert to factor
DT.aggd[, rel_yr := factor(rel_yr)]

## PLOT IT
title.boxplot_log_rev <- subtext("Revenue (FY '16) per Release", "each dot is one release")

P.boxplot_log_rev <- 
  ggBoxplotWithDots(DT=DT.aggd, x="yrs", y="total_gross_usd_fy16", title=title.boxplot_log_rev, yscale="log", y_is_dollars=TRUE, xlab="Release Year", ylab="Gross USD", ylab_append="(shown as log-scale)", relativetext.size=0.8
    , dots_alpha=0.08, dots_color="navy blue", dots_size=0.5, jwidth=0.15
    )

## Streaming percentage relative to total revenue
## looks like random noise
if (FALSE)
  qplot(data=DT.aggd[perc_streaming %ni% c(0, 1) & total_gross_usd_fy16 > 100], x=perc_streaming, y=total_gross_usd_fy16) + geom_point() + log.y()


qplot(data=DT.aggd, x=perc_streaming, y=total_gross_usd_fy16) + geom_point()
qplot(data=DT.aggd, x=perc_streaming) + geom_density()

## The PDF for some reason is giant
# f.rev_by_rel_yr.boxplot <- printToPDF(list(P.boxplot_log_rev), f.name="Revenue by Release Year", ncol=1)
f.rev_by_rel_yr.boxplot <- ggsave.out(plot=P.boxplot_log_rev, nm=f_sufx("Revenue by Release Year"), ext="png", footnote=orchardFootNote())

# --------------------------------------------------------------------------------------------------------------------------------------------

DT.aggd_by_label_and_yrs <- {
  DT.aggd[, list(
        total_gross_usd_for_label  = sum(total_gross_usd_fy16),
        number_of_releases         = .N,  ## the number of rows in the subset of data
        mean_gross_usd_per_release = sum(total_gross_usd_fy16) / .N  ## total total_gross_usd_fy16 divided by the number of releases in the subset
      )
    ## Subsset the data by label and yrs
    , keyby = list(
          labelid,
          yrs
    )
  ]
}

DT.aggd_by_label <- {
  DT.aggd[, list(
        total_gross_usd_for_label  = sum(total_gross_usd_fy16),
        number_of_releases         = .N,  ## the number of rows in the subset of data
        mean_gross_usd_per_release = sum(total_gross_usd_fy16) / .N  ## total total_gross_usd_fy16 divided by the number of releases in the subset
      )
    ## Subsset the data by label
    , keyby = list(
          labelid
    )
  ]
}

## ------------------------------------------------ ##
## Double checking
## ------------------------------------------------ ##
DT.aggd_by_label[,         list(total_rev_for_FY16=asCurr(sum(mean_gross_usd_per_release * number_of_releases)), `sum_of_means (meaningless)`=asCurr(sum(mean_gross_usd_per_release)), total_number_of_releases=formnumb(sum(number_of_releases), round=FALSE))]
DT.aggd_by_label_and_yrs[, list(total_rev_for_FY16=asCurr(sum(mean_gross_usd_per_release * number_of_releases)), `sum_of_means (meaningless)`=asCurr(sum(mean_gross_usd_per_release)), total_number_of_releases=formnumb(sum(number_of_releases), round=FALSE))]

DT.aggd_by_label[(number_of_releases > 45), list(sum(mean_gross_usd_per_release), sum(number_of_releases))]
## ------------------------------------------------ ##
## Double checking
## ------------------------------------------------ ##



DT.aggd_by_label_and_yrs[, bucketed_number_of_releases := Hmisc::cut2(number_of_releases, cuts=c(0,1, 2, 5, 7, 10, 20, 40, Inf))]

title.avg_usd_per_rel <- subtext("Average Revenue per Release", "each dot is one label")
## Plot the avg revenue per release, by label, bucketing the labels by the number of rleases they have
P.boxplot_of_avg_usd_per_release_by_label_by_the_labels_number_of_releases <- {
    ggBoxplotWithDots(DT.aggd_by_label_and_yrs, x="bucketed_number_of_releases", y="mean_gross_usd_per_release", yscale="log", y_is_dollars=TRUE, dots_alpha=0.08, dots_color="navy blue", dots_size=0.5, jwidth=0.15, title=NULL, xlab="Number of Releases associated with the Label", ylab="the average (mean) USD per release, for each label", ylab_append="shown on a log-scale"
      , outliers=FALSE, dots_on=TRUE)
}

## Plot the avg revenue per release, by label, bucketing the labels by the number of rleases they have
P.boxplot_of_avg_usd_per_release_by_label_and_yrs_by_the_labels_number_of_releases <- {
    ggBoxplotWithDots(DT.aggd_by_label_and_yrs, x="bucketed_number_of_releases", y="mean_gross_usd_per_release", yscale="log", y_is_dollars=TRUE, dots_alpha=0.08, dots_color="navy blue", dots_size=0.5, jwidth=0.15, title=NULL, xlab="Number of Releases associated with the Label", ylab="the average (mean) USD per release, for each label", ylab_append="shown on a log-scale"
      , outliers=TRUE, dots_on=FALSE, color="yrs", legend="bottom")
}

f.avg_usd_per_release_by_label.boxplots <- printToPDF(list(P.boxplot_of_avg_usd_per_release_by_label_by_the_labels_number_of_releases, P.boxplot_of_avg_usd_per_release_by_label_and_yrs_by_the_labels_number_of_releases), f.name=f_sufx("avg_usd_per_release_by_label"), height.per.plot=6, width.per.plot=9, main=title.avg_usd_per_rel)


## AVERAGE OF AVERAGES

min_labs_per_grp <- 18

## (1) By label and Yrs
  yrs <- levels(DT.aggd_by_label_and_yrs$yrs) %>% factor(., levels=.)
  DT.avg_of_mean_gross_usd_per_release_by_label_yrs <- CJ(at_least_x_releases=seq.int(max(DT.aggd_by_label_and_yrs$number_of_releases)-1), yrs=yrs)

  matchKey(DT.avg_of_mean_gross_usd_per_release_by_label_yrs, DT.aggd_by_label_and_yrs, key="yrs", superset.ok=FALSE)

  ## CALCULATE MEDIAN OF MEAN GROSS
  s.t({
    DT.avg_of_mean_gross_usd_per_release_by_label_yrs[
      , c("median_of_mean_gross", "mean_of_mean_gross", "sd_of_mean_gross", "number_of_labels", "running_total_of_gross_usd_for_labels_in_group") := 
            DT.aggd_by_label_and_yrs[.(yr)] [number_of_releases >= at_least_x_releases, list(median(mean_gross_usd_per_release), mean(mean_gross_usd_per_release), sd(mean_gross_usd_per_release), .N, sum(total_gross_usd_for_label))]
      , by=list(yr=yrs, at_least_x_releases)
    ]
  }, msg="calculating median of mean gross")

  ## VIEW
  head(DT.avg_of_mean_gross_usd_per_release_by_label_yrs[yrs == "BAD METADATA"][order(at_least_x_releases)], n=15)
  DT.aggd_by_label_and_yrs[yrs == "BAD METADATA"][order(number_of_releases)]

  DT.avg_of_mean_gross_usd_per_release_by_label_yrs[, text := NULL]
  ## add a text column for graphing
  DT.avg_of_mean_gross_usd_per_release_by_label_yrs[, perc_diff_in_labels := diffNA(number_of_labels) / number_of_labels]
  DT.avg_of_mean_gross_usd_per_release_by_label_yrs[perc_diff_in_labels < -.06 & (at_least_x_releases %in% c(2, 10, 25) | at_least_x_releases > 40), text := sprintf("  %i Labels\n  %s USD", number_of_labels, asCurr(running_total_of_gross_usd_for_labels_in_group, decim=0))]

  ## PLOT THEM
  P.median_avg_by_label_yrs <- ggLinegraph(DT=DT.avg_of_mean_gross_usd_per_release_by_label_yrs[number_of_labels > min_labs_per_grp], x="at_least_x_releases", y="median_of_mean_gross", color="yrs", dotsize.scale=0.8, xscale=NULL, smooth=FALSE, yscale="dollar", legend="bottom", xlims=c(0, 500), title=NULL)
  P.mean_avg_by_label_yrs <- ggLinegraph(DT=DT.avg_of_mean_gross_usd_per_release_by_label_yrs[number_of_labels > min_labs_per_grp], x="at_least_x_releases", y="mean_of_mean_gross", color="yrs", dotsize.scale=0.8, xscale=NULL, smooth=FALSE, yscale="dollar", legend="bottom", xlims=c(0, 500), title=NULL)



## (2) By label
  DT.avg_of_mean_gross_usd_per_release_by_label <- CJ(at_least_x_releases=seq.int(max(DT.aggd_by_label_and_yrs$number_of_releases)-1))

  ## CALCULATE MEDIAN OF MEAN GROSS
  s.t({
    DT.avg_of_mean_gross_usd_per_release_by_label[
      , c("median_of_mean_gross", "mean_of_mean_gross", "sd_of_mean_gross", "number_of_labels", "running_total_of_gross_usd_for_labels_in_group") := 
            DT.aggd_by_label [number_of_releases >= at_least_x_releases, list(median(mean_gross_usd_per_release), mean(mean_gross_usd_per_release), sd(mean_gross_usd_per_release), .N, sum(total_gross_usd_for_label))]
      , by=list(at_least_x_releases)
    ]
  }, msg="calculating median of mean gross")

  ## add a text column for graphing
  DT.avg_of_mean_gross_usd_per_release_by_label[, perc_diff_in_labels := diffNA(number_of_labels) / number_of_labels]
  DT.avg_of_mean_gross_usd_per_release_by_label[perc_diff_in_labels < -.06 & (at_least_x_releases %in% c(2, 10, 25) | at_least_x_releases > 40), text := sprintf("  %i Labels\n  %s USD", number_of_labels, asCurr(running_total_of_gross_usd_for_labels_in_group, decim=0))]


  ## PLOT THEM
  P.median_avg_by_label <- ggLinegraph(DT=DT.avg_of_mean_gross_usd_per_release_by_label[number_of_labels > min_labs_per_grp], x="at_least_x_releases", y="median_of_mean_gross", dotsize.scale=0.8, xscale=NULL, smooth=FALSE, yscale="dollar", y_is_dollars=TRUE, legend="bottom", title="Median", xlab_append=" ", xlims=c(0, 500))
  P.mean_avg_by_label <- ggLinegraph(DT=DT.avg_of_mean_gross_usd_per_release_by_label[number_of_labels > min_labs_per_grp], x="at_least_x_releases", y="mean_of_mean_gross", dotsize.scale=0.8, xscale=NULL, smooth=FALSE, yscale="dollar", y_is_dollars=TRUE, legend="bottom", title="Mean", xlab_append=" ", xlims=c(0, 500))


## COLLECT ALL GRAPHS INTO ONE LIST, THEN PLOT IT
  ll.average_of_averages <- list(
         median_avg_by_label     = P.median_avg_by_label
       , mean_avg_by_label       = P.mean_avg_by_label
       , median_avg_by_label_yrs = P.median_avg_by_label_yrs
       , mean_avg_by_label_yrs   = P.mean_avg_by_label_yrs
  )

## Export to PDF
title.avg_of_avgs <- subtext("Average price per release, by catalog size", "each dot represents the group of labels having at least 'x' many releases in their catalog")


f.average_of_avg_usd_per_release <- 
  printToPDF(ll.average_of_averages, f.name=f_sufx("average_of_avg_usd_per_release"), ncol=2, height.per.plot=5, main=title.avg_of_avgs) # , sub="Median on the left || Mean on the right"

## ------------------------------------------------------------------------------------------

files.all <-   c(
          f.rev_by_rel_yr.boxplot
        , f.avg_usd_per_release_by_label.boxplots
        , f.average_of_avg_usd_per_release
)
 
if (.Pfm == "Darwin") {
  .o(files.all)
} else {
  catnn("\nALL DONE\n", sprintf("   bringme(rsync=FALSE, overwrite=TRUE,  '%s')", files.all), "\n")
}

   .o('~/git/orch/plots/misc/Revenue by Release Year.png')
   .o('~/git/orch/plots/misc/avg_usd_per_release_by_label.pdf')
   .o('~/git/orch/plots/misc/average_of_avg_usd_per_release.pdf')



   bringme(rsync=FALSE, overwrite=TRUE,  '~/git/orch/plots/misc/Revenue by Release Year_FULLDATA.png')
   bringme(rsync=FALSE, overwrite=TRUE,  '~/git/orch/plots/misc/avg_usd_per_release_by_label_FULLDATA.pdf')
   bringme(rsync=FALSE, overwrite=TRUE,  '~/git/orch/plots/misc/average_of_avg_usd_per_release_FULLDATA.pdf')
   .o('~/git/orch/plots/misc/Revenue by Release Year_FULLDATA.png')
   .o('~/git/orch/plots/misc/avg_usd_per_release_by_label_FULLDATA.pdf')
   .o('~/git/orch/plots/misc/average_of_avg_usd_per_release_FULLDATA.pdf')