# 04 Aggregate DT.Spotify.Counts.by.user.product.date.r


# SAMPLE_ONLY <- TRUE
SAMPLE_ONLY <- FALSE

## POSSIBLY TAKE A SAMPLE
{
  if (SAMPLE_ONLY) {
    n <- 4e5
    DT.mini <- DT.Spotify.Counts.by.user.product.date[{set.seed(1); sampleInChunks(DT.Spotify.Counts.by.user.product.date, n=n)}]
  } else 
     DT.mini <- DT.Spotify.Counts.by.user.product.date
}


{
  keyColsProd <- c("year", "month", "product")
  setkeyIfNot(DT.mini, keyColsProd)
  setcolorderpt(DT.mini, keyColsProd)

  DT.Spotify.Counts.agg <- 
      merge(  DT.mini[, list(Total_Streams=sum(total_streams_per), Avg_Streams=mean(total_streams_per), Median_Streams=median(total_streams_per)), keyby=keyColsProd]
            , DT.mini[, list(Unique_Users = .N), keyby=keyColsProd]
           )
  setattr(DT.Spotify.Counts.agg, "isSample", SAMPLE_ONLY)

  DT.Spotify.Monthly.Users <- 
      DT.mini[, list(Total_Monthly_Users = .N), keyby=c(keyColsProd[1:2]) ]

  DT.Spotify.Counts.agg[DT.Spotify.Monthly.Users, Total_Monthly_Users := Total_Monthly_Users]
  DT.Spotify.Counts.agg[, Total_Monthly_Streams := sum(Total_Streams), by=c(keyColsProd[1:2]) ]

  DT.Spotify.Counts.agg[, Percent_of_Monthly_Streams := Total_Streams / Total_Monthly_Streams ]
  DT.Spotify.Counts.agg[, Percent_of_Monthly_Users := Unique_Users / Total_Monthly_Users ]

  DT.Spotify.Counts.agg[, Streams_per_User := Total_Streams / Unique_Users]


  ## Clean up product column
  cleanSpotify_ (DT.Spotify.Counts.agg)

  ## Key
  setkeyIfNot(DT.Spotify.Counts.agg, keyColsProd)

  try(notify("Agg Done"))
  jesusForData(DT.Spotify.Counts.agg, info="Aggd out cid, by product year month")
  notify("Jesus Done")
}

## NOT SURE WHAT THIS WAS ABOUT? 
DT.Spotify.Counts.agg[, list(users_all_prods = sum(Unique_Users)), keyby=list(year, month)]
DT.Spotify.Counts.agg[, Unique_Users / sum(Unique_Users), by=list(year, month)]


{
  ## Find customers with more than one product tier
  DT.UserProduct <- DT.mini[, list(MonthsUsed=.N), keyby=list(customerid, product)]
  Customers.withMultiProduct <- DT.UserProduct[, .N, by=customerid][N>1, customerid]

  setkeyIfNot(DT.mini, "customerid")

  ## Note to self, this method is faster than alternative. See Benchmark file
  DT.Upgraders <- DT.mini[Customers.withMultiProduct]
  cleanSpotify_(DT.Upgraders)
  DT.Upgraders <- DT.Upgraders[, list( year[-1L], month[-1L]
                           , ProductFrom = product[- (.N) ]
                           , ProductTo = product[-1L]
                           , ProductDiff=diff(as.integer(product))
                          )
                    , keyby=customerid]
  colsOrdered <- c("customerid", "year", "month", "ProductFrom", "ProductTo", "ProductDiff")
  setcolorderpt(DT.old, colsOrdered)

  jesusForData(DT.Upgraders)
  notify("DONE")
}