# 02 fb cleanup.r

#  fb[, summary(Spend)]
#  qs <- c(1, 5)  %x% 10^seq(1, -3)
#  qs <- sort(unique(c(qs, 100-qs)))
#  fb[, quantile(Spend, prob=qs/100)]
#  plot(density(log(fb$Spend)))


## PARAMETERS
maxClicksPerImpr.allowed = 5
Min.ImpressionsSpeed.Caps = c(I=35, S=.25)  # Use 0 or -Inf to turn off

DropZeroImpres = TRUE
DropZeroSpend = FALSE

# General, across the board, Impressions have to be >=  this value
min.Impressions.GENERAL = 0

previewOutliersInfo = TRUE

SkipOutlierRemoval = TRUE

## ------ START DROP LOW IMPRESSIONS OR ODD VALUES  ------- # 

  ## drop anything where there is more than X clicks per person for each person.
  fb <- fb[!( (maxClicksPerImpr.allowed * Impressions) <= Clicks )]

  ## Drop anything with low impressions and low spend
  fb <- fb[!(Impressions < Min.ImpressionsSpeed.Caps[["I"]] & Spend < Min.ImpressionsSpeed.Caps[["S"]])]

  ## Drop 
  if (DropZeroImpres)
    fb <- fb[Impressions != 0]
  if (DropZeroSpend)
    message("Not dropping Spend just yet") # TO PREVENT ACCIDENTAL # :  fb <- fb[Spend != 0]

  ## Drop rows with an Impressions ammount less than the chosen value
  fb <- fb[Impressions >= min.Impressions.GENERAL]

## ------ END DROP LOW IMPRESSIONS OR ODD VALUES  ------- # 


 
 ## SKIPPING
if (!SkipOutlierRemoval) {
  
  ## --------- START OUTLIER SELECTION ---------  #
    # We will add some calc columns to identify outliers. Aftewrads, we will drop them. 
    #   we bank the current names of the columns to compare against later and know what to drop
    Cols.to.crop.outliers <- c("Impressions", "Spend", "Clicks", "CTR", "CPM", "CPC")
    identifierCols <- c("Campaign", "Ad", "Date", "Label")
    identifierCols.wID <- c(identifierCols, nameToID(identifierCols))
  
    #
    #       KEEP THE OUTLIERS.   THEY ARE IMPORTANT. 
    # 
  
  
    names.beforeCalc <- copy(names(fb))
    addCalcCols_(fb)
  
    ## These are the rows to drop 
    rowsToDrop <- mutualTopRows(fb, Cols.to.crop.outliers, perc=0.999)
    RT <- mutualTopRows(fb, Cols.to.crop.outliers, perc=0.95, returnDT=TRUE)
    RT[, SUMS := rowSums(RT, na.rm=TRUE)]
    RT[SUMS> 1]
  
    # high Spend, high clicks, low Impressions
    RT[ Spend  &  Clicks &  Impressions,  Outlier := "High Spend, High Clicks, High Impressions"]
    RT[ Spend  &  Clicks & !Impressions,  Outlier := "High Spend, High Clicks, Not High Impressions"]
    RT[ Spend  & !Clicks &  Impressions,  Outlier := "High Spend, Not High Clicks, High Impressions"]
    RT[ Spend  & !Clicks & !Impressions,  Outlier := "High Spend, Not High Clicks, Not High Impressions"]
  
    RT[!Spend  &  Clicks &  Impressions,  Outlier := "Not High Spend, High Clicks, High Impressions"]
    RT[!Spend  &  Clicks & !Impressions,  Outlier := "Not High Spend, High Clicks, Not High Impressions"]
    RT[!Spend  & !Clicks &  Impressions,  Outlier := "Not High Spend, Not High Clicks, High Impressions"]
    RT[!Spend  & !Clicks & !Impressions,  Outlier := "Not High Spend, Not High Clicks, Not High Impressions"]
  
    fb[, Outlier := RT$Outlier ]
  
    ## DONT DROP EM
    rowsToDrop <- which(RT[, list(Impressions, Clicks, Spend)] == 1)
    ## Tracking what we are removing
    howManyDropping.per1K <- round((length(rowsToDrop) / nrow(fb)) * 1000, 2)
    message("Dropping ", howManyDropping.per1K, " per 1,000 rows  (", length(rowsToDrop), " / ", format(nrow(fb), big.mark=","), " total) " )
  
    fb[, sum(Spend)]
    fb[Impressions < 100, sum(Spend)]
  
    fb[Impressions < 50, lapply(.SD, sum, na.rm=TRUE), .SDcols=Cols.to.crop.outliers]
  
    if (previewOutliersInfo)
      fb[rowsToDrop, setdiff(c(identifierCols, Cols.to.crop.outliers), c("Campaign", "Ad")), with=FALSE]
  
    fb <- fb[!rowsToDrop]
  
  
    addedColsToDrop <- setdiff( names(fb), c(names.beforeCalc) )
    fb[, c(addedColsToDrop) := NULL]
  ## --------- END OUTLIER SELECTION ---------  #
  
  
  }