## screen -xRR TI

setScience("TelecomItalia", create=FALSE, load=FALSE, subl=FALSE)
setGitBranchToSystem(); .g()

{
  sample_upcs_that_should_not_be_matched <- c(1000000432534, 1000000438611, 10181023705) %>% as.idcol

  files.in <- extractFilesFromFolder(ingestDir, ext="tsv", full=TRUE)
  stopifnot(file.exists(files.in))

  ll_DTs <- lapply(files.in, function(f) setIDCols(fread(f, integer64="numeric")))

  ## Flatten, and clean up (1) names (2) date types
  DT.telecomitalia <- merge_list_DTs(ll_DTs, by="upc", all.x=TRUE, all.y=FALSE)
  ## Clean names
  setnames(DT.telecomitalia, "max(eqd.delivery_ended)", "date_delivered_vector_2.5")
  setnames(DT.telecomitalia, "max(date_delivered)", "date_delivered_vector_2_or_HDD")
  ## Clean Date types
  DT.telecomitalia[, date_delivered_vector_2.5      := as.Date(date_delivered_vector_2.5)]
  DT.telecomitalia[, date_delivered_vector_2_or_HDD := as.Date(date_delivered_vector_2_or_HDD)]

  ## Identify which UPCs have been sent via either delivery method
  DT.telecomitalia[(!is.na(date_delivered_vector_2.5) | !is.na(date_delivered_vector_2_or_HDD))
              , date_delivered_most_recently := maxn(date_delivered_vector_2.5, date_delivered_vector_2_or_HDD), by=upc]

  if (any(DT.telecomitalia[, sample_upcs_that_should_not_be_matched %in% upc]))
    stop ("Test UPC cases are (incorrectly) in the results table")

  ## Organize
  setkeyIfNot(DT.telecomitalia, c("upc", "date_delivered_most_recently"), organize=TRUE, verbose=FALSE)

  ## Extract to two separate tables
  DT.not_delivered_to_TI <- DT.telecomitalia[is.na(date_delivered_most_recently)]
  DT.delivered_to_telecomitalia <- DT.telecomitalia[!is.na(date_delivered_most_recently)]

  f.out <- {
    exportXLS.usingXLConnect(f.out=out.p("TelecomItalia_UPCs_with_track_version", ext="xlsx"), 
                              DTs=list(delivered=DT.delivered_to_telecomitalia, "not delivered" = DT.not_delivered_to_TI)
                            )
  }

  ## Don't send the email if sourcing this file by mistake
  if (FALSE)
    quickEmail(getTo("manning"), cc=getTo("data"), files=f.out, info="Updated TelecomItalia (corrected 2015-10-08)", body="Chris\n\nAttached is the Telecom Italia upc list, with the previous error fixed. This should be complete and accurate, but an extra check should not hurt.\n\nCheers,\nRick")
}