DT.tracks_per_upc <- sfQry("
SELECT upc,
  count(*) as count_of_total_tracks,
  count(DISTINCT CD) as count_of_discs, 
  count(distinct ISRC) as count_of_isrc, 
  count(distinct trackid) as count_of_trackid, 
  count(distinct track_id) as count_of_track_id
FROM production.dim_Track
WHERE 
        NOT (cd = 0 AND track_id = 0)
    AND NOT (track_unique_id = 0)
    AND NOT (source RLIKE '.*problem.*')
GROUP BY 1
order by 1, 2
")
setIDCols(DT.tracks_per_upc, showWarnings=FALSE)
setBoolCols(DT.tracks_per_upc, showWarnings=FALSE)
setkeyIfNot(DT.tracks_per_upc, upc)



if (FALSE) {
  ## problematic upcs are those where there is ONLY a "(problem)" type source
  problematic_upcs <- DT.tracks_per_upc[DT.tracks_per_upc[(source_has_problems)]][, .N, by=upc][N != 2, upc]
  ## Drop the problematic upcs
  DT.tracks_per_upc <- DT.tracks_per_upc[!problematic_upcs]
  ## upcs that have SOME tracks with problems
  upcs_with_problem_in_source <- DT.tracks_per_upc[DT.tracks_per_upc[(source_has_problems)]][, .N, by=upc][N == 2, upc]
}


if (FALSE) {
  if (FALSE)
  DTs <- sfQry("SELECT * FROM production.dim_track WHERE NOT (cd = 0 AND track_id = 0) AND NOT (track_unique_id = 0) AND NOT (source RLIKE '.*problem.*') AND  UPC = 99925344528 ")
  
  if (FALSE)
  DTs <- sfQry(sprintf("SELECT * FROM production.dim_Track where upc in (%s)", pasteC(problematic_upcs, C=", "))) %>% setIDCols

  setIDCols(DTs)
  DTs[, source := gsub("\\s\\s+", " ", source)]
  for (col in copy(names(DTs))) {
    if (all(is.na(DTs[[col]])))
      DTs[, (col) := NULL]
  }
  setkeyIfNot(DTs, upc, isrc, track_id, organize=TRUE)
  DTs[!grepl("problem", source)]
  DTs
  DT.tracks_per_upc[upc == "99925344528"]

  for (i in seq(20))  {
    print(DTs[.(upcs_with_problem_in_source[[i]])][order(source)])
    print(DT.tracks_per_upc[.(upcs_with_problem_in_source[[i]])])
    catn("-----------------------------------------------------------")
  }
}