if (FALSE)
  source("~/git/orch/src/ContentAnalysis/Hidden Content 03 by String Parts.r")

unlst <- function(x) {
## Wrapper function for unlist
  unlist(x, recursive=FALSE, use.names=FALSE)
}

clean_wrapper_for_wordsplit  <- function(x) {
  if (!is.atomic(x))
    stop ("x must be atomic")

  pat.karaoke <- "kar\\w?ok\\w?"


  x %>% 
    # gsub("\\)\\s*", ") ", .) %>% gsub("\\s*\\(", " (", .) %>% gsub("(;|:)\\s*", " ", .) %>% 
    gsub("(\\w)'(\\w{1,2}\\b)", "\\1\\2", .) %>%  
    gsub(pat.karaoke, "karaoke", x=., ignore.case=TRUE) %>% 
    clean_names_to_simple_alpha(punctuation=TRUE, force_space_around_all_punctuation=TRUE, words_to_rem=NULL, and_clean=FALSE, whitespace=FALSE, duplicate_whitespace=TRUE, inside_parens=FALSE, parens=TRUE, comma_force_trailing_space=TRUE, tolower=TRUE)
}

clean_wrapper <- function(x) {
  x %>% gsub("\\)\\s*", ") ", .) %>% gsub("\\s*\\(", " (", .) %>% 
  clean_names_to_simple_alpha(and_clean=TRUE, whitespace=FALSE, inside_parens=FALSE, comma_force_trailing_space=TRUE, tolower=TRUE)
}


string_to_n_length_parts <- function(x, n_length, ret_if_empty="", verbose=FALSE) {
  if (length(x) == 0)
    return(ret_if_empty)
  if (length(x) > 1)
    return(lapply(x, string_to_n_length_parts, n_length=n_length, ret_if_empty=ret_if_empty, verbose=verbose))

  nc <- nchar(x)
  if (!nc)
    return(ret_if_empty)
  if (nc < n_length)
    return(x)

  total_iterations <- (nc - n_length) + 1
  seq_iterations <- seq.int(total_iterations)
  ret <- lapply(seq_iterations, function(i) {
    substr(x, start=i, stop=i+n_length-1)
  })

  return(unique(unlist(ret, recursive=FALSE, use.names=FALSE)))
}

string_to_words <- function(x, stem=TRUE, ret_if_empty="", verbose=TRUE) {
  verboseMsg(verbose, "Beginning string_to_words() - x has length", length(x))

  if (length(x) == 0)
    return(list())

  ## if x is a list, one of two options. 
  ## simple list then unlist
  ## complex list, use lapply
  if (is.list(x)) {
    if (any(sapply(x, length) > 1)) {
      # return(lapply(x, string_to_words, stem=stem, ret_if_empty=ret_if_empty, verbose=verbose))
      stop ("x must be an atomic vector or a simple list.\n\nHint, how should string_to_words be processed for a nested list?\nWhat would you do with the results? ")
      # eg, see:   string_to_words(list(c("this is all one sentance"), c("These are two.", "Different Sentences. How should they be treated?")))

    }
    x %<>% unlist(recursive=FALSE, use.names=FALSE)
  }

  ## Count characters of x, for identifying empty strings later
  nc <- nchar(x)
  
  # ret <- clean_wrapper_for_wordsplit(x) %>% {strsplit(., "\\s")[[1]]}
  ret <- clean_wrapper_for_wordsplit(x) %>% strsplit("\\s")

  verboseMsg(verbose, "Done cleaning in string_to_words()")

  ## any empty strings make "" instead of list()
  ret[ nc == 0] <- list("")

  ## Note to self:  Attempted to include stem on the original x (or after cleaning) but 
  ##   stemDocument does not vectorize across words inside a single element.  It only changes the last word in each element.
  ## Therefore must lapply AFTER having done strsplit()
  if (stem)
    ret %<>% lapply(tm::stemDocument)

  return(ret)
}


words_to_groups <- function(vec_of_words, n_length=2, sep=" ", ret_if_empty="", verbose=TRUE) {
  L <- length(vec_of_words)

  if (L == 0)
    return(ret_if_empty)
  if (is.list(vec_of_words) && L > 1)
    return(lapply(vec_of_words, words_to_groups, n_length=n_length, sep=sep, ret_if_empty=ret_if_empty, verbose=verbose))

  # browser(text="F")
  if (L == 1)
    return(vec_of_words)
  if (L <= n_length)
    return(pasteC(vec_of_words, C=sep))

  total_iterations <- (L - n_length) + 1
  unique(unlist(lapply(seq.int(total_iterations), function(i) pasteC(vec_of_words[seq(i, i+n_length-1)], C=sep)), FALSE, FALSE))
}

## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~  FUNCTIONS ABOVE HERE  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ##
## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ##
##    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~    ##




## Take from DT.merge only the columns needed
DT.m_for_letters <- DT.merge.bak[, list(label_sc_group, label_owner, labelid, imprint, artist_name, release_name, upc_has_at_least_one_infrac)]

## Chop words
{
  verboseMsg(verbose, "Starting word cleanup and chopping -- note the time")

  catheader("cleanup")
  s.t(title="artist clean name", DT.m_for_letters[, artist_name_2 := clean_wrapper(artist_name)])
  s.t(title="release clean name", DT.m_for_letters[, release_name_2 := clean_wrapper(release_name)])

  # catheader("chopping by length")
  # s.t(title="release chop, n = 3", DT.m_for_letters[, release_name_3parts := string_to_parts(release_name_2, n=3)])
  # s.t(title="release chop, n = 4", DT.m_for_letters[, release_name_4parts := string_to_parts(release_name_2, n=4)])
  # s.t(title="release chop, n = 5", DT.m_for_letters[, release_name_5parts := string_to_parts(release_name_2, n=5)])
  # s.t(title="artist chop, n = 3", DT.m_for_letters[, artist_name_3parts := string_to_parts(artist_name_2, n=3)])
  # s.t(title="artist chop, n = 4", DT.m_for_letters[, artist_name_4parts := string_to_parts(artist_name_2, n=4)])
  # s.t(title="artist chop, n = 5", DT.m_for_letters[, artist_name_5parts := string_to_parts(artist_name_2, n=5)])

  catheader("chopping by word -- NOT stemming")
  s.t(title="release chop by words",   DT.m_for_letters[, release_name_words := string_to_words(release_name, stem=FALSE)])
  s.t(title="artist chop by words",    DT.m_for_letters[, artist_name_words := string_to_words(artist_name, stem=FALSE)])

  catheader("chopping by word -- WITH stemming")
  s.t(title="release chop by words",   DT.m_for_letters[, release_name_word_stems := string_to_words(release_name, stem=TRUE)])
  s.t(title="artist chop by words",    DT.m_for_letters[, artist_name_word_stems := string_to_words(artist_name, stem=TRUE)])

  catheader("combining words to n-groups")
  s.t(title="combining release, n = 2", DT.m_for_letters[, release.word_tupples_2 := words_to_groups(release_name_word_stems, n_length=2, sep=" ")])
  s.t(title="combining release, n = 3", DT.m_for_letters[, release.word_tupples_3 := words_to_groups(release_name_word_stems, n_length=3, sep=" ")])
  s.t(title="combining release, n = 4", DT.m_for_letters[, release.word_tupples_4 := words_to_groups(release_name_word_stems, n_length=4, sep=" ")])

  s.t(title="combining artist, n = 2", DT.m_for_letters[, artist.word_tupples_2:= words_to_groups(artist_name_word_stems, n_length=2, sep=" ")])
  s.t(title="combining artist, n = 3", DT.m_for_letters[, artist.word_tupples_3:= words_to_groups(artist_name_word_stems, n_length=3, sep=" ")])
  s.t(title="combining artist, n = 4", DT.m_for_letters[, artist.word_tupples_4:= words_to_groups(artist_name_word_stems, n_length=4, sep=" ")])

  verboseMsg(verbose, "Done word cleanup and chopping -- note the time")

  jesusForData(DT.m_for_letters, info="CleanerWordSplits", verbose=verbose)
}



## For manually selecting some rows with infracs. This is not needed in production
## CREATE DT.samp
{
  tmp.N_samp <- 10000
  tmp.N_samp <- 300
  set.seed(1)
  inds.sample <- DT.m_for_letters[,
      c(which(release_name %in% c("Sing It, Vol. 1 (Karaoke Version)", "Gentlemen (Karaoke Versions) [Originally Performed by Psy]", "Blurred Lines (Karaoke Version) [In the Style of Robin Thicke, T.I. & Pharrell]"))
        , sample(which(upc_has_at_least_one_infrac), tmp.N_samp/10, FALSE)
        , sample(which(!upc_has_at_least_one_infrac), tmp.N_samp, FALSE)
        ) %>% unique %>% sample(tmp.N_samp, FALSE) %>% sort
  ]
  DT.samp <- DT.m_for_letters[inds.sample]
  # DT.samp <- {rbind(DT.m_for_letters[(upc_has_at_least_one_infrac)][sample(50000, tmp.N_samp/10)], DT.m_for_letters[!(upc_has_at_least_one_infrac)][sample(1000000, tmp.N_samp)])}
}

## Count the occurrances for each column
s.t(title="counting infractions", {
  colsErrOcc <- c("has_infr", "occurances")
  grps <- c("release", "artist")
  ll.COUNTS_by_grp_col <- emptylist(grps)
  for (grp in grps) {
    grpCol <- sprintf("%s_name", grp)
    cols <- c("%s_name_words", "%s_name_word_stems", "%s.word_tupples_2", "%s.word_tupples_3", "%s.word_tupples_4") %>% sprintf(grp)
    ll.COUNTS_by_grp_col[[grp]] <- emptylist(cols)
    for (col in cols) {
      verboseMsg(verbose, "Executing Grp '", grp, "' and col '", col, "'", sep="", minw=100, seconds=TRUE)
      ll.COUNTS_by_grp_col[[c(grp, col)]] <- 
        DT.m_for_letters[, list(word=unlst(get(col)), has_infr=sumn(upc_has_at_least_one_infrac), occurances=.N), by=c("labelid", grpCol)
                       ][, lapply(.SD, sumn), by=word, .SD=colsErrOcc
                       ][, percInfrac := has_infr / occurances
                       ][order(has_infr, decreasing=TRUE), rank_by_count := seq.int(.N)
                       ][order(percInfrac, decreasing=TRUE), rank_by_cropperc := (seq.int(.N) - sum(has_infr < 10)) %>% ifelse(. > 0, ., NA_integer_)
                       ][order(percInfrac, decreasing=TRUE), rank_by_fullperc := seq.int(.N)
                       ][order(has_infr, decreasing=TRUE)]
     }
  }
})

cls()
for (i in seq(ll.COUNTS_by_grp_col)) {
  for (j in seq(ll.COUNTS_by_grp_col[[i]])) {
    catheader(names(ll.COUNTS_by_grp_col[[i]])[[j]], endl=0)
    # print(head(ll.COUNTS_by_grp_col[[c(i, j)]], 25))
    print(head(ll.COUNTS_by_grp_col[[c(i, j)]][order(has_infr, decreasing=TRUE)][has_infr > 30 & percInfrac > 0.90], 30))
  }
}

&&&& TODO:  The ll.COUNTS_by_grp_col above is incorrect..  Hereis an example
&&&&        The issue is somewhere in the expansion
DT.jg <- DT.m_for_letters[sapply(artist_name_words, function(a) any(a=="glowacki"))]
{
  col <- "artist_name_words"
  grpCol <- "artist_name"
  DT.jg[, list(word=unlst(get(col)), has_infr=sumn(upc_has_at_least_one_infrac), occurances=.N), by=c("labelid", grpCol)
                         ][, lapply(.SD, sumn), by=word, .SD=colsErrOcc
                         ][word == "glowacki"]
}

## TEMP - quick and dirty to send to JP
# words <- ll.COUNTS_by_grp_col$release$release_name_words[percInfrac > 0.5 & has_infr >= 10, word]
# 
# DT.merge[, has_word := FALSE]
# for (word in words) {
#   catheader(word)
#   DT.merge[, has_word := has_word | grepl(word, release_name, ignore.case=TRUE)]
# }
# 
# upcs <- DT.merge[(has_word), upc]
# upcs2 <- DT.hidden[(upc %in% upcs)][date >= "2015-05-01", upc]
# 
# releases <- DT.merge[upc %in% upcs2, release_name]
# words_releases <- DT.m_for_letters[release_name %in% releases, release_name_words]
# 
# words_using <- words[(words %in% unlist(words_releases, FALSE, FALSE))]
# 
# ll.COUNTS_by_grp_col$release$release_name_words[word %in% words_using][order(has_infr, decreasing=TRUE)]
# ll.COUNTS_by_grp_col$release$release_name_words[word %in% words_using][order(percInfrac, decreasing=TRUE)]
# 
# ll.COUNTS_by_grp_col$release$release_name_words[(has_infr > 100 & percInfrac > 0.25)]
# ll.COUNTS_by_grp_col$release$release_name_words[(percInfrac < 0.25)]
# ll.COUNTS_by_grp_col$release$release_name_words[order(has_infr, decreasing=TRUE)] [1:90]
# ll.COUNTS_by_grp_col$release$release_name_words[grepl("^vol", word)][has_infr > 5]
# 
# 
# 
# ll.COUNTS_by_grp_col$artist$artist_name_words[(has_infr > 100 & percInfrac > 0.25)]
# 
# ll.COUNTS_by_grp_col$artist$artist_name_words[(has_infr > 3000 & percInfrac > 0.25) | (has_infr > 500 & percInfrac > 0.5) | (has_infr > 50 & percInfrac > 0.9)][order(percInfrac, decreasing=TRUE)] %>% print(nrow=333)
# 

## Count more accurately
if (FALSE)
{
  ll.precisions    <- emptylist(grps)
  ll.precisions_90 <- emptylist(grps)

  DT.merge.recent <- DT.merge[release_date_added >= today() - 200]

  for (grp in grps) {
    rm(inds, inds_90)
    catheader("GROUP: ", grp, prel=3, endl=1)
    grpCol <- sprintf("%s_name", grp)

    ## Identify words;  They must have the following criteria
    words <- ll.COUNTS_by_grp_col[[sprintf(c("%s", "%s_name_words"), grp)]
                ][ (has_infr > 10 & percInfrac > 0.25) | has_infr > 1000 & percInfrac > 0.15 | (percInfrac > 0.5 & has_infr > 5), unique(word)]

    ll.Infracs_caught    <- emptylist(words)
    ll.Infracs_caught_90 <- emptylist(words)

    for (.w in words) {
      catheader(.w, prel=0, endl=0)

      inds_90 <- grepl(.w, DT.merge.recent[[grpCol]], ignore.case=TRUE)
      ll.Infracs_caught_90[[.w]] <- 
          DT.merge.recent[, list(  filter_against=grpCol
                              ,           word = .w
                              ,   infracs_caught=sumn(inds_90 &  upc_has_at_least_one_infrac)
                              , false_positives=sumn(inds_90 & !upc_has_at_least_one_infrac)
                              ,   total_flagged=sumn(inds_90)
                              ,   first_90     = TRUE
                              )]

      inds <- grepl(.w, DT.merge[[grpCol]], ignore.case=TRUE)
      ll.Infracs_caught[[.w]] <- 
               DT.merge[, list(  filter_against=grpCol
                              ,           word = .w
                              ,   infracs_caught=sumn(inds &  upc_has_at_least_one_infrac)
                              , false_positives=sumn(inds & !upc_has_at_least_one_infrac)
                              ,   total_flagged=sumn(inds)
                              ,   first_90     = FALSE
                              )]

    }

    ll.precisions_90[[grp]] <- rbindlist(ll.Infracs_caught_90)[, filter_against := grpCol][, first_90 := TRUE]
    ll.precisions[[grp]]    <- rbindlist(ll.Infracs_caught)[, filter_against := grpCol][, first_90 := FALSE]
  }

  ll.precisions_90[[grp]][infracs_caught > 11 & false_positives < 100][order(false_positives)]
  ll.precisions[[grp]][infracs_caught > 11 & false_positives < 100][order(false_positives)]

  ## This shows that this is kind of meaningless
  DT.merge.recent[, list(lifetime_infracs=sumn(upc_has_at_least_one_infrac), total_releases=.N)]
  catches.by_release <- ll.precisions_90[["release"]][word %in% words.release][, lapply(.SD, sumn), .SD=c("infracs_caught", "false_positives", "total_flagged")]
}




## ALTERNATE --- count by all words
if (FALSE) 
{
  words.artist <- c("trybal", "khs", "clasical", "ventes", "floorfillers", "rodrick", "release", "ringtonefeeder", "momma", "klassik", "pick", "mechanics", "jvc", "networks", "2010s", "große", "bernie", "shirelles", "getting", "begin", "spectres", "health", "said", "bowles", "archived", "sunset", "gainsbourg", "comptones", "cochran", "me", "adrenalin", "chet", "platters", "baker", "covered", "puente", "hallyday", "edge", "singers", "conducted", "cliff", "lagrot", "hitmakers", "cardio", "vagabonds", "future", "wonders", "feel", "deluxe", "country", "generation", "shindig", "bvox", "chords", "bubble", "vibe", "spots", "grandmastaz", "saturday", "2000s", "since", "straight", "presley", "chaos", "pub", "elvis", "american", "rockers", "feast", "serge", "klassickuts", "shawn", "strueres", "ink", "belafonte", "orchester", "mims", "entertainer", "heaven", "jukebox", "garland", "pure", "judy", "life", "helisek", "tribute", "99", "energy", "let", "stars", "academy", "ultimate", "metro", "renegade", "club", "monsters", "allstars", "infinite", "mytones", "children", "union", "backtrax", "professionals", "mr", "wildlife", "mania", "new", "idols", "undercover", "night", "sbi", "cowboys", "songs", "theme", "kiboomu", "dean", "halloween", "callas", "standards", "cash", "neill", "nation", "o", "collective", "platinum", "around", "brothers", "sinatra", "relax", "365")
  words.release <- c("performed", "originally", "legends", "tribute", "mi", "portrait", "portrait", "artist", "artist", "top", "great", "two", "make", "mine", "mine", "albums", "albums", "price", "price", "double", "double", "trax", "jukebox", "jukebox", "30", "vocals", "demonstration", "deluxe", "audio", "ever", "greats", "xmas", "five", "chorus", "absolutely", "drews", "lovers", "melody", "yours", "in2christmas", "sbi", "75", "songbook", "backing", "verano", "verano", "101", "justin", "minaj", "nicki", "gangnam", "spears", "swift", "britney", "snow", "newport", "chet", "chet", "million", "drake", "baker", "kanye", "sellers", "dreamboat", "dreamboat", "bieber", "chainz", "pitbull", "wiz", "gallery", "khalifa", "direction", "125", "clarkson", "eminem", "festive", "jezykowa", "wersja", "polska", "cyrus", "judy", "maroon", "rida", "tyga", "sheeran", "toppers", "toppers", "miley", "adele", "songz", "trey", "paisley", "iam", "calvin", "lily", "garland", "tina", "tinie", "tempah", "charts", "rae", "schoolboy", "azalea", "lovesongs", "chiddy", "2010s", "trainor", "meghan", "gainsbourg", "vamps", "dubz", "corbin", "lumineers", "pontoon", "generique", "tko", "kip", "brandon", "idina", "menzel", "swindell", "beiber", "tinashe", "maejor", "grouplove", "lolly", "interviewed", "wrabel", "loveable", "songlist", "manors", "bae", "maxsta", "keely", "tympany", "dustin", "inductee", "tapout", "hoskyns", "pinmonkey", "potions", "kongos", "ultimative", "devlin", "bizzle", "inedite", "redfoo", "longue", "cashin", "hoodie", "tumblr", "fakir", "jaden", "luh", "455", "menorca", "vance", "kiesza", "ppl", "odg", "chotis", "veranos", "futurebound", "saxophon", "whipping", "provider", "jahmene", "voncelle", "cherri", "throwbackthursday", "cooldown", "afire", "amitri")

  words.artist_highlighted <- c("singers","edge","tribute","2010s","stars","ultimate","sbi","standards","brothers","o","neill","idols","country","american","mytones","allstars","365","metro","club","hitmakers","future","nation","sunset","union","helisek","ringtonefeeder","trybal","clasical","khs","ventes","floorfillers","rodrick","release","momma","klassik","pick","mechanics","jvc","networks")
  words.release_highlighted <- c("performed","originally","legends","tribute","mi","portrait","portrait","artist","artist","top","great","two","make","mine","mine","albums","albums","price","price","double","double","trax","jukebox","jukebox","30","vocals","demonstration","deluxe","audio","ever","greats","jahmene","voncelle","beiber","hoodie","tumblr","maxsta","wrabel","tapout","bizzle")

  words.artist <-   words.artist_highlighted
  words.release <-   words.release_highlighted


  DT.merge.rates <- DT.merge[release_date_added >= today() - 200, list(release_date_added, label_name, release_name, artist_name, upc_has_at_least_one_infrac)]

  ## RELEASE
  s.t(title="Determining caught.by_release", {    
    catheader("release words", endl=0)
    DT.merge.rates[, caught.by_release := FALSE]
    for (.w in words.release) {
      catn(.w)
      DT.merge.rates[, caught.by_release := caught.by_release | grepl(.w, release_name, ignore.case=TRUE)]
    }
  })

  ## ARTIST
  s.t(title="Determining caught.by_artist", {    
    catheader("artist words", endl=0)
    DT.merge.rates[, caught.by_artist := FALSE]
    for (.w in head(words.artist16, 30)) {
      catn(.w)
      DT.merge.rates[, caught.by_artist := caught.by_artist | grepl(.w, artist_name, ignore.case=TRUE)]
    }
  })

  DT.merge.rates[,  true_positive.by_release := caught.by_release &  upc_has_at_least_one_infrac]
  DT.merge.rates[, false_positive.by_release := caught.by_release & !upc_has_at_least_one_infrac]

  DT.merge.rates[,  true_positive.by_artist := caught.by_artist &  upc_has_at_least_one_infrac]
  DT.merge.rates[, false_positive.by_artist := caught.by_artist & !upc_has_at_least_one_infrac]

  DT.merge.rates[,  true_positive.by_a_or_r := (caught.by_artist | caught.by_release) &  upc_has_at_least_one_infrac]
  DT.merge.rates[, false_positive.by_a_or_r := (caught.by_artist | caught.by_release) & !upc_has_at_least_one_infrac]
}

RECALL    := percent of   infracs caught / all infracs (ie correct TRUE)
PRECISION := percent of   infracs caught / all identified as infracs (ie TRUE / TRUE + FALSE POSITIVES)

## Extract column names for easy sapplying
caughtCols <- nwhich(sapply(DT.merge.rates, is.logical))
TPCols <- extract("true_positive", caughtCols)
FPCols <- extract("false_positive", caughtCols)
T_totalCol <- "upc_has_at_least_one_infrac"
F_totalCol <- "total_releases"

# DT.sums_of_rates <- {
#   DT.merge.rates[, lapply(.SD, sumn) %>% c(total_releases=.N), .SD=caughtCols][, lapply(.SD, as.integer)] %>% 
#   setnames("upc_has_at_least_one_infrac", "total_releases_with_infracs") %>% t %>% as.data.table(TRUE) %>% 
#   {.[, type := ifelse(grepl("\\.", rn), strsplit(rn, "\\.") %>% sapply(head, 1), "total")  %>% toFactorWithExpectedLevels(c("total", "caught", "true_positive", "false_positive")) ]} %>% 
#   {.[, by := ifelse(grepl("\\.by", rn), removeText(".*by_", rn), "total") ]} %>%
#   setkey(type) %T>% print %>% {.}
# }

sum_of_rates <- DT.merge.rates[, lapply(.SD, sumn) %>% c(total_releases=.N), .SD=caughtCols]
true_positives <- sum_of_rates[, (.SD / .SD[[T_totalCol]]) %>% lapply(fwp), .SD=c(TPCols, T_totalCol)] %>% {.[, (T_totalCol) := NULL]}
false_positives <- sum_of_rates[, (.SD / .SD[[F_totalCol]]) %>% lapply(fwp), .SD=c(FPCols, F_totalCol)] %>% {.[, (F_totalCol) := NULL]}

true_positives[, total_releases := sum_of_rates[[T_totalCol]]]
false_positives[, total_releases := sum_of_rates[[F_totalCol]]]

print(true_positives); print(false_positives)

f.out <- out.p("true and false positive rates", ext=".xlsx")
exportXLS.usingXLConnect(f.out, DTs.list=list(true_positives=true_positives, false_positives=false_positives))

quickEmail(f.out, getRS())

































col.a <- "artist.word_tupples_4"
col.r <- "release.word_tupples_4"
grpCol.r <- "release_name"
DT.samp[, list(word.r=unlst(get(col.r)), word.a=unlst(get(col.a)), has_infr=sumn(upc_has_at_least_one_infrac), occurances=.N), by=c("labelid", grpCol.r)
               ][, lapply(.SD, sumn), by=list(word.a, word.r), .SD=colsErrOcc
               ]



DT.rel.counts.word_raw[word == ""][, list(release_name == st, .N), by=release_name]
DT.rel.counts.word_raw[]
DT.rel.counts.word_raw[labelid == 22407][grepl("Henry", release_name)]

st <- "For Henry F. Farny 1904 34° 48'N / 111° 54' W 3308/4708"
strsplit(clean_wrapper_for_wordsplit(st), "\\W")

DT.rel.counts.word <- DT.rel.counts.word_raw[, lapply(.SD, sumn), by=list(labelid, word), .SD=c("has_infr", "occurances")]
DT.rel.counts.word[, word_stem := tm::stemDocument(word)]
DT.rel.counts.word_stem <- DT.rel.counts.word[, lapply(.SD, sumn), by=word_stem, .SD=c("has_infr", "occurances")]
DT.rel.counts.word_stem


## check 
 DT.rel.counts.word_raw[labelid == 22407][grepl("Henry", release_name)]


DT.samp[, list(w=unlst(release_name_words), has_infr=sumn(upc_has_at_least_one_infrac), occurances=.N), by=release_name][, lapply(.SD, sumn), by=w, .SD=c("has_infr", "occurances")]



DT.samp[, list(w=unlist(release_name_4parts, FALSE, FALSE), has_infr=percTrue(upc_has_at_least_one_infrac))]
[, list(perc=percTrue(has_infr),  .N), by=w][N>10][!grepl("\\w\\s+\\w", w)][order(perc, decreasing=TRUE)][1:15]
DT.samp[, list(w=unlist(release_name_5parts, FALSE, FALSE), has_infr=percTrue(upc_has_at_least_one_infrac))][, list(perc=percTrue(has_infr),  .N), by=w][N>10][!grepl("\\w\\s+\\w", w)][order(perc, decreasing=TRUE)][1:15]

DT.m_for_letters[, list(w=unlist(release_name_words,  FALSE, FALSE), has_infr=percTrue(upc_has_at_least_one_infrac))][, list(perc=percTrue(has_infr),  .N), by=w][N>10][!grepl("\\w\\s+\\w", w)][order(perc, decreasing=TRUE)][1:15]

col <- "artist_name_word_stems"
print(artist_counts <- DT.m_for_letters[, list(L=if (.N>1) length(get(col)[[1]]) else length(get(col)) ), by=artist_name][, .N, keyby=L], nrow=333)
DT.m_for_letters[, list(L=if (.N>1) length(get(col)[[1]]) else length(get(col)) ), by=artist_name][L == 0]
DT.m_for_letters[artist_name == "."]

DT.m_for_letters[, L.art := length(artist_name_word_stems), by=artist_name]
unique(DT.m_for_letters[L.art > 8000], by="artist_name")


atest <- c("Various Artists", "The Karaoke Channel", "ProSound Karaoke Band")
DT.m_for_letters[artist_name %in% atest][, list(L=if (.N>1) length(get(col)[[1]]) else length(get(col)) ), by=artist_name]
DT.m_for_letters[, all_duplicated_or_length_one(get(col)), by=artist_name][!(V1)]

all_duplicated_or_length_one <- function(x) length(x) == 1 || (all(tail(duplicated(x), -1)))

initDict("/usr/local/Cellar/wordnet/3.1/dict")


DT.m_for_letters[, list(w=unlist(get(col),  FALSE, FALSE), has_infr=upc_has_at_least_one_infrac)][, list(perc=percTrue(has_infr),  .N), by=w][N>10][!grepl("\\w\\s+\\w", w)][order(perc, decreasing=TRUE)][1:15]