split_to_words_and_symbols <- function(x) {
  strsplit(x, "(?=[^[:alnum:]])", perl=TRUE)
}

word_set_diff <- function(from, to) {
## example of when this fails
# split_to_words_and_symbols("Piano Sonata No. 3 in F Minor, Op. 5: IV. Intermezzo. Andante molto")
# split_to_words_and_symbols("Piano Sonata No. 3 in F Minor, Op. 5: IV. Intermezzo - Andante molto")

  ret <- mapply(function(f, t) {print(f) ; valueIfErr(setdiff(toupper(f), toupper(t)), "<ERROR>") %>% pasteC(C="|")}
    , f=split_to_words_and_symbols(from)
    , t=split_to_words_and_symbols(to)
  )
  message("reminder: word_set_diff is imperfect since it looks at setdiff(), not word order")

  return(ret)
}

show_pat <- function(pat, nc.max=30, col=c("content_title", "old_value", "new_value", "added", "removed"), ignore.case=TRUE, .SDcols=c("content_title", "edit_type", "old_value", "new_value", "characters_removed", "characters_added")) {
  col <- match.arg(col)
  if (col %in% c("added", "removed"))
    col %<>% paste0("characters_",.)
  DT.in[grepl(pat, get(col), ignore.case=ignore.case)
      # , list(edit_type, old_value, new_value, characters_removed, characters_added)
      , .SD, .SDcols=.SDcols
      ][nchar(new_value) < nc.max]
}

show_top_errors <- function(nc.max=35, N_min=2, ignore=c("PRIMARY", "FEATURING", "PERFORMER")) {
    DT.in[nchar(characters_removed) <= nc.max, .N, keyby=list(characters_removed, characters_added)
    ][order(N, decreasing=TRUE)
    ][N >= N_min
    # ][!(characters_removed == "" & characters_added == "")
    ][!(characters_removed == "<ERROR>" & characters_added == "<ERROR>")
    ][characters_removed %ni% ignore & characters_added %ni% ignore]
}