is_various_artists <- function(artist_name, already_cleaned=FALSE) {
## looks for pattern of various artists in a string

  if (!length(artist_name))
    return(logical(0L))
  
  if (!is.character(artist_name))
    stop("'artist_name' must be a string")

  if (!already_cleaned) {
    artist_name %<>% clean_names_to_simple_alpha(whitespace=TRUE, inside_parens=TRUE, parens=TRUE, trim=TRUE, tolower=TRUE)
  }

  pat <- c("^v\\.?a\\.?$", "variou?s\\s*artist[a-z]?s?", "artista?s?\\s*varios", "vari.{,5}\\s*artisti") 
  pat %<>% regOr(escape=FALSE)

  ## No need to ignore case because the text is cleaned
  grepl(pat, artist_name, ignore.case=FALSE)
}


get_dim_artist_cleaned <- function(DT.artists_cleaned=copy(get_dim_artist(refresh=refresh_dim_artist)), refresh_dim_artist=FALSE, confirm_unique_hash=TRUE, verbose=TRUE) {
  stopifnot(require(digest))
  force(DT.artists_cleaned)

  verboseMsg(verbose, "Getting DT.artists_cleaned  -------------------------------- ")

  if (!all(c("artist_name_cleaned", "artist_name_cleaned_nows") %in% names(DT.artists_cleaned))) {
    verboseMsg(verbose, "Adding artist_name_cleaned / artist_name_cleaned_nows to DT.artists_cleaned", sep="", time=FALSE)
    add_cleaned_column_(DT.artists_cleaned, "artist_name", whitespace="both")
  }

  DT.artists_cleaned[, is_various_artists := is_various_artists(artist_name_cleaned, already_cleaned=TRUE)]

  ## Create a flat string of IDs per each name, and cleaned name
  verboseMsg(verbose, "Adding CSV by name for each column")
  if ("csv.aid_by_name"               %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, csv.aid_by_name               := vector_to_flat_string(artistid), by=artist_name]
  if ("csv.aid_by_name_cleaned"       %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, csv.aid_by_name_cleaned       := vector_to_flat_string(artistid), by=artist_name_cleaned]
  if ("csv.aid_by_name_cleaned_nows"  %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, csv.aid_by_name_cleaned_nows  := vector_to_flat_string(artistid), by=artist_name_cleaned_nows]

  ## Hash them for smaller strings
  verboseMsg(verbose, "Adding HASH by name for each column")
  if ("hash.aid_by_name"               %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, hash.aid_by_name               := digest(csv.aid_by_name,              algo="murmur32")]
  if ("hash.aid_by_name_cleaned"       %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, hash.aid_by_name_cleaned       := digest(csv.aid_by_name_cleaned,      algo="murmur32")]
  if ("hash.aid_by_name_cleaned_nows"  %ni% names(DT.artists_cleaned)) DT.artists_cleaned[, hash.aid_by_name_cleaned_nows  := digest(csv.aid_by_name_cleaned_nows, algo="murmur32")]

  if (confirm_unique_hash) {
    verboseMsg(verbose, "Confirming unique hash by name ... ")
    confirmed <- TRUE
    for (sfx in c("", "_cleaned", "_cleaned_nows"))
      if (1 != nrow(unique(DT.artists_cleaned, by=c(paste0("csv.aid_by_name", sfx)))[, .N, by=c(paste0("hash.aid_by_name", sfx))])) {
        warning("uniqueness of the hash may NOT be confirmed for sfx == ", sfx, "\n")
        confirmed <- FALSE
      }
    if (confirmed)
      verboseMsg(verbose, "uniqueness of the hash confirmed in get_dim_artist_cleaned()", time=FALSE)
  } 


  verboseMsg(verbose, "Done Getting DT.artists_cleaned  -------------------------------- ")
  return(invisible(DT.artists_cleaned))
}


get_artist_id_from_fuzzy_artist_name <- function(artist_name, refresh_dim_artist=FALSE, full_return=FALSE, verbose=TRUE) {
  # DT.ret <- data.table(
  #     artist_name=artist_name
  #   , artist_name_semi_cleaned = clean_names_to_simple_alpha(artist_name, whitespace=FALSE, inside_parens=TRUE, parens=TRUE, trim=TRUE, tolower=TRUE, duplicate_whitespace=TRUE)
  #   , artist_name_cleaned = artist_name
  #   )

  DT.artists_cleaned <- get_dim_artist_cleaned(refresh_dim_artist=refresh_dim_artist, verbose=TRUE)

  # DT.ret <- data.table(artist_name=iconv(artist_name, to=convert_to, from=convert_from))
  DT.ret <- data.table(artist_name=artist_name, row_number=seq_along(artist_name))
  add_cleaned_column_(DT.ret, cols=c("artist_name"))

  ## Add in matches
  sfxs <- c("", "_cleaned", "_cleaned_nows")
  message("You are about to get three warnings for 'bmerge(...) .. known encoding UTF-8 ...' \nYou can disregard these")
  for (n in sfxs) {
    kCol <- sprintf("artist_name%s", n)
    iCol <- sprintf("csv.aid_by_name%s", n)
    # j.expr <- sprintf("(%s) := i.%1$s", iCol)
    matchKey(DT.ret, DT.artists_cleaned, keyCols=kCol, superset.ok=FALSE, verbose=FALSE)
    DT.ret[unique(DT.artists_cleaned, by=kCol), (iCol) := get(iCol)]
    catn()
  }

  ## We ultimately want to create a tall table, with each artist_name repeaeted as many rows 
  ##  as it has matches for artistid
  ##  and indicate in each row whether the match is exact or using 'name_cleaned' or 'name_cleaned_nows'
  ## 
  ##  To execute, strsplit each CSV to create three different DTs, adding match_type to each DT
  ##  Stack the DTs together, set match_type to a factor (so that we can find its 'min' easily)
  ##  Then take the min match_type by artistid 
  ##  Take along the metadata, ie the 'byCols'

  ## Metadata columns
  byCols <- paste0("artist_name", sfxs) %>% c("row_number", .)
  ## Which columns to split, one at a time
  parseCols <- sprintf("csv.aid_by_name%s", sfxs)

  ### Split and Stack them
  DT.large_ret <- lapply(parseCols, function(pCol) {
    DT.ret[, strsplit(get(pCol), ","), by=byCols][, match_type := removeText("csv.aid_", pCol)]
  }) %>% rbindlist %>% setnames("V1", "artistid")

  ## Set to factor to find the 'min'
  DT.large_ret[, match_type := toFactorWithExpectedLevels(match_type, paste0("by_name", sfxs))]

  ## Take the min (ie, sort then take the first), by each artistid-metadata combination
  return(
    DT.large_ret[, list(match_type = sort(match_type)[[1]]), keyby=c(byCols, "artistid")]
  )
}