setScience(proj="ContentAnalysis", subProj="ContentDefects", subl=TRUE, load=FALSE)

folder.in <- ingest.p("iTunes Content Defects")
f.in <- as.path(folder.in, "Content Defects - Edit Detail - Music - Monthly - The Orchard Enterprises_201601 - Three Month.xlsx")
stopifnot(file.exists(f.in))

gc()
DT.in <- XLStoDT(f.in, sheetAsCol=FALSE)
## If more than one sheet in the file, the results will be a *list of data.tables*
stopifnot(is.data.table(DT.in), nrow(DT.in)>1)
cleanColNamesForSQL_(DT.in)
BackUpOrRestore("DT.in", force=TRUE)

## DROP NAs
DT.in <- DT.in[!is.na(old_value) & !is.na(new_value)]

## DROP Classical, Opera etc
if (FALSE) {
  genres_ignoring <- c("Classical", "Opera")
  DT.in <- DT.in[primary_genre %ni% genres_ignoring]
}

# old <- DT.in[1:2, old_value]
# new <- DT.in[1:2, new_value]
# DT.in[, iconv(old_value[4], from="latin1", to="utf8")]
# DT.in[, iconv(old_value[4])]

DT.in[, characters_removed := word_set_diff(old_value, new_value)]
DT.in[, characters_added   := word_set_diff(new_value, old_value)]

DT.in[toupper(old_value) == toupper(new_value), c("characters_added", "characters_removed") := "Capitalization Variant"]

DT.in[characters_removed == "-"]

feature_words <- c("performer", "featuring", "primary", "remixer", "director", "with")

DT.in[, offense_group := NULL]
DT.in[grepl(regOr(feature_words), characters_added, ignore.case=TRUE) & grepl(regOr(feature_words), characters_removed, ignore.case=TRUE), offense_group := "Confusing rule about 'Primary' 'Featuring' 'Performer' etc"]
DT.in[characters_removed == "FEAT|." & characters_added == "WITH", offense_group := "Confusing rule about 'Primary' 'Featuring' 'Performer' etc"]
DT.in[characters_removed == "Capitalization Variant" & characters_added == "Capitalization Variant", offense_group := "Capitalization Variant"]
DT.in[characters_removed == "<ERROR>" & characters_added == "<ERROR>", offense_group := "Unknown Issue (possible UTF-8 error)"]
DT.in[characters_removed == "" & characters_added == "(|LIVE|)", offense_group := "'(Live)' Corrected"]


DT.in[characters_removed == "" & characters_added == "(|REMASTERED|)", offense_group := "'(Remaster)' Added"]
DT.in[grepl("\\b(I|II|III|IV|V)\\b", characters_added), offense_group := "Roman Numerals"]
DT.in[grepl("É", characters_added), offense_group := "Accented non-english words"]


DT.in[grepl("PART", characters_removed), offense_group := "'Part' ~~> 'Pt.'"]
DT.in[grepl("PT", characters_added) & is.na(offense_group), offense_group := "'Pt.' Missing or improperly styled"]
DT.in[characters_added %in% c("(|)", "[|]") & is.na(offense_group), offense_group := "Parens or Brackets"]
DT.in[is.na(offense_group) & characters_removed == "-", offense_group := "Superfluous Hyphen"]
DT.in[grepl("(\\S/|\\S/)", old_value) & grepl("(\\s/\\s)", new_value), offense_group := "Spaces around ' / ' slashes"]

DT.in[is.na(offense_group) & nchar(removeText("\\S+", old_value)) > nchar(removeText("\\S+", new_value)) & characters_removed == "" & characters_added == "", offense_group := "Superfluous Space Removed"]
DT.in[is.na(offense_group) & nchar(removeText("\\S+", old_value)) < nchar(removeText("\\S+", new_value)) & characters_removed == "" & characters_added == "", offense_group := "Superfluous Space Added (often after period)"]
DT.in[characters_added == "" & characters_removed == "" & is.na(offense_group), offense_group := "Word Rearrangement"]
DT.in[characters_added %in% c(",", ",|.") & characters_removed == "" & is.na(offense_group), offense_group := "Comma missing (generally before 'Pt. X') -- Possibly also missing a period after 'Pt'"]
DT.in[grepl("'", characters_added), offense_group := "Apostrophe Missing"]

live_words <- c("LIVE", "INSTRUMENTAL", "VERSION", "VIVO")
DT.in[grepl(regOr(live_words, underbound=TRUE), characters_added) & is.na(offense_group), offense_group := "added version such as Live, Version, Instrumental, etc"]
DT.in[grepl("FEAT\\|", characters_removed) & characters_added == "" & is.na(offense_group), offense_group := "Removed 'Featuring' information"]
DT.in[characters_removed =="AND" & characters_added == "&", offense_group := "'and' ~~> '&'"]
DT.in[grepl("LIVE", characters_removed) & characters_added == "", offense_group := "Removed 'Live'"]
DT.in[primary_genre %in% c("Classical", "Opera") & is.na(offense_group), offense_group := "Misc. Classical/Opera Errors"]

DT.in[is.na(offense_group), offense_group := "  -- All Other Issues -- "]


# ------- 
DT.err_types <- DT.in[, .N, by=offense_group][order(N, decreasing=TRUE)]

## Tickets by type
f.out.err_types <- writeDT(DT.err_types)
reveal(f.out.err_types)

## For bullethead specifically
issues_for_bullethead <- c("Confusing rule about 'Primary' 'Featuring' 'Performer' etc", "Removed 'Featuring' information", "Word Rearrangement")
f.out.issues_for_bh <- writeDT(DT.in[offense_group %in% issues_for_bullethead], base="tickets for bullethead")
reveal(f.out.issues_for_bh)

## --------------------- ##

show_top_errors(Inf, N=1, NULL)[, sum(N)]
nrow(DT.in)

DT.in[characters_removed == "-" & characters_added == ""][1:2]
DT.in[characters_removed == "" & characters_added == ""][1:2]

show_pat(regOr(c("(|)", "[|]"), escape=TRUE), 85, "ad")
show_pat("remastered", 85, "ad", .SDcols=names(DT.in))


show_pat("FEATURING", 85, "ad")
show_pat("FEATURING", 85, "rem")



## one issue
DT.in[grepl("live", old_value, ignore.case=TRUE), list(edit_type, old_value, new_value, characters_added, characters_removed)][nchar(new_value) < 30]


## NEEDS LOOK AHEAD
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
testString <- "take me to funky town"
FF2 <- "(?<=.)(?=funky)"
strsplit(testString, FF2, perl=TRUE)

FF <- "(?=funky)"
testString <- "take me to funky town"
strsplit(testString,FF,perl=TRUE)
# [[1]]
# [1] "take me to " "f"           "unky town"  

FF <- "(?=funky)"
testString <- "funky take me to funky funky town"
strsplit(testString,FF,perl=TRUE)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  strsplit(new, "")

}

diff_is_capitalization <- function() {
  utf8ToInt("b") - utf8ToInt("B")
}