## TODO: change the attributes to data.table info

if (FALSE) {
    DT[, percTrue(is.na(get(col))), by=date_file]

    lapply(breakageFiles, function(f) readLines(f) %>% head(6) %>% c("======================", f, "--------------\n") %>% catnn) %>% invisible()

    debug(readSpotifyFiles)
    readSpotifyFiles(breakageFiles, append.content_owner=TRUE, append.file_name=FALSE, verbose=FALSE)
}


simpleClean_of_spotify_DT_ <- function(DT, remove_file_metadata=TRUE, convertFactors=TRUE) {
  if (!is.data.table(DT))
    stop ("Something is wrong. DT sent to simpleClean_of_spotify_DT_() is NOT a data.table")

  DT.nm <- capture.output(substitute(DT))

  ## Convert factors to columns
  setFactorsToChars_(DT)

  ## Possible columns which contain the date data
  startCol  <- intersect(c("Start date", "Report start date"), names(DT))
  endCol    <- intersect(c("End date",   "Report end date"),   names(DT))

  ## Generally, we want to make month := `Start date` and drop `End date`
  ## All the extra lines are to confirm that the data is complete and as expected
  if (length(startCol) == 1 && length(endCol) == 1)  {
    ## ERROR CHECK
    if ("month" %in% names(DT))
      stop ("column 'month' already exists in DT")

    ## CLEAN UP Mis-formatted DATES and check for NAs
    for (col in c(startCol, endCol)) {
      # DT[grepl("^\\d{2}/\\d{2}/\\d{2}", get(col)), (col) := as.character(as.Date(get(col), format="%m/%d/%y", origin=.origin))]
      if (any(is.na(DT[[col]])))
        warning ("There are NAs in `", col, "`for\n     ", DT.nm, "[date_file %in% as.Date(c", pasteQ(DT[is.na(get(col)), sort(unique(date_file))]), ")]" )
    }

    if (any(is.na(DT[, .SD, .SDcols=c(startCol, endCol)])))
      warning ("There are NAs in dat cols of ", DT.nm)

    if (all( DT[, (get(startCol) == get(endCol)) | isWholeMonth(get(startCol), get(endCol), warn.change=FALSE)] )) {
        DT[, (endCol) := NULL]
        setnames(DT, startCol, "month")
        if (all(DT[, date_file == month]))
          DT[, date_file := NULL]
        else 
          warning ("DT$date_file does NOT equal DT$month for ", DT.nm)
    } else {
      warning ("Date columns were not cleaned for ", DT.nm)
    }
  }

  ## The Data Point & Report Type field are constant across the whole report. 
  ##  instead, set it as an attribute of the file
  ## (The only time this optino will generally be FALSE is when the DT is DT.MISSING_DATA)
  ##
  ## TODO:  Add it as info of the DT
  if (remove_file_metadata) {
    if ("Data point" %in% names(DT)) {
      setattr(DT, "data_point", DT[1, `Data point`])
      DT[, `Data point` := NULL]
    }
    if ("Report type" %in% names(DT)) {
      setattr(DT, "report_type", DT[1, `Report type`])
      DT[, `Report type` := NULL]    
    }
  }

  ## Clean up names
  ## ------------------------------------------------------------------ ##
      # nms <- names(DT)
      # nms <- spaceToUnderscore(cleanWS(tolower(nms)))
      # nms <- gsub("\\#", "No.", nms)
      # nms <- gsub("(_?)>(_?)", "_above_", nms)
      # nms <- gsub("licensor", "orchard", nms)
      # nms <- gsub("^product_tag$", "product", nms)
      # setnames(DT, nms)

  if (any(grepl("Rightholder", names(DT), ignore.case=TRUE)))
    cleanRightsholderColumn_(DT)

  ## NEW METHOD
  ## Clean up the names
  names(DT) %>% 
    tolower(.)                       %>%
    cleanWS(.)                       %>%
    spaceToUnderscore(.)             %>%
    gsub("ad-supported", "adsupported", .)   %>%
    # gsub(" ", "_", .)                %>%
    gsub("\\-", "_", .)              %>%
    gsub("\\(|\\)", "_", .)          %>%
    gsub("%", "percent", .)          %>%
    gsub("\\#", "number", .)         %>%
    gsub("(_?)>(_?)", "_above_", .)  %>%
    gsub("licensor", "orchard", .)   %>%
    gsub("^product_tag$", "product", .)   %>%
    gsub("__+", "_", .)              %>%
    gsub("_+$", "", .)               %>%
    setnames(DT, .)

  ## ------------------------------------------------------------------ ##

  ## Clean country column and add country metadata
  if ("country" %in% names(DT) && nchar(DT$country) <= 2) {
    setnames(DT, "country", "country_code")
    addCountry.byCode_(DT, colsToBring=c("country_name", "region_group", "continent"), showWarnings=TRUE)
    if ("country_name" %in% names(DT)) {
      DT[country_name == "Taiwan, Province Of China", country_name := "Taiwan"]
      DT[is.na(country_name), country_name := country_code]
    }
    ## Any unmatched region_groups are set to NA
    if ("region_group" %in% names(DT)) {
      DT[is.na(region_group), region_group := "ROW"]
      region_group_to_factor_(DT)
    }
  }

  ## TODO: Do we need to check for alternate spellings??
  ## ---------------------------------------------------------------------
  ## Convert Total Tracks to integer64, so that we can sum the whole of it
  convert_to_int64_if_sum_breaks_numeric_(DT, c("Total tracks", "total_tracks"), showWarnings=FALSE)
  # old version # 
  # old version #  ## First confirm that it is in the DT.  Then check if the sum is beyond machine limit.
  # old version #  for (tt in c("Total tracks", "total_tracks")) {
  # old version #    if (tt %in% names(DT)) {
  # old version #      if (sum(as.numeric(DT[[tt]])) > (.Machine$double.xmax/10)) {
  # old version #        DT[, (tt) := bit64::as.integer64(as.numeric(get(tt)))]
  # old version #      }
  # old version #      else 
  # old version #        DT[, (tt) := as.numeric(get(tt))]
  # old version #    }
  # old version #  }

  ## Some percentage columns are presented as whole integers instead of a proper decimal. Convert these. 
  colsPercent <- extract("_share|percentage", names(DT))
  for (col in colsPercent)
    DT[, (col) := validPercentage.quick_0_1(get(col), verbose=FALSE)]

  ## The starting '3' in '3rd' in category messes ordering. Convert to 'third'
  if ("category" %in% names(DT))
    DT[category == "3rd-party", category := "third-party"]

  ## Add product_description if it is not present and the dictionary exists
  ## ------------------------------------------------------------------ ##
  if (exists("DT.spotify_product_dict"))
    if ("product" %in% names(DT) && "product_description" %ni% names(DT)) {
      ## In some of the spotify data, the 'product' column is in fact the product description.
      ## We check if that's the case and if so, indicate
      if (percTrue(DT$product %in% DT.spotify_product_dict$product_description))
        warning ("\n\n------------  \n\tIt looks like product in this table is product_description")

      if (any(DT$product %in% DT.spotify_product_dict$product)) {
        addColsFrom_(DT, DT.spotify_product_dict, colsToBring="product_description", joinCols="product")
        DT[is.na(product_description), product_description := product]
      }
  }

  ## Convert these indicated columns to factors
  ## ------------------------------------------------------------------ ##
  if (convertFactors) {
    for (colName in c("product", "product_description", "type_of_trial", "category"))
      if (colName %in% names(DT)) {
        lev <- DT[, unique(as.character(get(colName)))]
        ## Order, numbers first, then letters
        lev <- lev[order(as.num.nowarn(gsub("(^\\d+).*", "\\1", lev)), lev)]
        ## make sure no levels got left out
        lev <- c(lev, setdiff(DT[[colName]], lev))
        DT[, (colName) := factor(get(colName), levels=lev)]
        rm(lev)  
      }
  }

  ## Convert date columns and add quarter information
  ## ------------------------------------------------------------------ ##
  dateCols <- getDateColNames(DT, semiloose=TRUE)
  if (length(dateCols)) {
    try(DT[, (dateCols) := lapply(.SD, as.Date, origin=.origin), .SDcols = dateCols])
    ## Add Quarter date columns
    for (dateCol in dateCols)
      addQuarter_(DT, dateCol=dateCol)
  }

  ## Auto-Convert integers and numbers as appropriate
  convertNumberCols_(DT, verbose=FALSE)


  ## ------------------------------------------------------------------ ##
  ## Set key
  ## ------------------------------------------------------------------ ##
  possibleKeys <- c(dateCols, "product", "product_description", "country_code", "country", "type_of_ad", "category", "platform", "type_of_trial")
  keys <- intersect(possibleKeys, names(DT))
  setkeyIfNot(DT, keys, organize=TRUE, verbose=FALSE)


  ## organize slightly
  if ("file" %in% names(DT))
  setcolorderpt(DT, endCols="file", showWarnings=FALSE)

  return(invisible(DT))
}