# -------------------------------------------------------------------------------------------------------------------------  #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  #                                                                                                                            #
  #           File Name              :  setcolorder.r                                                                          #
  #           Last Updated Funclist  :  08 Feb 2015,  5:12 AM (Sunday)                                                         #
  #                                                                                                                            #
  #           Author Name            :  Rick Saporta                                                                           #
  #           Author Email           :  RickSaporta@gmail.com                                                                  #
  #           Author URL             :  www.github.com/rsaporta                                                                #
  #                                                                                                                            #
  #           Packages Called        :  NA                                                                                     #
  #           Packages Used via NS   :  NA                                                                                     #
  #                                                                                                                            #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  #                                                                                                                            #
  #   setcolorderpt      ( x, startCols=neworder, endCols=endcols, failOnMissingCols=FALSE, showWarnings=TRUE                  #
  #                        , endcols=NULL, neworder=NULL, sort.middle.cols=FALSE )                                             #
  #   setcolorder_suffix ( DT, suffixes, startCols=key(DT), endCols=NULL                                                       #
  #                        , sortAllColsAlphabetically.exceptStartCols=FALSE, sep.forSuffix="." )                              #
  #                                                                                                                            #
  #                                                                                                                            #
  #                                                     <END FUNCS>                                                            #
  #  -----------------------------------------------------------------------------------------------------------------------   #
  # -------------------------------------------------------------------------------------------------------------------------  #

## setcolorder.r

## NOTE TO SELF on usage of  functionName_ ()  
##   ie on using an udnerscore for functions that modify by reference. 
##   MD's convention is that functions that modify by ref should start with set..()
##   I dont necessarilly aggree, but I do believe that using  setFooBar_() is redundant


## ----------------------------- ##
## This file has: 
## ----------------------------- ##
##    setcolorder_suffix()
##    setcolorderpt()
## ----------------------------- ##



setcolorderpt <- function(x, startCols=neworder, endCols=endcols, failOnMissingCols=FALSE, showWarnings=TRUE,  endcols=NULL, neworder=NULL, sort.middle.cols=FALSE) {
# Set Col Order Pt (as in, partly)
# Wrapper to setcolorder, where this function allows for a partial 
#   list of column names.  If this list is missing names (relative to names(x))
#   then it is filled with the remaining names from x
  # use the key as default


  if (!missing(endcols))
    warning ("'endcols' in setcolorderpt() has been deprecated.\n  use 'endCols' instead   (note the capitalization in 'Cols'")
  if (!missing(neworder))
    warning ("'neworder' in setcolorderpt() has been deprecated.\n  use 'startCols' instead")

  ## if neighter startCols nor endCols are given and x has a key, use key(x)
  if (!length(c(startCols, endCols)) && !is.null(key(x)))
    startCols <- key(x)

  # Check that there are not any duplicate names
  if (length(startCols) || length(endCols)) {
    if (any(wh <- duplicated(c(startCols, endCols)))) {
      ## identify which columns for warning message
      wh <- c(startCols, endCols)[wh]

      ## Remove duplicates from each indivdual group
      startCols <- startCols[!duplicated(startCols)]
      endCols  <- endCols[!duplicated(endCols)]
      ## Remove any elements of startCols that might be in endCols
      endCols  <- setdiff(endCols,  startCols)
      
      if(showWarnings)
        warning(warningCols("\nThere were duplicate column names in `c(startCols, endorder)` which have been removed.\nOffenders were:  ", wh))
    }
  }

  # ------------------------------------------------------------------ #
  # check that there aren't any duplicate names in the DT, which will cause errors
  # ------------------------------------------------------------------ #
  if (anyDuplicated(names(x)))
    stop ("\n\nCannot use setcolorderpt when there are duplicate column names.\nOffending columns are:\n   ", pasteQand(names(x)[duplicated(names(x))]))

  # ------------------------------------------------------------------ #
  # check that there aren't any names given that are not names of the DT
  # ------------------------------------------------------------------ #
    if (any(miss <- !(startCols %in% names(x)))) {
      msg <- paste0("\n[from setcolorderpt]. The following column names in `startCols` are not in the names of `", pasteC(as.expression(substitute(x))), "`\n\t", 
                     paste0(startCols[miss], collapse="\n\t"), "\n",
                     ifelse(!failOnMissingCols, "These columns will be ignored.\n", "") )
      if(failOnMissingCols)
         stop(msg)
      # else
   
      if (showWarnings)
        warning(msg)
      startCols <- startCols[!miss]
    }

        # ------------------------------------------------- #
    if (any(miss <- !(endCols %in% names(x)))) {
      msg <- paste0("\nThe following column names in `endCols` are not in the names of `", pasteC(as.expression(substitute(x))), "`\n\t", 
                     paste0(endCols[miss], collapse="\n\t"), "\n",
                     ifelse(!failOnMissingCols, "These columns will be ignored.\n", "") )
      if(failOnMissingCols)
         stop(msg)
      # else

      if (showWarnings)     
        warning(msg)
      endCols <- endCols[!miss]
    }
  # ------------------------------------------------------------------ #

  ## The new order should be alll of the columns of x as they should now appear
  middleCols <- setdiff(names(x), c(startCols, endCols))
  if (sort.middle.cols)
    middleCols <- sort(middleCols)

  finalOrdering <- c(startCols, middleCols, endCols)

  ## 2015-01-25:  I dont remember what this line is here for. 
  ##     Possibly something to do with when a data.table has improper names
  ##     Speciifcally, duplicate names. 
  missing <- names(x)[!names(x) %in% intersect(names(x), finalOrdering)]

  setcolorder(x, finalOrdering)
}



# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #


## THIS IS AN EXAMPLE FOR setcolorder_suffix ()
if (FALSE) 
{
    DT <- data.table(structure(list(storeid = c(1L, 286L), paidunits.acc = c(677062317,7898943243), revenue.acc = c(96220921.905908, 40467849.784654), paidunits.anal = c(74832038, 9839247383), revenue.anal = c(114808591.108702,0)), .Names = c("storeid", "paidunits.acc", "revenue.acc", "paidunits.anal","revenue.anal"), sorted = "storeid", class = c("data.table","data.frame"), row.names = c(NA, -2L)), key='storeid')
    suffixes <- c(".acc", ".anal")

    sortAllColsAlphabetically.exceptStartCols=FALSE
    sep.forSuffix="."

    DT[, ZZ_midCol2 := "middle2"]
    DT[, midCol1 := "middle1"]
    DT[, midCol3 := "middle3"]
    DT[, thirdCol := "third"]
    DT <- {set.seed(17); setcolorderpt(DT, sample(names(DT)))}
    (setcolorderpt(DT, c("midCol1", "thirdCol", "revenue.acc", "ZZ_midCol2", "paidunits.anal")))

    DT[, secondCol := "second"]
    DT[, bFIRST := "first_last"]
    DT[, fifthCol := "5"]
    DT[, aLAST := "last_last"]
    DT[, cSECOND := "second_last"]
    DT[, sixthCol := "6"]
    DT[, fourthCol := "4"]
    endCols <- c("bFIRST", "cSECOND", "aLAST")
    startCols <- c("storeid", "secondCol", "thirdCol", "fourthCol", "fifthCol", "sixthCol")
    suffixes <- c(".anal", ".acc")

    hr <- pasteR(50)
    cat("\n", hr, "\n\n\t\tBEFORE:   \n\n")
    print(DT)
    setcolorder_suffix(DT, suffix=suffixes, startCols=startCols, endCols=endCols)
    cat("\n", hr, "\n\n\t\tAFTER:   \n\n")
    print(DT)
}

# ------------------------------------------------------------------------------------- #


setcolorder_suffix <- function(DT, suffixes, startCols=key(DT), endCols=NULL, sortAllColsAlphabetically.exceptStartCols=FALSE, sep.forSuffix=".") {

  blank_suf <- "[]"

  if (!is.null(startCols) && any(startCols %ni% names(DT)))
    stop("Some cols in startCols are not in DT: ", pasteQand(setdiff(names(DT), startCols)))

  if (length(sep.forSuffix) > 1)
    stop ("Do not know how to proces sep.forSuffix larger than one item")

  if (!is.data.table(DT))
    stop ("DT must be a data.table")

  ## pattern for spliitng
  pat.sep    <- regOr(sep.forSuffix, escape=TRUE)

  ## if no suffixes are given, split the names and find them
  if (missing(suffixes))
    suffixes <- unique(unlist(sapply(strsplit(names(DT), pat.sep), function(x) if (length(x) > 1) tail(x, 1))))
  
  if (!length(suffixes)) {
    warning ("suffixes has no length (even after auto-search based on sep='", sep.forSuffix, "')\n  HINT: use setcolorderpt() for simple col ordering")
    ## TODO:  Have not tested, but the rest should work fine.  Otherwise call setcolorderpt(DT, .. )
  }

  suffixes   <- sub(pat.sep, "", suffixes)
  pat.suffix <- paste0(regOr(paste0(sep.forSuffix, suffixes), escape=TRUE), "$")

  suffixes_w_blank <- c(blank_suf, suffixes)


# middlepart: 
# sort all cols alphabetically
# sort suf_cols, but have them be last
# preserve order of suf_cols, but have them be last
# preserve order of all cols, but group suf_cols

  # ----------------- #
  nms <- copy(names(DT))
  DT.order <- data.table(nms=nms, origorder = seq(nms))
  DT.order[, has_suffix := grepl(pat.suffix, nms)]
  DT.order[!(has_suffix),  c("firstPart", "suf") := list(nms, blank_suf)]
  DT.order[ (has_suffix),  c("firstPart", "suf") := sapplyt(splitOnLast(nms, pat.sep), function(x) c(x[[1]], tail(x, 1))) ]
  DT.order[, suf := factor(suf, levels=suffixes_w_blank)]

  ## Identify if it is start, end or middle
  DT.order[, is_StartCol := nms %in% startCols]
  DT.order[, is_EndCol   := nms %in% endCols]
  DT.order[, is_middle := !(is_StartCol) & !(is_EndCol) ]

  ## Initialize the ordercol's
  DT.order[, paste0("ordercol", 1:4) := 0L]

  ## First ordering is based on start and end cols
  DT.order[, ordercol1 := 0L]
  DT.order[(is_StartCol), ordercol1 :=  as.integer((.N*2)-(match(nms, startCols)))]
  DT.order[(is_EndCol),   ordercol1 :=  as.integer(-match(nms, endCols)) ]


  ## ---------------------------------------------------------- ##
  if (FALSE && "preserve order of all cols, but group suf_cols") {
    # midCol1
    # revenue.anal
    # revenue.acc
    # ZZ_midCol2
    # paidunits.anal
    # paidunits.acc 
    # midCol3
    DT.order[ , ordercol2 := -seq(.N)]
    DT.order[ , ordercol2 := max(ordercol2), by=firstPart]
    DT.order[order(suf), ordercol3 := -.GRP, by=suf]
    ## SEE just middle
    DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)][(is_middle)]
    ## RETURN
    DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)]
  }
  ## ---------------------------------------------------------- ##
  if (FALSE && "preserve order of all cols, but alphabetize suf_cols at end") {

    # midCol1
    # ZZ_midCol2
    # midCol3
    # paidunits.anal     <~~~ [p]aidunits  before [r]evenue
    # paidunits.acc 
    # revenue.anal       <~~~ [r]evenue 
    # revenue.acc

    DT.order[(is_middle)]

    ## Bring all the non-suffix columns to front
    DT.order[, ordercol2 := 0L]
    DT.order[order(has_suffix, is_middle), ordercol2 := -.GRP, by=list(has_suffix, is_middle)]
    # DT.order[order(has_suffix, is_middle, origorder), ordercol2 := -seq(.N)]

    ## Organize all of the suffix columns: alphabetically, firspart then suffix
    DT.order[, ordercol3 := 0L]
    DT.order[order(has_suffix, suf, origorder), ordercol3 := -.GRP, by=list(firstPart, suf)]
    DT.order[(has_suffix), ordercol3 := ordercol3[-origorder == max(-origorder)], by=firstPart]

    ## Organize all of the suffix columns: alphabetically, firspart then suffix
    DT.order[, ordercol4 := 0L]
    DT.order[order(has_suffix, suf), ordercol4 := -seq(.N)]

    ## SEE just middle
    DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)][(is_middle)]
    ## RETURN
    DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)]
  }

  ## ---------------------------------------------------------- ##
  if (TRUE || "preserve order of all cols, moving suf_cols to end, but preserving their relative order") {
      # midCol1
      # ZZ_midCol2
      # midCol3
      # revenue.anal
      # revenue.acc
      # paidunits.anal
      # paidunits.acc 

      DT.order[, paste0("ordercol", 2:4) := 0L]

      ## First sort by has_suffix and orignal order
      ##  But taking any with same firstPart and bringing them together
      DT.order[order(is_middle, has_suffix, origorder), ordercol2 := -seq(.N)]
      DT.order[ , ordercol2 := max(ordercol2), by=firstPart]

      ## Next sort according to suffix.  No need for colorder4
      DT.order[order(suf), ordercol3 := -seq(.N)]

      ## SEE just middle
      DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)][(is_middle)]
      ## RETURN
      DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)]
  }


  nms.new_order <- DT.order[order(ordercol1, ordercol2, ordercol3, ordercol4, decreasing=TRUE)]$nms

  setcolorderpt(DT, nms.new_order)

  return(invisible(DT))
}


### OLD VERSION of setcolorder_suffix
##     
##     
##      nms <- sort(names(DT))
##      selfname_(nms)
##     
##      patSuffix <- function(suffix) 
##        paste0(escapeRegEx(suffix), "$")
##     
##      for (suffix in suffixes)
##        nms <- gsub(patSuffix(suffix), "", nms)
##     
##      if (sortAllColsAlphabetically.exceptStartCols) {
##        nms.ret <- c(startCols, sort(nms)[sort(nms) %ni% startCols])
##      ## Otherwise, sort only the duplicated columns, according to when they first appeared
##      } else {
##        for (nm in nms[duplicated(nms)]) {
##          wh <- which(nms == nm)
##          if (length(wh)) {
##     
##          }
##          nms[-wh]
##     
##        }
##     
##        &&& LEFT OFF HERE
##      }
##     }
##     
##     colNamesFromVector(nms.ret)
##