"# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #"
if (exists(".duplicate.sourcing.check", envir=globalenv()) && !identical(.duplicate.sourcing.check, FALSE)) {
  notifyAndEmail("in duplicate check the browser")
  warning("In Duplicate detected in file    ~/git/orch/src/MgmtReport/GL Automation - Mgmt Report Ingestion.r")
  # browser(text="duplicate detected in file")
} else {
  assign(".duplicate.sourcing.check", TRUE, envir=globalenv())
}

"# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #"


## RESET
# .RESET(width=262)

## .RESET should have loaded all utils.  Confirm
stopifnot(exists("setScience"))

{

  ## Flag to identify if currently forecasting
  thisRunIsAForecast <- !(!exists("fakeToday") || is.null(fakeToday))

  ## FLASGS SPECIFIC TO AUTOMATION
  {
    if (!exists("start_from_last_file"))
      start_from_last_file <- exists("files_actually_sourcing") && exists("last_file_ingested") && last_file_ingested != tail(files_actually_sourcing, 1)

    options("verbose.BackUpOrRestore"=FALSE)
    options("qry.dont.setkey"=TRUE)


    ## VERBOSE
    assignIfNotExist(verbose,         TRUE)

    ## Logical: Whether to skip the database query (ie, whether to re-pull the OA data)
    assignIfNotExist(skip_query,      FALSE)

    ## Logical: Whether to skip the ingestion of the GL file, specifically (but do ingest others)
    assignIfNotExist(skip_ingest_GL,  FALSE)

    ## Logical: Whether to skip the ingestion of all the XLSX files
    assignIfNotExist(skip_ingest,     FALSE)

    skip_forecast       <- FALSE  && !thisRunIsAForecast ## never, if forcasting
    skip_exportOfBudget <- FALSE  ||  thisRunIsAForecast ## always, if forecasting

    
    ## Logical: Whether to skip the export of the budget 
    skip_exportOfResults <-  FALSE  && !thisRunIsAForecast ## never, if forcasting

    ## Strings: File name patterns to skip
    skip.patterns <- c("new method", "z ADDITIONAL FROM")
    
    ## Integers: Part numbers to skip
    skip.parts    <-  c()

    parts_importing <- 0:2
  }

  ## Set environment
  {
    ## Clear any .bak 
    rm(list=lsos(pat="\\.bak", type="DT", showWarnings=FALSE)$Name)
    ## repull git 
    .g(verbose=FALSE)
    cls(35)
    ## set connection
    setScience(proj="MgmtReport", subl=FALSE, create=FALSE, quiet=TRUE, load=FALSE)
    setDBall()
  }

  ## Grab the files to import
  {
    files_to_source <- dir(srcDir, all=TRUE)
    nms.files_to_source <- basename(files_to_source)
    names(files_to_source) <- nms.files_to_source

    ## files_to_source to keep have this pattern
    pat.GLfiles <- "^GL_pt(\\d).*\\.r"

    ## identify which files_to_source we will keep, ignore the rest
    nms.files_to_source <- extract(pat.GLfiles, nms.files_to_source)

    files_to_source <- files_to_source[nms.files_to_source]    

    if (thisRunIsAForecast) {
      skip.patterns <- unique(c(skip.patterns, "GL_pt2_d", "GL_pt2_e", "GL_pt2_f", "GL_pt2_g", "GL_pt2_ZZ"))
    }

    ## REMOVE ANY files_to_source WHICH ARE FLAGGED TO BE SKIPPED
    {
      number_of_files_before_removing <- length(files_to_source)
      if ((skip_query) && !thisRunIsAForecast) {

        loadFromJesus("DT.acc",  over=FALSE, dont.fail.ifexists=TRUE, verbose=TRUE)
        loadFromJesus("DT.anal", over=FALSE, dont.fail.ifexists=TRUE, verbose=TRUE)

        files_to_source <- files_to_source[!grepl("GL_pt1_. Import", nms.files_to_source)]
        nms.files_to_source <- names(files_to_source)
      }

      if ((skip_ingest_GL || skip_ingest) && !thisRunIsAForecast) {

        loadFromJesus("DT.GL",         over=FALSE, dont.fail.ifexists=TRUE, verbose=TRUE)
        loadFromJesus("last_modified", over=FALSE, dont.fail.ifexists=TRUE, verbose=TRUE)

        files_to_source <- files_to_source[!grepl("GL_pt1_. Ingest GL", nms.files_to_source)]
        nms.files_to_source <- names(files_to_source)
      }


      if ((skip_ingest) && !thisRunIsAForecast) {
        loadFromJesus("DT.supplychain_percentages", over=FALSE, dont.fail.ifexists=TRUE, verbose=TRUE)
        files_to_source <- files_to_source[!grepl("GL_pt1_. Ingest", nms.files_to_source)]
        nms.files_to_source <- names(files_to_source)
      }

          
      if ((skip_exportOfResults) && !thisRunIsAForecast) {
        files_to_source <- files_to_source[files_to_source != "GL_pt2_ZZ Join Cols and Export.r"]
        nms.files_to_source <- names(files_to_source)
      }


      for (.pat in skip.patterns) {
        files_to_source <- files_to_source[!grepl(.pat, nms.files_to_source)]
        nms.files_to_source <- names(files_to_source)
        rm(.pat)
      }
    }

    ## split up into pts
    parts <- as.numeric(gsub(pat.GLfiles, "\\1", nms.files_to_source))

    ## Identify which files will actually get sourced
    files_actually_sourcing <- files_to_source[parts %in% setdiff(parts_importing, skip.parts) ]

    ## When dev'ing, we might want to restart from mid run
    if (start_from_last_file && exists("last_file_ingested")) {
      if (last_file_ingested == tail(files_actually_sourcing, 1))
        warning ("\n\n'last_files_ingested' is the last file in files_to_source.\nDid you mean to set  start_from_last_file=FALSE ")
      files_actually_sourcing <- files_actually_sourcing[seq(files_actually_sourcing) > match(last_file_ingested, files_actually_sourcing)]
      ## reset start_from_last_file
      start_from_last_file <- FALSE
    }

    if (verbose) {
      cat (sprintf("\n\n\t-- Sourcing %i files --\n(of %i identifed candidates, and %i total in folder)\n\n", length(files_actually_sourcing),  length(files_to_source), number_of_files_before_removing))
      Sys.sleep(2) ## Wait two seconds so user can read message
    }

    if (!length(files_actually_sourcing))
      stop ("There are no files to source (ie, none made it through the filters")
  }
  ## // end  Grab the files to import



  ### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ##

  ##  EXECUTE  ##
  {
    if (thisRunIsAForecast && exists("DT.acc") && exists("DT.anal") && identical(attr(DT.anal, "fakeToday"), fakeToday))
      files_actually_sourcing <- files_actually_sourcing[files_actually_sourcing %ni% "GL_pt1_a Import OA from DB Querying Aggregated Tables.r"]
    
    for (f in files_actually_sourcing ) 
    {
      message(sprintf("**  %s  **\n**\tImporting file:   %-56s %-16s**\n**  %1$s  **", pasteR(93), f, ifelse(thisRunIsAForecast, pasteQ(as.character(fakeToday), q="") ,"")))
      source(src.p(f))
      last_file_ingested <- f
    }
  } ## // end   Execute

}