# DumpTables_to_S3.r

setScience(proj="MgmtReport", subProj="DumpTables", load=FALSE, subl=FALSE)
getSubProj()

setPrompt(full=TRUE)
TESTRUN <- FALSE
schemas <- c("production", "bi", "spotify") %>% selfname_
delim="\v"
manifest=FALSE
gzip=TRUE



cluster <- getOption("db.defaultcluster.out")
ll.tables <- schemas %>% lapply(., qShowTables, row=FALSE, includePerm=TRUE, cluster=cluster)

for (schema in schemas) {
  for (tbl in ll.tables[[schema]]$name) {
    cat (tbl, " - ", schema == "production" && tbl <= 'dim_track_sync', "\n")
}}



sinkOn()
stamp=timeStamp()
for (schema in schemas) {
  for (tbl in ll.tables[[schema]]$name) {
    
    if (schema == "production" &&  which(ll.tables[[schema]]$name == tbl) <= 36) {
      cat("Skipping '", tbl, "'\n")
      next;
    }

    bucket <- s3_p("table_dumps", sprintf("cluster%02i", cluster), schema, tbl, stamp, prefix=tbl)
    qry <- makeQry(tbl=tbl, schema=schema, colsToPull="*", expandStar=FALSE, aggFunc=NULL, limit=if(TESTRUN) 8)
    stbl <- schemaPaste(schema=schema, tbl=tbl)
    SelectAndUnload(qry, tbl=stbl, bucket=bucket, delim=delim, manifest=manifest, gzip=gzip, check_bucket_exists=FALSE, cluster=cluster, verbose.showSettings=FALSE, notify=FALSE)
    if (!(s3_ls(bucket, verbose=FALSE) %>% length) > 1)
      warning(stbl, " may not have unloaded correctly", call.=FALSE)
  }
}

SelectAndUnload

"n";