# Move Files to S3 2015-08-01.r

screen -xRR S3_move
R

if (FALSE)
source("~/git/orch/src/Spotify Playlist Webscrape/00 Featured Playlist Setup.r")

setScience(proj="Spotify Playlist Webscrape", subProj="S3Move", load=FALSE)



### ------------- MOVING FILES TO S3  -------------------- ###
###  2015-08-19
THIS STILL NEEDS WORK
### ------------- MOVING FILES TO ALT FOLDER (on external HDD) USING RSYNC  -------------------- ###

&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      ## move these two folders
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      bucket_base <- s3_p("spotify", "web_scrape_project")
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      folders_to_move <- c("parsed_json", "scraped_files")
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      for (fldr_t_m in folders_to_move) {
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        folder <- out.p(fldr_t_m)
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        bucket <- s3_p(bucket_base, fldr_t_m, warn_on_as.path=FALSE)
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        add_slash <- . %>% ifelse (!grepl("/$", .), paste0(., "/"), .)
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        folder %<>% add_slash %>% paste0("*")
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        bucket %<>% add_slash
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::        # s3_upload(file.local=folder, bucket=bucket, recursive=TRUE, verbose=TRUE)
&&& ALMOST READY -- FIX SUBFOLDER DESTINATION &&& ::      }


### ------------- MOVING FILES TO ALT FOLDER (on external HDD) USING RSYNC  -------------------- ###
###  2015-08-19
###  This moves the files to the the /data/ volume using rsync.  Then deletes them once they have been moved ###
### ------------- MOVING FILES TO ALT FOLDER (on external HDD) USING RSYNC  -------------------- ###


## EXTRA setScience for when manually triggering
setScience(proj="Spotify Playlist Webscrape", subProj="S3Move", load=FALSE)

subFolder <- "scraped_files"

outAlt <- as.path("/data/rick_large/gitData/orch/out/", getProjName())

from <- out.p(subFolder)
  to <- as.path(outAlt, subFolder)

## Confirm first child is in from and to
first_child <- "api.spotify.com"
stopifnot(first_child %in% dir(from),   first_child %in% dir(to))

## Ultimately, to should look like this. 
##   "/data/rick_large/gitData/orch/out/Spotify Playlist Webscrape/scraped_files"
stopifnot(path.expand(to) == "/data/rick_large/gitData/orch/out/Spotify Playlist Webscrape/scraped_files")

## Create the cmd;  Don't check_exists if running on local machine.
##  I run this on local machine when remote is full and I just need to create the cmd
##   rsync -a -v --ignore-existing --remove-source-files ~/git/orch/out/Spotify\ Playlist\ Webscrape/scraped_files/ /data/rick_large/gitData/orch/out/Spotify\ Playlist\ Webscrape/scraped_files
cmd <- make_rsync_to_copy_files(from=from, to=to, remove_source_after_copy=TRUE, check_exists={.Pfm!="Darwin"})
# cmd <- make_rsync_to_copy_files(from=paste0(from, "/api.spotify.com/v1/users/11*"), to=paste0(to, "/api.spotify.com/v1/users/"), add_trailing_slash=FALSE, remove_source_after_copy=TRUE)


catnn(cmd, "")
system(cmd)