# process featured playlists files - part 02.r

---------------------------------------------------------------------------------------
NOTE TO SELF: 
---------------------------------------------------------------------------------------
Process Part 02
    After all of the JSON has been converted to CSV and saved to disk
    Ingest the CSV and send them to snowflake
---------------------------------------------------------------------------------------


.us()
options("snowflake_inuse" = TRUE)
if (!exists("projName") || projName != "Spotify Playlist Webscrape")
  source("~/git/orch/src/Spotify Playlist Webscrape/00 Featured Playlist Setup.r")


csv_subfolder_for_featuredplaylists <- "featured_playlists_parsed_csv"
csvfiles_featuredplaylists <- extractFilesFromFolder(folder=data.p(csv_subfolder_for_featuredplaylists), pattern="^featuredPlaylists", ext="csv", full.names=TRUE, recursive=TRUE)
DT.featured_playlists <- csvfiles_featuredplaylists %>% lapply(fread, sep=",") %>% rbindlist


tbl <- "featured_playlists"
schema <- "dev.spotify"
comment <- 'Webscrape of Spotify Featured Playlists by datetime and country'

colnamesToChange <- c(datetime_featured_UTC = "timestamp")

# if ("timestamp" %in% names(DT.featured_playlists))
#   setnames(DT.featured_playlists, "timestamp", "datetime_featured_UTC")

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# saveImageTo()
# loadImageOf()

debugOn("ingestIntoSQL")
options("snowflake_inuse" = TRUE)
ret <- ingestIntoSQL(DT.featured_playlists, tbl=tbl, nmsToChange=colnamesToChange, schema=schema, comment=comment, diststyle=NULL, sortkey=NULL
                  , quoteBools=TRUE, boolean_type="VARCHAR(1)", datetime_type="TIMESTAMP_NTZ", upperColnames=FALSE, quoteColnames=FALSE, drop=TRUE, maxBytes_for_insertStatement=as.integer(1e5 / 3))


qry_create_insert <- makeSQLtable(DT.featured_playlists, table.name=tbl, schema=schema, comment=comment, diststyle=NULL, boolean_type="INT", datetime_type="TIMESTAMP_NTZ", upperColnames=FALSE, quoteColnames=FALSE)