# credits.starting_usage <- c(
#            looker =  9000
#       , analytics =  1500
#  , tech_engineers =   700
# )

credits.starting_usage <- c(
           looker =  13100
      , analytics =   2050
 , tech_engineers =   1050
)


credits.monthly_rate_for_linear_growth <- c(
           looker =  0.5 / 12
      , analytics =  0.5 / 10
 , tech_engineers =  0.5 / 13
)


## Starting Sizes for tables in exponential growth
starting_sizes_TB.expo_growth <- c(
    applemusic_raw = 0.4  ## Need to double check

  , youtube_analytics = 0.81
  , bi.analytics_track_level = 1.66
  , bi.analytics_release_level = 0.75

  , production.staging_raw_spotify_v2 =  2.45
  , production.fact_analytics =  1.98
  , production.fact_analytics_error =  0.35
  , production.dim_tables =  0.009
  , Other_Expo_Data = 4.3
  , New_Data_expo = 3.1
)

## Starting Sizes for tables in linear growth
starting_sizes_TB.linear_growth <- c(
    bi.accounting = 0.52
  , production.fact_sales =  0.28
  , production.staging_fact_analytics =  0.09
  , itunes_raw = 1.1  ## Need to double check
  , shazam = 0.8
  , spot_stream_counts = 0.52
  , spot_user_counts = 1.52
  , DEV_DATA = 2.85
  , Other_Linear_Data = 3.5
  , New_Data_linear = 1.2
)



## Starting Sizes for tables in exponential growth
include_in_vertica.expo_growth <- c(
    applemusic_raw = TRUE  ## Need to double check

  , youtube_analytics = TRUE
  , bi.analytics_track_level = FALSE
  , bi.analytics_release_level = FALSE

  , production.staging_raw_spotify_v2 = TRUE
  , production.fact_analytics = TRUE
  , production.fact_analytics_error = TRUE
  , production.dim_tables = TRUE
  , Other_Expo_Data = TRUE
  , New_Data_expo = TRUE
)

## Starting Sizes for tables in linear growth
include_in_vertica.linear_growth <- c(
    bi.accounting = TRUE
  , production.fact_sales = TRUE
  , production.staging_fact_analytics = TRUE
  , itunes_raw = TRUE  ## Need to double check
  , shazam = TRUE
  , spot_stream_counts = FALSE
  , spot_user_counts = FALSE
  , DEV_DATA = FALSE
  , Other_Linear_Data = TRUE
  , New_Data_linear = TRUE
)



# ==================================================================================

## Starting Sizes for tables in exponential growth
months_to_double.expo_growth <- c(
    applemusic_raw = 13  ## Need to double check

  , youtube_analytics = 7
  , bi.analytics_track_level = 9
  , bi.analytics_release_level = 12

  , production.staging_raw_spotify_v2 =  8
  , production.fact_analytics =  10
  , production.fact_analytics_error =  10
  , production.dim_tables =  10
  , Other_Expo_Data = 12
  , New_Data_expo = 12
)

## Starting Sizes for tables in linear growth
monthly_rate.linear_growth <- c(
    bi.accounting = 0.08
  , production.fact_sales =  0.08
  , production.staging_fact_analytics =  0.04
  , itunes_raw = 0.10  ## Need to double check
  , shazam = 0.04
  , spot_stream_counts = 0.13
  , spot_user_counts = 0.15
  , DEV_DATA = 0.25
  , Other_Linear_Data = 0.1
  , New_Data_linear = 0.1
)


# ==================================================================================

sizes_of_all_tables <- function(refresh=FALSE, overwrite.ifexists_and_loading_from_disk=FALSE) {
    if (.Pfm == "Darwin" || !refresh) {
      loadFromJesus("DT.tbls_in_snowflake", dont.fail.ifexists=TRUE, overwrite.ifexists=overwrite.ifexists_and_loading_from_disk)
    } else {
      DT.tbls_in_snowflake <- sfShowTables(all_dbnames=TRUE, cleanBytes=FALSE)
      jesusForData(DT.tbls_in_snowflake)
    }

    DT.tbls_in_snowflake

    ## Clean names
    setnames(DT.tbls_in_snowflake, c("database_name", "schema_name", "name"), c("dbname", "schema", "tbl"))

    ## Setkey
    setkeyIfNot(DT.tbls_in_snowflake, dbname, schema, tbl, organize=TRUE, verbose=FALSE)

    ## Drop "old_as_of"  tables
    DT.tbls_in_snowflake <- DT.tbls_in_snowflake[!grepl("old_asof_2", tbl)]

    ## Clean comments
    DT.tbls_in_snowflake[, comment := ifelse(nchar(comment)>99, {substr(comment, 1, 97) %>% paste0("...")}, comment)]

    ## LOOK AT SOME COUNTS
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes))), by=list(dbname, schema)][order(dbname, bytes, decreasing=TRUE)]
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes))), by=list(dbname)][order(dbname, bytes, decreasing=TRUE)]
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes))), by=schema][order(bytes, decreasing=TRUE)]
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes)))][order(bytes, decreasing=TRUE)]

    ## MORE COUNTS
    DT.tbls_in_snowflake[dbname != "PROD"][bytes > 1000000]
    DT.tbls_in_snowflake[dbname == "BACKUPS"][bytes > 1000000]
    DT.tbls_in_snowflake[(schema == "BI")][order(bytes, decreasing=TRUE)]

    ## MORE COUNTS
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes))), by=list(dbname)][order(bytes, decreasing=TRUE)]
    DT.tbls_in_snowflake[, list(bytes=sum(bytes), bytes_str=formatBytes(sum(bytes))), by=list(dbname)][order(bytes, decreasing=TRUE)]

    return(DT.tbls_in_snowflake)
}