## We want to identify any data that is in OA that is not in the GL DT.split_percs DT.mgmtr_summary_tall ## Some stores are apeparing in OA BEFORE they appear in the GL ## Here is an example of something causing an error. DT.acc[store_name == "Boomkat"] DT.mgmtr_summary_tall[store_name == "Boomkat"] DT.compare[store_name == "Boomkat"] matchKey(DT.acc, DT.mgmtr_summary_tall, kCols.main) DT.compare <- merge( DT.acc [ , list(gross.oa=sum(gross, na.rm=TRUE), inOA=TRUE), keyby=kCols.main] , DT.mgmtr_summary_tall [!is.na(storeid), list(gross.gl=sum(gross, na.rm=TRUE), inGL=TRUE), keyby=kCols.main] , all=TRUE ) # matchKey(DT.split_percs, DT.mgmtr_summary_tall, kCols.main) # DT.compare <- merge( DT.split_percs [ , list(gross.gl=sum(OA_gross_forDSOM, na.rm=TRUE), inGL=TRUE), keyby=kCols.main] # , DT.mgmtr_summary_tall [!is.na(storeid), list(gross.oa=sum(gross, na.rm=TRUE), inOA=TRUE), keyby=kCols.main] # , all=TRUE # ) ## If these are NA, then they should be FALSE since not in the respective data DT.compare[is.na(inOA), inOA := FALSE] DT.compare[is.na(inGL), inGL := FALSE] ## Drop any stores that NEVER appear in OA. These have not been properly matched yet setkeyIfNot(DT.compare, kCols.store) DT.compare <- DT.compare[!DT.compare[ , all(!inGL), keyby=kCols.store][(V1)]] ## clear suppressWarnings( { colsToClear <- c("first_date_store_is_in_GL", "last_date_store_is_in_GL", "times_missing_from_GL", "Number_of_times_store_appears_in_GL_for_2013_2014", "Number_of_times_store_appears_in_OA_for_2013_2014") DT.compare[, (colsToClear) := NULL] }) ## Grab the min / max date by store for GL suppressWarnings( { DT.compare[, first_date_store_is_in_GL := min(date[inGL], na.rm=TRUE), by=kCols.store] DT.compare[, last_date_store_is_in_GL := max(date[inGL], na.rm=TRUE), by=kCols.store] }) if (!any(DT.compare[, inOA & !inGL & (gross.oa != 0 | is.na(gross.oa))])) warning ("There are no offenders that in OA but not in GL") DT.compare[(date >= "2013-01-01" & (gross.gl != 0 | is.na(gross.gl))), Number_of_times_store_appears_in_GL_for_2013_2014 := lunique(date[inGL]), by=kCols.store] DT.compare[(date >= "2013-01-01" & (gross.oa != 0 | is.na(gross.oa))), Number_of_times_store_appears_in_OA_for_2013_2014 := lunique(date[inOA]), by=kCols.store] ## For those rows in OA but not in GL, consider them "missing" if the date is strictly between the min and max ## also, make sure gross is none-zero DT.compare[(inOA & !inGL & (gross.oa != 0 | is.na(gross.oa))), times_missing_from_GL := {lunique(date[(date > first_date_store_is_in_GL) & (date < last_date_store_is_in_GL)])}, by=kCols.store] stopifnot(!is.null(DT.compare[["times_missing_from_GL"]])) if (!any(DT.compare[, removeNA(times_missing_from_GL) > 0])) warning ("There are no missing values from GL") DT.missing_from_GL <- unique(DT.compare[(times_missing_from_GL > 0) & (gross.oa != 0), list(times_in_GL=Number_of_times_store_appears_in_GL_for_2013_2014, times_in_OA=Number_of_times_store_appears_in_OA_for_2013_2014, times_missing_from_GL, dates_missing=pasteC(date, C=",")), by=kCols.store]) ## HAve a look at the example of "Leap Frog" DT.compare[gross.oa != 0 & (date > first_date_store_is_in_GL) & storeid==500] DT.missing_from_GL DT.missing.dates <- as.data.table(DT.compare[(missing_from_GL)] [, table(store_name, date)]) }, by=storeid] stopifnot(DT.compare[, !any(inOA & !inGL)]) warning("There are ", DT.compare[(inOA & !inGL), lunique(storeid)], " unique stores that have at least one month where they appear in OA but are missing in GL" ## show the high gross values DT.compare[, diff_in_gross := gross.gl - gross.oa] DT.compare[, perc_diff_in_gross := percOf(gross.oa, outOf=gross.gl)] DT.compare[, perc_diff_in_gross_by_store := percOf(sum(gross.oa, na.rm=TRUE), outOf=sum(gross.gl, na.rm=TRUE)), by=storeid] DT.compare[, mean(perc_diff_in_gross ), by=storeid] DT.compare[, mean(perc_diff_in_gross_by_store ), by=storeid] DT.compare[, mean(perc_diff_in_gross_by_store ), by=storeid][abs(V1) > 1]