using MySql.Data.MySqlClient; using Sony.Filtr.Database; using Sony.Filtr.Tasks.Models; using System.Collections.Generic; using System.Threading.Tasks; using Sony.Filtr.Utility.Extensions; using System; using System.Linq; using Serilog.Context; using System.Threading; using Serilog; using System.IO; using Sony.Filtr.Utility; using Serilog.Formatting.Json; namespace Sony.Filtr.Tasks { public class HealthcheckManager { private List jobIds; private List jobIdsDontAlarmMissedStart; public HealthcheckManager() { } public HealthcheckManager(string jobIds, string jobIdsDontAlarm) { if (string.IsNullOrEmpty(jobIds)) { this.jobIds = GetTaskIds().Result; } else { this.jobIds = ParseStringArrayToIntArray(jobIds); } if (string.IsNullOrEmpty(jobIdsDontAlarm)) { jobIdsDontAlarmMissedStart = new List(); } else { this.jobIdsDontAlarmMissedStart = ParseStringArrayToIntArray(jobIdsDontAlarm); } ConfigureDataDog(); } private static void ConfigureDataDog() { var logFileName = "log.json"; if (File.Exists(logFileName)) { File.Delete(logFileName); } Serilog.Log.Logger = new LoggerConfiguration().Enrich.FromLogContext() .WriteTo.File(new JsonFormatter(renderMessage: true), logFileName) .CreateLogger(); } public async Task> GetScheduledStatistics() { var i = 0; List statistics = new List(); foreach (var id in jobIds) { var statisticsForSpecificTask = await GetStatisticsForSpecificTask(id, DateTime.Now.AddDays(-60), DateTime.Now.AddDays(1)); var firstItem = statisticsForSpecificTask.OrderByDescending(t => t.StartTime).FirstOrDefault(); if (firstItem == null) { var defaultStatistic = (await GeDefaultTaskInformation(id)).FirstOrDefault(); if (defaultStatistic == null) { Serilog.Log.Logger.Information($"TaskID:{id}, IsTaskRunning: {false}"); continue; } using (LogContext.PushProperty("JobId", defaultStatistic.TaskId)) using (LogContext.PushProperty("JobAlias", defaultStatistic.TaskName)) using (LogContext.PushProperty("JobStartTime", DateTime.Now)) { Serilog.Log.Logger.Information($"IsTaskRunning: {false}"); } continue; } var runs = new List(); foreach (var item in statisticsForSpecificTask) { runs.Add(new RunItem() { StartTime = item.StartTime, EndTime = item.EndTime, IsError = item.IsError, ErrorMessage = item.ErrorMessage }); var isFreshEvent = item.StartTime.Value.AddMinutes(2) > DateTime.UtcNow; using (LogContext.PushProperty("JobId", item.TaskId)) using (LogContext.PushProperty("JobAlias", item.TaskName)) using (LogContext.PushProperty("JobStartTime", item.StartTime)) { if (item.IsError && isFreshEvent) { Serilog.Log.Logger.Error($"IsError: True"); } if (item.ErrorMessage != null && isFreshEvent) { Serilog.Log.Logger.Error($"ErrorMessage: {item.ErrorMessage}"); } if (item.StartTime != null && isFreshEvent) { Serilog.Log.Logger.Information($"Task started"); } if (item.EndTime != null && isFreshEvent) { Serilog.Log.Logger.Information($"Task finished"); } } } var runStatistics = new TaskRunStatistics() { Id = firstItem.TaskId, TaskName = firstItem.TaskName, Runs = runs, UnfinishedTaskRun = await UnfinishedRun(runs), }; var isTaskRunning = runStatistics.UnfinishedTaskRun != null ? true : false; runStatistics.IsTaskRunning = isTaskRunning; i++; var isNeedAlarm = IsNeedAlarmMissedTaskLaunch(statisticsForSpecificTask, jobIdsDontAlarmMissedStart); using (LogContext.PushProperty("JobId", firstItem.TaskId)) using (LogContext.PushProperty("JobAlias", firstItem.TaskName)) using (LogContext.PushProperty("JobStartTime", firstItem.StartTime)) { Serilog.Log.Logger.Information($"IsTaskRunning: {isTaskRunning}"); if (isNeedAlarm.Item1) { Serilog.Log.Logger.Error($"Task launch was missed. Please take a look. Task should be run at: {isNeedAlarm.Item2}"); } } statistics.Add(runStatistics); } return null; } private Tuple IsNeedAlarmMissedTaskLaunch(List statisticsForSpecificTask, List jobIdsDontAlarm) { if (jobIdsDontAlarmMissedStart.Contains(statisticsForSpecificTask.First().TaskId)) { return new Tuple(false, DateTime.UtcNow); } if (statisticsForSpecificTask.Any(x=> x.EndTime == null && x.StartTime.Value.AddMinutes(30) > DateTime.UtcNow)) { return new Tuple(false, DateTime.UtcNow); } var todayStatistics = statisticsForSpecificTask.Where(x => x.StartTime.Value.Date == DateTime.UtcNow.Date); if (todayStatistics.Count() != 0 && todayStatistics.OrderByDescending(x => x.StartTime).First().EndTime == null) { return new Tuple(false, DateTime.UtcNow); } bool isNeedToRunToday = false; int alreadyProcessedRunsToday = 0; var dayOfWeekToday = DateTime.UtcNow.DayOfWeek; var dateToday = DateTime.UtcNow.Date; var lastRunToday = DateTime.UtcNow.Date.AddDays(-1); Dictionary> datesByDateOfWeekProcessed = new Dictionary>(); foreach (var item in statisticsForSpecificTask) { var closure = item; if (closure.StartTime.Value.Date == dateToday) { alreadyProcessedRunsToday++; if (closure.StartTime.Value > lastRunToday) { lastRunToday = closure.StartTime.Value; } } if (closure.StartTime.Value.DayOfWeek == dayOfWeekToday) { var list = datesByDateOfWeekProcessed.GetValueOrDefault(closure.StartTime.Value.Date); if (list != null) { list.Add(closure.StartTime.Value); if (list.Count > 1) { isNeedToRunToday = true; } } else { list = new List(); list.Add(closure.StartTime.Value); var t = closure.StartTime.Value.Date; datesByDateOfWeekProcessed.Add(t, list); } } } List listToCalculatePeriodic = new List(); if (isNeedToRunToday) { foreach (var item in datesByDateOfWeekProcessed.Where(x=>x.Key.Date!=DateTime.UtcNow.Date)) { var closure = item; var tempProcessed = alreadyProcessedRunsToday; if (closure.Value.Count() <= tempProcessed) { continue; } var previousWeekRun = datesByDateOfWeekProcessed.Where(x => x.Key.Date == DateTime.UtcNow.Date.AddDays(-7)).FirstOrDefault(); if (previousWeekRun.Value != null && previousWeekRun.Value.Count() > 12 && alreadyProcessedRunsToday > 2) { return new Tuple(false, DateTime.UtcNow); } var itemToAdd = closure.Value.OrderBy(i => i).ToList()[tempProcessed]; if (itemToAdd.AddMinutes(30).TimeOfDay > lastRunToday.TimeOfDay) { listToCalculatePeriodic.Add(itemToAdd); } } if (listToCalculatePeriodic.Count() == 0) { return new Tuple(false, DateTime.UtcNow); } var threshold =((double)listToCalculatePeriodic.Count() / (double)datesByDateOfWeekProcessed.Where(x => x.Key.Date != DateTime.UtcNow.Date).Count()); var isFreshDates = true; foreach (var item in listToCalculatePeriodic.OrderByDescending(x=>x).Take(3)) { if (item.Date < DateTime.UtcNow.Date.AddDays(-21)) { isFreshDates = false; } } if (listToCalculatePeriodic.Count() < 2 && threshold <= 0.25 && !isFreshDates) { return new Tuple(false, DateTime.UtcNow); } if (isFreshDates) { listToCalculatePeriodic = listToCalculatePeriodic.OrderByDescending(x => x).Take(3).ToList(); } var approximateTimeToday = CalculateAvarageStartTime(listToCalculatePeriodic); TimeSpan ts = new TimeSpan(approximateTimeToday.Hour, approximateTimeToday.Minute, approximateTimeToday.Second); DateTime date = DateTime.UtcNow.Date + ts; if (date < lastRunToday.AddMinutes(15) && date > lastRunToday.AddMinutes(-15)) { return new Tuple(false, DateTime.UtcNow); } if (date.AddMinutes(30) < DateTime.UtcNow) { return new Tuple(true, date); } } return new Tuple(false, DateTime.UtcNow); } private DateTime CalculateAvarageStartTime(List dates) { var count = dates.Count; double temp = 0D; for (int i = 0; i < count; i++) { temp += dates[i].TimeOfDay.Ticks / (double)count; } var average = new DateTime((long)temp); return average; } private List ParseStringArrayToIntArray(string numberStr) { string[] splitted = numberStr.Split(','); int[] nums = new int[splitted.Length]; for (int i = 0; i < splitted.Length; i++) { nums[i] = int.Parse(splitted[i]); } return nums.ToList(); } public async Task UnfinishedRun(List runs) { var unfinishedTaskRun = runs.Where(x => x.EndTime == null && x.StartTime.Value.Date == DateTime.UtcNow.Date && x.StartTime == runs.Select(z => z.StartTime).Max()); if (unfinishedTaskRun.Count() == 0) { return null; } else { var unfinishedRun = new UnfinishedRun(); unfinishedRun.StartDate = (DateTime)unfinishedTaskRun.Select(x => x.StartTime).Max(); var estimatedFinish = CalculateEstimatedFinish(runs, unfinishedRun.StartDate); if (estimatedFinish == null) { return unfinishedRun; } unfinishedRun.EstimatedFinish = estimatedFinish; long tempPercentage; try { tempPercentage = Convert.ToInt64((DateTime.UtcNow - unfinishedRun.StartDate).TotalMilliseconds / ((TimeSpan)(estimatedFinish - unfinishedRun.StartDate)).TotalMilliseconds * 100); } catch { tempPercentage = 100; } unfinishedRun.Percentage = tempPercentage > 100 ? 100 : tempPercentage; return unfinishedRun; } } public DateTime? CalculateEstimatedFinish(List runs, DateTime startDateOfUnfinishedRun) { Dictionary> orderedRuns = new Dictionary>(); var distinctDates = runs.Select(x => x.StartTime.Value.Date).Distinct().ToList(); foreach (var date in distinctDates) { var runsSpecificDate = runs.Where(x => x.StartTime.Value.Date == date).OrderBy(x => x.StartTime).ToList(); for (var i = 0; i < runsSpecificDate.Count(); i++) { List runItemsList; orderedRuns.TryGetValue(i + 1, out runItemsList); if (runItemsList == null) { runItemsList = new List(); orderedRuns.Add(i + 1, runItemsList); } runItemsList.Add(runsSpecificDate[i]); } } var unfinishedRunPosition = GetRunOrderPerDay(runs, startDateOfUnfinishedRun); List listToCalculateAverageTime; do { listToCalculateAverageTime = orderedRuns.GetValueOrDefault(unfinishedRunPosition); unfinishedRunPosition--; var temp = listToCalculateAverageTime.Where(c => c.EndTime != null).Count() == 0; } while (unfinishedRunPosition > 0 && listToCalculateAverageTime != null && listToCalculateAverageTime.Where(c => c.EndTime != null).Count() == 0); if (listToCalculateAverageTime == null) { return null; } var calculateAverageExecutionTime = CalculateAverageExecutionTime(listToCalculateAverageTime); startDateOfUnfinishedRun = startDateOfUnfinishedRun.AddMilliseconds(calculateAverageExecutionTime); return startDateOfUnfinishedRun; } public double CalculateAverageExecutionTime(List runs) { List runsInMilliseconds = new List(); var counter = 0; double runSum = 0; foreach (var run in runs) { if (run.StartTime != null && run.EndTime != null) { var executionTimeInMilliseconds = (run.EndTime - run.StartTime).Value.TotalMilliseconds; runSum += executionTimeInMilliseconds; counter++; } } var average = runSum / (counter == 0 ? 1 : counter); return average; } public int GetRunOrderPerDay(List runs, DateTime startDateOfUnfinishedRun) { var runsPerDay = runs.Where(x => x.StartTime.Value.Date == startDateOfUnfinishedRun.Date).OrderBy(x => x.StartTime).ToList(); int i = 1; var finalPosition = i; foreach (var item in runsPerDay) { if (item.EndTime == null) { finalPosition = i; } i++; } return finalPosition; } public async Task> GetScheduledTaskRunStatistics() { List statistics = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { using (var cmd = new MySqlCommand(@"SELECT stl.ScheduledTaskId AS TaskId, st.Name as TaskName, stl.Started AS StartTime, stl.Finished AS EndTime, stl.Error AS IsError, stl.ErrorMessage AS ErrorMessage FROM tblScheduledTaskLog AS stl INNER JOIN tblScheduledTask AS st ON stl.ScheduledTaskId = st.Id ORDER BY Started DESC", conn)) { // cmd.Parameters.AddWithValue("@scheduledTaskName", scheduledTaskName); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { statistics.Add(new TaskRunStatisticsDTO(reader)); } } } } return statistics; } public async Task> GetStatisticsForSpecificTask(int taskId, DateTime startDate, DateTime endDate) { List statistics = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { using (var cmd = new MySqlCommand(@"SELECT stl.ScheduledTaskId AS TaskId, st.Name as TaskName, stl.Started AS StartTime, stl.Finished AS EndTime, stl.Error AS IsError, stl.ErrorMessage AS ErrorMessage FROM tblScheduledTaskLog AS stl INNER JOIN tblScheduledTask AS st ON stl.ScheduledTaskId = st.Id WHERE stl.ScheduledTaskId=@scheduledTaskId AND Started > @startDate AND Started <= @endDate ORDER BY Started DESC", conn)) { cmd.Parameters.AddWithValue("@scheduledTaskId", taskId); cmd.Parameters.AddWithValue("@startDate", startDate); cmd.Parameters.AddWithValue("@endDate", endDate); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { statistics.Add(new TaskRunStatisticsDTO(reader)); } } } } return statistics; } public async Task> GeDefaultTaskInformation(int taskId) { List statistics = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { using (var cmd = new MySqlCommand(@"SELECT stl.ScheduledTaskId AS TaskId, st.Name as TaskName, stl.Started AS StartTime, stl.Finished AS EndTime, stl.Error AS IsError, stl.ErrorMessage AS ErrorMessage FROM tblScheduledTaskLog AS stl INNER JOIN tblScheduledTask AS st ON stl.ScheduledTaskId = st.Id WHERE stl.ScheduledTaskId=@scheduledTaskId LIMIT 1", conn)) { cmd.Parameters.AddWithValue("@scheduledTaskId", taskId); using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { statistics.Add(new TaskRunStatisticsDTO(reader)); } } } } return statistics; } public async Task> GetTaskIds() { List ids = new List(); using (var conn = await DatabaseHandler.GetOpenReadOnlyConnectionAsync()) { using (var cmd = new MySqlCommand("SELECT Id FROM tblScheduledTask GROUP by Id", conn)) { using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { ids.Add(reader.GetInt32("Id")); } } } } return ids; } } }