# Datadog configs for creating custom monitors data "aws_secretsmanager_secret" "datadog_api" { name = "${var.environment}/datadog/DD_API_KEY" } data "aws_secretsmanager_secret_version" "datadog_api" { secret_id = data.aws_secretsmanager_secret.datadog_api.id } data "aws_secretsmanager_secret" "datadog_app_key" { name = "${var.environment}/datadog/DD_APP_KEY" } data "aws_secretsmanager_secret_version" "datadog_app_key" { secret_id = data.aws_secretsmanager_secret.datadog_app_key.id } data "datadog_role" "datadog_admin_role" { filter = "Datadog Admin Role" } # Terraform 0.13+ uses the Terraform Registry: terraform { backend "s3" { bucket = "orcd-terraform-state" key = "prod/frontend-moneyhub/datadog/terraform.tfstate" region = "us-east-1" encrypt = "true" } required_providers { datadog = { source = "DataDog/datadog" } } } resource "datadog_monitor" "error_monitor" { name = "🚨Frontend Monitor: Errors Detected" type = "error-tracking alert" message = "${var.error_monitor_message} \n\n ${var.notification_alerts_endpoint}" query = "error-tracking-rum(\"@application.id:3305504d-227a-4d39-8283-f3751e0f5aa9 ${var.error_monitor_exclusions}\").rollup(\"count\").by(\"@issue.id\").last(\"5m\") > 10" monitor_thresholds { critical = 10 warning = 6 } enable_logs_sample = false notify_audit = false on_missing_data = "default" include_tags = true renotify_interval = 30 renotify_statuses = [ "alert" ] escalation_message = "${var.error_monitor_escalation_message} ${var.notification_main_endpoint}" priority = 3 tags = [ "env:${var.environment}", "service_name:${var.service_name}", "application_family:${var.application_family}" ] } resource "datadog_dashboard_json" "customer_accounting_dashboard" { dashboard = <", "value": 4000000000 }, { "palette": "yellow_on_white", "comparator": ">", "value": 2500000000 }, { "palette": "green_on_white", "comparator": "<=", "value": 2500000000 } ], "formula": "query3" }, { "alias": "Errors", "cell_display_mode": "number", "conditional_formats": [ { "palette": "red_on_white", "comparator": ">", "value": 300000000 }, { "palette": "yellow_on_white", "comparator": ">", "value": 100000000 }, { "palette": "green_on_white", "comparator": "<=", "value": 100000000 } ], "formula": "query4" }, { "alias": "Error Rate", "number_format": { "unit": { "type": "canonical_unit", "unit_name": "percent" } }, "conditional_formats": [ { "comparator": ">", "value": 10, "palette": "white_on_red" }, { "comparator": ">", "value": 5, "palette": "black_on_light_yellow" } ], "formula": "query4 / query1 * 100" } ] } ], "has_search_bar": "never", "custom_links": [ { "link": "/dash/integration/rum_errors_browser?tpl_var_viewName={{@view.name.value}}", "label": "Analyze errors for {{@view.name.value}}" }, { "link": "/dash/integration/rum_resource_browser_and_mobile?tpl_var_viewName={{@view.name.value}}", "label": "View resources for {{@view.name.value}}" } ] }, "layout": { "x": 0, "y": 0, "width": 8, "height": 6 } }, { "id": 3793458078010682, "definition": { "title": "Average Request Rate for OwsMoneyhub", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:trace.fastapi.request.hits{service:ows-moneyhub}.as_rate()", "aggregator": "avg" } ], "formulas": [ { "formula": "query1" } ] } ], "autoscale": true, "precision": 2, "timeseries_background": { "type": "area" } }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 1947051727358847, "definition": { "title": "Largest Contentful Paint P75", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "formulas": [ { "formula": "query2" } ], "queries": [ { "data_source": "rum", "name": "query2", "indexes": [ "*" ], "compute": { "aggregation": "pc75", "metric": "@view.largest_contentful_paint" }, "group_by": [], "search": { "query": "@type:view @session.type:user service:moneyhub" } } ], "conditional_formats": [ { "palette": "black_on_light_red", "value": 4000000000, "comparator": ">" }, { "palette": "black_on_light_yellow", "value": 2500000000, "comparator": ">" }, { "palette": "black_on_light_green", "value": 2500000000, "comparator": "<=" } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "timeseries_background": { "yaxis": { "include_zero": true }, "type": "area" } }, "layout": { "x": 8, "y": 2, "width": 4, "height": 2 } }, { "id": 5045799819969654, "definition": { "title": "Average View Load Time", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "formulas": [ { "formula": "query2" } ], "queries": [ { "data_source": "rum", "name": "query2", "indexes": [ "*" ], "compute": { "aggregation": "avg", "metric": "@view.loading_time" }, "group_by": [], "search": { "query": "@session.type:user service:moneyhub @type:view" } } ], "conditional_formats": [ { "palette": "black_on_light_red", "value": 4000000000, "comparator": ">" }, { "palette": "black_on_light_yellow", "value": 2500000000, "comparator": ">" }, { "palette": "black_on_light_green", "value": 2500000000, "comparator": "<=" } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "timeseries_background": { "yaxis": { "include_zero": true }, "type": "area" } }, "layout": { "x": 8, "y": 4, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 0, "width": 12, "height": 9 } }, { "id": 768525539368952, "definition": { "title": "User Analytics", "background_color": "vivid_blue", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 6536184716757834, "definition": { "title": "Views by Country", "type": "geomap", "requests": [ { "formulas": [ { "formula": "query1", "limit": { "count": 100, "order": "desc" } } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [ { "facet": "@geo.country_iso_code", "limit": 100, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "@type:view service:moneyhub" } } ] } ], "style": { "palette": "hostmap_blues", "palette_flip": false }, "view": { "focus": "WORLD" } }, "layout": { "x": 0, "y": 0, "width": 7, "height": 4 } }, { "id": 6441147331661948, "definition": { "type": "note", "content": "Take note of any large discrepancies between views from the current week and the previous week. If necessary, use the global time picker to go back in time to check if this trend persists.\n", "background_color": "blue", "font_size": "14", "text_align": "left", "vertical_align": "top", "show_tick": true, "tick_pos": "50%", "tick_edge": "left", "has_padding": true }, "layout": { "x": 7, "y": 0, "width": 5, "height": 2 } }, { "id": 539205467652780, "definition": { "title": "Top Actions", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "formulas": [ { "formula": "query1", "limit": { "count": 10, "order": "desc" } } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [ { "facet": "@action.name", "limit": 10, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "service:moneyhub" } } ] } ], "style": {} }, "layout": { "x": 7, "y": 2, "width": 5, "height": 3 } }, { "id": 3209279074905470, "definition": { "title": "Views of Moneyhub", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "formulas": [ { "formula": "query1" } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "cardinality", "metric": "@view.id" }, "group_by": [], "search": { "query": "@type:view env:prod service:moneyhub" } } ] } ], "autoscale": true, "precision": 2, "timeseries_background": { "type": "bars", "yaxis": {} } }, "layout": { "x": 0, "y": 4, "width": 3, "height": 2 } }, { "id": 6096803175835848, "definition": { "title": "Time spent on App", "title_size": "16", "title_align": "left", "show_legend": false, "type": "distribution", "xaxis": { "max": "auto", "scale": "linear", "min": "auto", "include_zero": true }, "yaxis": { "max": "auto", "scale": "linear", "min": "auto", "include_zero": true }, "requests": [ { "request_type": "histogram", "query": { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "avg", "metric": "@view.time_spent" }, "group_by": [], "search": { "query": "service:moneyhub @type:view" } }, "style": { "palette": "dog_classic" } } ] }, "layout": { "x": 3, "y": 4, "width": 4, "height": 2 } }, { "id": 6713111765023805, "definition": { "title": "Most Visited Pages", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "formulas": [ { "formula": "query1", "limit": { "count": 10, "order": "desc" } } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [ { "facet": "@view.name", "limit": 10, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "@type:view @session.type:user @application.id:3305504d-227a-4d39-8283-f3751e0f5aa9" } } ] } ], "style": {} }, "layout": { "x": 7, "y": 5, "width": 5, "height": 5 } }, { "id": 4095589301239190, "definition": { "title": "Avg. View Time", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "formulas": [ { "formula": "query1" } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "avg", "metric": "@view.time_spent" }, "group_by": [], "search": { "query": "service:moneyhub env:prod @type:view" } } ] } ], "autoscale": true, "precision": 2 }, "layout": { "x": 0, "y": 6, "width": 2, "height": 2 } }, { "id": 237284585241778, "definition": { "title": "Views Over Time VS Week Before", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "on_right_yaxis": false, "formulas": [ { "alias": "this week", "formula": "query1" } ], "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "cardinality", "metric": "@view.id" }, "group_by": [], "search": { "query": "service:moneyhub @type:view" } } ], "response_format": "timeseries", "style": { "palette": "cool", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" }, { "on_right_yaxis": false, "formulas": [ { "alias": "week before", "formula": "week_before(query0)" } ], "queries": [ { "data_source": "rum", "name": "query0", "indexes": [ "*" ], "compute": { "aggregation": "cardinality", "metric": "@view.id" }, "group_by": [], "search": { "query": "service:moneyhub @type:view" } } ], "response_format": "timeseries", "style": { "palette": "grey", "line_type": "dotted", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear", "label": "", "min": "auto", "max": "auto" }, "markers": [] }, "layout": { "x": 2, "y": 6, "width": 5, "height": 2 } }, { "id": 5063137881132240, "definition": { "title": "Which pages do people come from?", "title_size": "16", "title_align": "left", "requests": [ { "formulas": [ { "formula": "query1" } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [ { "facet": "@view.referrer", "limit": 10, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "@type:view service:moneyhub" } } ] } ], "type": "sunburst" }, "layout": { "x": 0, "y": 8, "width": 7, "height": 4 } }, { "id": 4258418418767120, "definition": { "title": "Unique Visitors", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "formulas": [ { "formula": "query3" } ], "queries": [ { "data_source": "rum", "name": "query3", "indexes": [ "*" ], "compute": { "aggregation": "cardinality", "metric": "@usr.id" }, "group_by": [], "search": { "query": "@type:session @session.type:user service:moneyhub" } } ], "response_format": "scalar" } ], "autoscale": true, "precision": 2, "timeseries_background": { "type": "area" } }, "layout": { "x": 7, "y": 10, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, "y": 9, "width": 12, "height": 13 } }, { "id": 2770215097801196, "definition": { "title": "Error Analytics", "title_align": "left", "background_color": "vivid_pink", "show_title": true, "type": "group", "layout_type": "ordered", "widgets": [ { "id": 2915757084255944, "definition": { "title": "Error rate on Frontend-Moneyhub", "title_size": "16", "title_align": "left", "show_legend": false, "legend_layout": "auto", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "formulas": [ { "alias": "% views with errors", "formula": "(1 - (query5 - query7) / query5) * 100" } ], "queries": [ { "data_source": "rum", "name": "query5", "indexes": [ "*" ], "compute": { "aggregation": "count", "interval": 1800000 }, "group_by": [], "search": { "query": "@type:view service:moneyhub env:prod" } }, { "data_source": "rum", "name": "query7", "indexes": [ "*" ], "compute": { "aggregation": "count", "interval": 1800000 }, "group_by": [], "search": { "query": "@type:view @view.error.count:>0 service:moneyhub env:prod" } } ], "response_format": "timeseries", "style": { "palette": "red", "line_type": "solid", "line_width": "normal" }, "display_type": "line" } ], "yaxis": { "include_zero": true, "scale": "linear", "label": "", "min": "auto", "max": "auto" }, "markers": [] }, "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 457200343594786, "definition": { "title": "Code Errors on Frontend-Moneyhub", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "horizontal", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "on_right_yaxis": false, "formulas": [ { "alias": "Last week code errors", "formula": "week_before(query1)" } ], "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [], "search": { "query": "@type:error -@error.source:network service:moneyhub env:prod" } } ], "response_format": "timeseries", "style": { "palette": "grey", "line_type": "dotted", "line_width": "normal" }, "display_type": "line" }, { "on_right_yaxis": false, "formulas": [ { "alias": "Code errors", "formula": "query1" } ], "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [], "search": { "query": "@type:error -@error.source:network service:moneyhub env:prod" } } ], "response_format": "timeseries", "style": { "palette": "red", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear", "label": "", "min": "auto", "max": "auto" }, "markers": [] }, "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 4148724916971746, "definition": { "title": "500 errors on Frontend Moneyhub", "title_size": "16", "title_align": "left", "show_legend": true, "legend_layout": "horizontal", "legend_columns": [ "avg", "min", "max", "value", "sum" ], "type": "timeseries", "requests": [ { "on_right_yaxis": false, "formulas": [ { "alias": "Last week network errors", "formula": "week_before(query2)" } ], "queries": [ { "data_source": "rum", "name": "query2", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [], "search": { "query": "@resource.status_code:>=500 service:moneyhub env:prod" } } ], "response_format": "timeseries", "style": { "palette": "grey", "line_type": "dotted", "line_width": "normal" }, "display_type": "line" }, { "on_right_yaxis": false, "formulas": [ { "alias": "Network errors", "formula": "query1" } ], "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count" }, "group_by": [], "search": { "query": "@resource.status_code:>=500 service:moneyhub env:prod" } } ], "response_format": "timeseries", "style": { "palette": "red", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear", "label": "", "min": "auto", "max": "auto" }, "markers": [] }, "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 4738644932713550, "definition": { "title": "Issues on Frontend-Moneyhub", "title_size": "16", "title_align": "left", "requests": [ { "query": { "query_string": "service:moneyhub", "data_source": "rum_issue_stream" }, "columns": [ { "field": "details", "width": "auto" }, { "field": "count", "width": "auto" }, { "field": "volume", "width": "auto" } ], "response_format": "event_list" } ], "type": "list_stream" }, "layout": { "x": 0, "y": 2, "width": 8, "height": 3 } }, { "id": 6942271192215317, "definition": { "title": "Top Views with Highest Error Rate", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "formulas": [ { "alias": "% of Views with Errors", "formula": "(1 - (query1 - query2) / query1) * 100", "limit": { "count": 10, "order": "desc" } } ], "conditional_formats": [ { "palette": "white_on_red", "value": 0, "comparator": ">" } ], "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "count", "interval": 3600000 }, "group_by": [ { "facet": "@view.name", "limit": 250, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "@type:view @session.type:user service:moneyhub env:prod" } }, { "data_source": "rum", "name": "query2", "indexes": [ "*" ], "compute": { "aggregation": "count", "interval": 3600000 }, "group_by": [ { "facet": "@view.name", "limit": 250, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "@type:view @view.error.count:>0 @session.type:user service:moneyhub env:prod" } } ] } ], "style": {} }, "layout": { "x": 8, "y": 2, "width": 4, "height": 4 } }, { "id": 1778354570340202, "definition": { "title": "Top Actions Triggering Errors by Number of Impacted Users", "title_size": "16", "title_align": "left", "type": "query_table", "requests": [ { "response_format": "scalar", "queries": [ { "data_source": "rum", "name": "query1", "indexes": [ "*" ], "compute": { "aggregation": "cardinality", "metric": "@session.id" }, "group_by": [ { "facet": "@action.name", "limit": 30, "sort": { "order": "desc", "aggregation": "cardinality", "metric": "@session.id" } }, { "facet": "@view.url_path_group", "limit": 30, "sort": { "order": "desc", "aggregation": "cardinality", "metric": "@session.id" } } ], "search": { "query": "@action.error.count:>0 service:moneyhub env:prod @type:action" } }, { "data_source": "rum", "name": "query2", "indexes": [ "*" ], "compute": { "aggregation": "avg", "metric": "@action.error.count" }, "group_by": [ { "facet": "@action.name", "limit": 1000, "sort": { "order": "desc", "aggregation": "cardinality", "metric": "@session.id" } } ], "search": { "query": "@action.error.count:>0 service:moneyhub env:prod @type:action" } } ], "formulas": [ { "cell_display_mode": "bar", "formula": "query1", "limit": { "count": 900, "order": "desc" } }, { "cell_display_mode": "bar", "formula": "query2" } ] } ], "has_search_bar": "never" }, "layout": { "x": 0, "y": 5, "width": 8, "height": 3 } }, { "id": 6758913485293692, "definition": { "title": "Average Latency of OwsMoneyhub", "title_size": "16", "title_align": "left", "type": "query_value", "requests": [ { "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:trace.fastapi.request.duration{service:ows-moneyhub}", "aggregator": "avg" } ], "conditional_formats": [ { "comparator": ">", "value": 3, "palette": "custom_bg", "custom_bg_color": "#ffc2c2" }, { "comparator": ">", "value": 2, "palette": "black_on_light_yellow" }, { "comparator": "<", "value": 2, "palette": "white_on_green" } ], "formulas": [ { "formula": "query1" } ] } ], "autoscale": true, "precision": 2, "timeseries_background": { "type": "area" } }, "layout": { "x": 8, "y": 6, "width": 4, "height": 2 } }, { "id": 1986181082862768, "definition": { "title": "OwsMoneyhub Resource Error Logs", "title_size": "16", "title_align": "left", "requests": [ { "response_format": "event_list", "query": { "data_source": "trace_stream", "query_string": "@_top_level:1 service:ows-moneyhub @http.status_code:(NOT 2**) env:prod", "indexes": [] }, "columns": [ { "field": "timestamp", "width": "auto" }, { "field": "service", "width": "auto" }, { "field": "resource_name", "width": "auto" }, { "field": "@duration", "width": "auto" }, { "field": "@http.method", "width": "auto" }, { "field": "@http.status_code", "width": "auto" }, { "field": "span_count", "width": "auto" }, { "field": "latency_breakdown", "width": "auto" } ] } ], "type": "list_stream" }, "layout": { "x": 0, "y": 8, "width": 12, "height": 5 } }, { "id": 7994204465000926, "definition": { "title": "Lambdas Error Count By Error Type", "title_size": "16", "title_align": "left", "requests": [ { "response_format": "scalar", "formulas": [ { "formula": "query1", "limit": { "count": 10, "order": "desc" } } ], "queries": [ { "data_source": "spans", "name": "query1", "indexes": [ "trace-search" ], "compute": { "aggregation": "count" }, "group_by": [ { "facet": "error.type", "limit": 10, "sort": { "order": "desc", "aggregation": "count" } } ], "search": { "query": "env:prod service:(lambda-moneyhub-generate-attachments OR lambda-moneyhub-custom-reports) status:error" } } ], "style": { "palette": "classic" } } ], "type": "sunburst", "legend": { "type": "automatic" } }, "layout": { "x": 0, "y": 13, "width": 6, "height": 4 } }, { "id": 5565029038579354, "definition": { "title": "Customer Accounting Monitors", "type": "manage_status", "display_format": "countsAndList", "color_preference": "background", "hide_zero_counts": true, "query": "moneyhub", "sort": "status,asc", "count": 50, "start": 0, "summary_type": "monitors", "show_priority": false, "show_last_triggered": false }, "layout": { "x": 6, "y": 13, "width": 6, "height": 8 } }, { "id": 8900097405043848, "definition": { "title": "OwsMoneyhub Errors Grouped By Resource ", "type": "treemap", "requests": [ { "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", "query": "sum:trace.fastapi.request.errors{service:ows-moneyhub} by {resource_name}.as_count()", "aggregator": "sum" } ], "style": { "palette": "semantic" }, "formulas": [ { "formula": "query1" } ] } ] }, "layout": { "x": 0, "y": 17, "width": 6, "height": 4 } } ] }, "layout": { "x": 0, "y": 0, "width": 12, "height": 22, "is_column_break": true } } ], "template_variables": [], "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" } EOF }