Skip to content

Commit 3eab1e8

Browse files
authored
refactor(app/outbound): remove status counters from NewRecordResponse (#4300)
**nb:** this branch is based upon #4299, and #4298. in #4299 we made some prepatory adjustments to the outbound proxy's route-backend metrics layer, and in #4298 we introduced a new `linker-http-prom::status` metrics layer that can be used to count response status codes, in a manner that is agnostic with respect to the particular protocol that instrumented traffic is using. this branch performs a sequences of changes oriented towards two concrete goals: **1:** integrate the `NewRecordStatusCode` middleware into the outbound proxy's route and backend metrics layers, and **2:** remove status code measurement from the `NewRecordResponse` middleware. it's worth stating explicitly that this maintains the existing behavior of the outbound proxy, and that (1) and (2) must be performed at the same time to maintain parity. <img width="498" height="262" alt="image" src="https://github.com/user-attachments/assets/5030e7cd-33df-4cc3-b5cc-5f4db8247bd5" /> to demonstrate that parity is maintained, let's compare metrics from this branch against a snapshot of metrics exported in `main`. **🟰 comparing the metrics** metrics were scraped from the proxy, using a small traffic generation deployment that runs `curl` against a container running `http-echo`. a snapshot of the route and backend status/duration counters on main: ``` # HELP outbound_http_route_request_duration_seconds The time between request initialization and response completion. # TYPE outbound_http_route_request_duration_seconds histogram # UNIT outbound_http_route_request_duration_seconds seconds outbound_http_route_request_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 0.5594713199999999 outbound_http_route_request_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 outbound_http_route_request_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 outbound_http_route_request_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 outbound_http_route_request_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 outbound_http_route_request_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 outbound_http_route_request_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 284 # HELP outbound_http_route_request_statuses Completed request-response streams. # TYPE outbound_http_route_request_statuses counter outbound_http_route_request_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname="",http_status="200",error=""} 284 # HELP outbound_http_route_backend_response_duration_seconds The time between request completion and response completion. # TYPE outbound_http_route_backend_response_duration_seconds histogram # UNIT outbound_http_route_backend_response_duration_seconds seconds outbound_http_route_backend_response_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 0.4626854930000003 outbound_http_route_backend_response_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="0.025",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="0.1",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="0.25",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 outbound_http_route_backend_response_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 284 # HELP outbound_http_route_backend_response_statuses Completed responses. # TYPE outbound_http_route_backend_response_statuses counter outbound_http_route_backend_response_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name="",http_status="200",error=""} 284 ``` this was repeated after applying `config.linkerd.io/proxy-version` to use a proxy with these patches applied. a snapshot of the route and backend status/duration counters with these changes: ``` # HELP outbound_http_route_request_duration_seconds The time between request initialization and response completion. # TYPE outbound_http_route_request_duration_seconds histogram # UNIT outbound_http_route_request_duration_seconds seconds outbound_http_route_request_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 0.2557158190000001 outbound_http_route_request_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 outbound_http_route_request_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 outbound_http_route_request_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 outbound_http_route_request_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 outbound_http_route_request_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 outbound_http_route_request_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 128 # HELP outbound_http_route_request_statuses Completed request-response streams. # TYPE outbound_http_route_request_statuses counter outbound_http_route_request_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname="",http_status="200",error=""} 128 # HELP outbound_http_route_backend_response_duration_seconds The time between request completion and response completion. # TYPE outbound_http_route_backend_response_duration_seconds histogram # UNIT outbound_http_route_backend_response_duration_seconds seconds outbound_http_route_backend_response_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 0.209469089 outbound_http_route_backend_response_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="0.025",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="0.1",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="0.25",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 outbound_http_route_backend_response_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 128 # HELP outbound_http_route_backend_response_statuses Completed responses. # TYPE outbound_http_route_backend_response_statuses counter outbound_http_route_backend_response_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name="",http_status="200",error=""} 128 ``` because there are a large number of metrics and labels, a character-level diff of the two looked like so. as we'd hope, the only differences observed were values of particular time series. in other words, the same traffic yielded the same number of time series with the same labels. ``` diff --git a/recorded.txt b/recorded.txt index 41d4c53..5bf5c13 100644 --- a/recorded.txt +++ b/recorded.txt @@ -1,29 +1,29 @@ # HELP outbound_http_route_request_duration_seconds The time between request initialization and response completion. # TYPE outbound_http_route_request_duration_seconds histogram # UNIT outbound_http_route_request_duration_seconds seconds outbound_http_route_request_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} 0.{+2+}55[-94-]71[-3-]{+58+}19[-9999999-]{+0000001+} outbound_http_route_request_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] outbound_http_route_request_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] outbound_http_route_request_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] outbound_http_route_request_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] outbound_http_route_request_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] outbound_http_route_request_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname=""} {+1+}28[-4-] # HELP outbound_http_route_request_statuses Completed request-response streams. # TYPE outbound_http_route_request_statuses counter outbound_http_route_request_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",hostname="",http_status="200",error=""} {+1+}28[-4-] # HELP outbound_http_route_backend_response_duration_seconds The time between request completion and response completion. # TYPE outbound_http_route_backend_response_duration_seconds histogram # UNIT outbound_http_route_backend_response_duration_seconds seconds outbound_http_route_backend_response_duration_seconds_sum{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} 0.[-46-]2[-685-]{+09+}4{+6+}9[-30-]0[-00003-]{+89+} outbound_http_route_backend_response_duration_seconds_count{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="0.025",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="0.05",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="0.1",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="0.25",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="0.5",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="1.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="10.0",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] outbound_http_route_backend_response_duration_seconds_bucket{le="+Inf",parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name=""} {+1+}28[-4-] # HELP outbound_http_route_backend_response_statuses Completed responses. # TYPE outbound_http_route_backend_response_statuses counter outbound_http_route_backend_response_statuses_total{parent_group="core",parent_kind="Service",parent_namespace="simple-app",parent_name="simple-app-v1",parent_port="80",parent_section_name="",route_group="",route_kind="default",route_namespace="",route_name="http",backend_group="core",backend_kind="Service",backend_namespace="simple-app",backend_name="simple-app-v1",backend_port="80",backend_section_name="",http_status="200",error=""} {+1+}28[-4-] ``` --- * feat(app/outbound): add status counters to route metrics Signed-off-by: katelyn martin <[email protected]> * feat(app/outbound): add status counters to backend metrics Signed-off-by: katelyn martin <[email protected]> * feat(app/outbound): add route status counting middleware Signed-off-by: katelyn martin <[email protected]> * feat(app/outbound): add backend status counting middleware Signed-off-by: katelyn martin <[email protected]> * refactor(app/outbound): remove status counter from duration middleware Signed-off-by: katelyn martin <[email protected]> * nit(http/prom): nitpick a todo comment Signed-off-by: katelyn martin <[email protected]> --------- Signed-off-by: katelyn martin <[email protected]>
1 parent 337c296 commit 3eab1e8

File tree

8 files changed

+207
-183
lines changed

8 files changed

+207
-183
lines changed

linkerd/app/outbound/src/http/logical/policy/route.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,12 @@ where
135135
// Set request extensions based on the route configuration
136136
// AND/OR headers
137137
.push(extensions::NewSetExtensions::layer())
138-
.push(metrics::layer(&metrics.requests, &metrics.body_data))
138+
.push(metrics::layer(
139+
&metrics.requests,
140+
&metrics.statuses,
141+
&metrics.body_data,
142+
))
143+
.push_on_service(crate::http::BoxResponse::layer())
139144
.check_new::<Self>()
140145
.check_new_service::<Self, http::Request<http::BoxBody>>()
141146
// Configure a classifier to use in the endpoint stack.

linkerd/app/outbound/src/http/logical/policy/route/backend/metrics.rs

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use linkerd_app_core::{metrics::prom, svc};
33
use linkerd_http_prom::{
44
body_data::response::{BodyDataMetrics, ResponseBodyFamilies},
55
count_reqs::{RequestCount, RequestCountFamilies},
6-
record_response,
6+
record_response, status,
77
stream_label::{LabelSet, StreamLabel},
88
};
99

@@ -22,15 +22,14 @@ where
2222
{
2323
requests: RequestCountFamilies<labels::RouteBackend>,
2424
responses: ResponseMetrics<L>,
25+
statuses: status::StatusMetrics<L::StatusLabels>,
2526
body_metrics: ResponseBodyFamilies<labels::RouteBackend>,
2627
}
2728

28-
type ResponseMetrics<L> = record_response::ResponseMetrics<
29-
<L as StreamLabel>::DurationLabels,
30-
<L as StreamLabel>::StatusLabels,
31-
>;
29+
type ResponseMetrics<L> = record_response::ResponseMetrics<<L as StreamLabel>::DurationLabels>;
3230

33-
type Instrumented<T, N> = NewRecordBodyData<NewCountRequests<NewResponseDuration<T, N>>>;
31+
type Instrumented<T, N> =
32+
NewRecordBodyData<NewCountRequests<NewRecordStatusCode<T, NewResponseDuration<T, N>>>>;
3433
type NewRecordBodyData<N> =
3534
linkerd_http_prom::body_data::response::NewRecordBodyData<ExtractRecordBodyDataParams, N>;
3635
type NewCountRequests<N> = linkerd_http_prom::count_reqs::NewCountRequests<ExtractRequestCount, N>;
@@ -52,6 +51,7 @@ where
5251
let RouteBackendMetrics {
5352
requests,
5453
responses,
54+
statuses,
5555
body_metrics,
5656
} = metrics.clone();
5757

@@ -60,10 +60,11 @@ where
6060

6161
let record = NewRecordDuration::layer_via(ExtractRecordDurationParams(responses.clone()));
6262
let count = NewCountRequests::layer_via(ExtractRequestCount(requests.clone()));
63+
let status = NewRecordStatusCode::layer_via(ExtractStatusCodeParams::new(statuses.clone()));
6364
let body_data =
6465
NewRecordBodyData::layer_via(ExtractRecordBodyDataParams(body_metrics.clone()));
6566

66-
body_data.layer(count.layer(record.layer(inner)))
67+
body_data.layer(count.layer(status.layer(record.layer(inner))))
6768
})
6869
}
6970

@@ -84,10 +85,15 @@ where
8485
pub fn register(reg: &mut prom::Registry, histo: impl IntoIterator<Item = f64>) -> Self {
8586
let requests = RequestCountFamilies::register(reg);
8687
let responses = record_response::ResponseMetrics::register(reg, histo);
88+
let statuses = status::StatusMetrics::register(
89+
reg.sub_registry_with_prefix("response"),
90+
"Completed responses",
91+
);
8792
let body_metrics = ResponseBodyFamilies::register(reg);
8893
Self {
8994
requests,
9095
responses,
96+
statuses,
9197
body_metrics,
9298
}
9399
}
@@ -104,7 +110,7 @@ where
104110

105111
#[cfg(test)]
106112
pub(crate) fn get_statuses(&self, l: &L::StatusLabels) -> prom::Counter {
107-
self.responses.get_statuses(l)
113+
self.statuses.metric(l)
108114
}
109115

110116
#[cfg(test)]
@@ -126,6 +132,7 @@ where
126132
Self {
127133
requests: Default::default(),
128134
responses: Default::default(),
135+
statuses: Default::default(),
129136
body_metrics: Default::default(),
130137
}
131138
}
@@ -141,6 +148,7 @@ where
141148
Self {
142149
requests: self.requests.clone(),
143150
responses: self.responses.clone(),
151+
statuses: self.statuses.clone(),
144152
body_metrics: self.body_metrics.clone(),
145153
}
146154
}

linkerd/app/outbound/src/http/logical/policy/route/metrics.rs

Lines changed: 58 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use linkerd_app_core::{
66
};
77
use linkerd_http_prom::{
88
body_data::request::{BodyDataMetrics, NewRecordBodyData, RequestBodyFamilies},
9-
record_response,
9+
record_response, status,
1010
stream_label::{
1111
error::LabelError,
1212
status::{LabelGrpcStatus, LabelHttpStatus},
@@ -22,10 +22,7 @@ pub(super) mod test_util;
2222
#[cfg(test)]
2323
mod tests;
2424

25-
pub type RequestMetrics<R> = record_response::RequestMetrics<
26-
<R as StreamLabel>::DurationLabels,
27-
<R as StreamLabel>::StatusLabels,
28-
>;
25+
pub type RequestMetrics<R> = record_response::RequestMetrics<<R as StreamLabel>::DurationLabels>;
2926

3027
#[derive(Debug)]
3128
pub struct RouteMetrics<R, B>
@@ -39,6 +36,7 @@ where
3936
{
4037
pub(super) retry: retry::RouteRetryMetrics,
4138
pub(super) requests: RequestMetrics<R>,
39+
pub(super) statuses: status::StatusMetrics<R::StatusLabels>,
4240
pub(super) backend: backend::RouteBackendMetrics<B>,
4341
pub(super) body_data: RequestBodyFamilies<labels::Route>,
4442
}
@@ -90,27 +88,46 @@ pub struct ExtractBodyDataMetrics<X> {
9088
extract: X,
9189
}
9290

91+
/// An `N`-typed [`NewService<T>`] with status code metrics.
92+
pub type NewRecordStatusCode<T, N> = status::NewRecordStatusCode<
93+
N,
94+
ExtractStatusCodeParams<<T as MkStreamLabel>::StatusLabels>,
95+
T,
96+
<T as MkStreamLabel>::StatusLabels,
97+
>;
98+
99+
/// [`NewRecordStatusCode<T, N>`] parameters.
100+
type StatusCodeParams<T> = status::Params<T, <T as MkStreamLabel>::StatusLabels>;
101+
102+
/// Extracts parameters for request status code metrics.
103+
#[derive(Clone, Debug)]
104+
pub struct ExtractStatusCodeParams<L>(status::StatusMetrics<L>);
105+
93106
pub fn layer<T, N>(
94107
metrics: &RequestMetrics<T::StreamLabel>,
108+
statuses: &status::StatusMetrics<T::StatusLabels>,
95109
body_data: &RequestBodyFamilies<labels::Route>,
96110
) -> impl svc::Layer<
97111
N,
98112
Service = NewRecordBodyData<
99-
NewRecordDuration<T, RequestMetrics<T::StreamLabel>, N>,
113+
NewRecordStatusCode<T, NewRecordDuration<T, RequestMetrics<T::StreamLabel>, N>>,
100114
ExtractBodyDataParams,
101115
ExtractBodyDataMetrics<T>,
102116
>,
103117
>
104118
where
119+
N: svc::NewService<T>,
105120
T: Clone + MkStreamLabel,
106121
T: svc::ExtractParam<labels::Route, http::Request<http::BoxBody>>,
122+
T::StatusLabels: Clone,
107123
{
108124
let record = NewRecordDuration::layer_via(ExtractRecordDurationParams(metrics.clone()));
125+
let status = NewRecordStatusCode::layer_via(ExtractStatusCodeParams(statuses.clone()));
109126
let body_data = NewRecordBodyData::layer_via(ExtractBodyDataParams(body_data.clone()));
110127

111128
svc::layer::mk(move |inner| {
112129
use svc::Layer;
113-
body_data.layer(record.layer(inner))
130+
body_data.layer(status.layer(record.layer(inner)))
114131
})
115132
}
116133

@@ -153,6 +170,7 @@ where
153170
Self {
154171
requests: Default::default(),
155172
backend: Default::default(),
173+
statuses: Default::default(),
156174
retry: Default::default(),
157175
body_data: Default::default(),
158176
}
@@ -172,6 +190,7 @@ where
172190
Self {
173191
requests: self.requests.clone(),
174192
backend: self.backend.clone(),
193+
statuses: self.statuses.clone(),
175194
retry: self.retry.clone(),
176195
body_data: self.body_data.clone(),
177196
}
@@ -195,11 +214,16 @@ where
195214
Self::RESPONSE_BUCKETS.iter().copied(),
196215
);
197216

217+
let statuses = status::StatusMetrics::register(
218+
reg.sub_registry_with_prefix("request"),
219+
"Completed request-response streams",
220+
);
198221
let retry = retry::RouteRetryMetrics::register(reg.sub_registry_with_prefix("retry"));
199222
let body_data = RequestBodyFamilies::register(reg);
200223

201224
Self {
202225
requests,
226+
statuses,
203227
backend,
204228
retry,
205229
body_data,
@@ -257,6 +281,33 @@ where
257281
}
258282
}
259283

284+
// === impl ExtractStatusCodeParams ===
285+
286+
impl<L> ExtractStatusCodeParams<L> {
287+
pub fn new(metrics: status::StatusMetrics<L>) -> Self {
288+
Self(metrics)
289+
}
290+
}
291+
292+
impl<L, T> svc::ExtractParam<StatusCodeParams<T>, T> for ExtractStatusCodeParams<L>
293+
where
294+
T: Clone,
295+
T: MkStreamLabel<StatusLabels = L>,
296+
L: Clone,
297+
{
298+
fn extract_param(&self, target: &T) -> StatusCodeParams<T> {
299+
let Self(metrics) = self;
300+
301+
let mk_stream_label = target.clone();
302+
let metrics = metrics.clone();
303+
304+
StatusCodeParams {
305+
mk_stream_label,
306+
metrics,
307+
}
308+
}
309+
}
310+
260311
// === impl LabelHttpRsp ===
261312

262313
impl<P> From<P> for LabelHttpRsp<P> {

0 commit comments

Comments
 (0)