Skip to content

Commit c6048fa

Browse files
danielblandodanielblando
authored
Add users stats http api to ingester (#6178)
* Add users stats http api to ingester Signed-off-by: Daniel Deluiggi <[email protected]> * Changelog Signed-off-by: Daniel Deluiggi <[email protected]> * Change name for loadedBlocks Signed-off-by: Daniel Deluiggi <[email protected]> --------- Signed-off-by: Daniel Deluiggi <[email protected]> Signed-off-by: Daniel Blando <[email protected]>
1 parent 15ad4de commit c6048fa

File tree

10 files changed

+326
-136
lines changed

10 files changed

+326
-136
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* [ENHANCEMENT] Ruler: Add new ruler metric `cortex_ruler_rule_groups_in_store` that is the total rule groups per tenant in store, which can be used to compare with `cortex_prometheus_rule_group_rules` to count the number of rule groups that are not loaded by a ruler. #5869
66
* [ENHANCEMENT] Ingester/Ring: New `READONLY` status on ring to be used by Ingester. New ingester API to change mode of ingester #6163
77
* [ENHANCEMENT] Ruler: Add query statistics metrics when --ruler.query-stats-enabled=true. #6173
8+
* [ENHANCEMENT] Ingester: Add new API `/ingester/all_user_stats` which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178
89
* [ENHANCEMENT] Distributor: Add new `cortex_reduced_resolution_histogram_samples_total` metric to to track the number of histogram samples which resolution was reduced. #6182
910

1011
## 1.18.0 in progress

‎docs/api/_index.md‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ For the sake of clarity, in this document we have grouped API endpoints by servi
3232
| [Flush blocks](#flush-blocks) | Ingester || `GET,POST /ingester/flush` |
3333
| [Shutdown](#shutdown) | Ingester || `GET,POST /ingester/shutdown` |
3434
| [Ingesters ring status](#ingesters-ring-status) | Ingester || `GET /ingester/ring` |
35+
| [Ingester tenants stats](#ingester-tenants-stats) | Ingester || `GET /ingester/all_user_stats` |
3536
| [Ingester mode](#ingester-mode) | Ingester || `GET,POST /ingester/mode` |
3637
| [Instant query](#instant-query) | Querier, Query-frontend || `GET,POST <prometheus-http-prefix>/api/v1/query` |
3738
| [Range query](#range-query) | Querier, Query-frontend || `GET,POST <prometheus-http-prefix>/api/v1/query_range` |
@@ -242,7 +243,7 @@ GET /distributor/all_user_stats
242243
GET /all_user_stats
243244
```
244245

245-
Displays a web page with per-tenant statistics updated in realtime, including the total number of active series across all ingesters and the current ingestion rate (samples / sec).
246+
Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series across all ingesters as well as the current ingestion rate (samples / sec).
246247

247248
### HA tracker status
248249

@@ -297,6 +298,15 @@ GET /ring
297298

298299
Displays a web page with the ingesters hash ring status, including the state, healthy and last heartbeat time of each ingester.
299300

301+
### Ingester tenants stats
302+
303+
```
304+
GET /ingester/all_user_stats
305+
306+
```
307+
308+
Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series from a specific ingester as well as the current ingestion rate (samples / sec).
309+
300310
### Ingester mode
301311

302312
```

‎pkg/api/api.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,7 @@ type Ingester interface {
284284
FlushHandler(http.ResponseWriter, *http.Request)
285285
ShutdownHandler(http.ResponseWriter, *http.Request)
286286
RenewTokenHandler(http.ResponseWriter, *http.Request)
287+
AllUserStatsHandler(http.ResponseWriter, *http.Request)
287288
ModeHandler(http.ResponseWriter, *http.Request)
288289
Push(context.Context, *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error)
289290
}
@@ -292,6 +293,8 @@ type Ingester interface {
292293
func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) {
293294
client.RegisterIngesterServer(a.server.GRPC, i)
294295

296+
a.indexPage.AddLink(SectionAdminEndpoints, "/ingester/all_user_stats", "Usage Statistics")
297+
295298
a.indexPage.AddLink(SectionDangerous, "/ingester/flush", "Trigger a Flush of data from Ingester to storage")
296299
a.indexPage.AddLink(SectionDangerous, "/ingester/shutdown", "Trigger Ingester Shutdown (Dangerous)")
297300
a.indexPage.AddLink(SectionDangerous, "/ingester/renewTokens", "Renew Ingester Tokens (10%)")
@@ -300,6 +303,7 @@ func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) {
300303
a.RegisterRoute("/ingester/flush", http.HandlerFunc(i.FlushHandler), false, "GET", "POST")
301304
a.RegisterRoute("/ingester/shutdown", http.HandlerFunc(i.ShutdownHandler), false, "GET", "POST")
302305
a.RegisterRoute("/ingester/renewTokens", http.HandlerFunc(i.RenewTokenHandler), false, "GET", "POST")
306+
a.RegisterRoute("/ingester/all_user_stats", http.HandlerFunc(i.AllUserStatsHandler), false, "GET")
303307
a.RegisterRoute("/ingester/mode", http.HandlerFunc(i.ModeHandler), false, "GET", "POST")
304308
a.RegisterRoute("/ingester/push", push.Handler(pushConfig.MaxRecvMsgSize, a.sourceIPs, i.Push), true, "POST") // For testing and debugging.
305309

‎pkg/distributor/distributor.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
"github.com/cortexproject/cortex/pkg/cortexpb"
3131
"github.com/cortexproject/cortex/pkg/ha"
32+
"github.com/cortexproject/cortex/pkg/ingester"
3233
ingester_client "github.com/cortexproject/cortex/pkg/ingester/client"
3334
"github.com/cortexproject/cortex/pkg/ring"
3435
ring_client "github.com/cortexproject/cortex/pkg/ring/client"
@@ -1319,7 +1320,7 @@ func (d *Distributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetad
13191320
}
13201321

13211322
// UserStats returns statistics about the current user.
1322-
func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) {
1323+
func (d *Distributor) UserStats(ctx context.Context) (*ingester.UserStats, error) {
13231324
replicationSet, err := d.GetIngestersForMetadata(ctx)
13241325
if err != nil {
13251326
return nil, err
@@ -1336,7 +1337,7 @@ func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) {
13361337
return nil, err
13371338
}
13381339

1339-
totalStats := &UserStats{}
1340+
totalStats := &ingester.UserStats{}
13401341
for _, resp := range resps {
13411342
r := resp.(*ingester_client.UserStatsResponse)
13421343
totalStats.IngestionRate += r.IngestionRate
@@ -1354,17 +1355,11 @@ func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) {
13541355
return totalStats, nil
13551356
}
13561357

1357-
// UserIDStats models ingestion statistics for one user, including the user ID
1358-
type UserIDStats struct {
1359-
UserID string `json:"userID"`
1360-
UserStats
1361-
}
1362-
13631358
// AllUserStats returns statistics about all users.
13641359
// Note it does not divide by the ReplicationFactor like UserStats()
1365-
func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) {
1360+
func (d *Distributor) AllUserStats(ctx context.Context) ([]ingester.UserIDStats, error) {
13661361
// Add up by user, across all responses from ingesters
1367-
perUserTotals := make(map[string]UserStats)
1362+
perUserTotals := make(map[string]ingester.UserStats)
13681363

13691364
req := &ingester_client.UserStatsRequest{}
13701365
ctx = user.InjectOrgID(ctx, "1") // fake: ingester insists on having an org ID
@@ -1389,28 +1384,41 @@ func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) {
13891384
s.RuleIngestionRate += u.Data.RuleIngestionRate
13901385
s.NumSeries += u.Data.NumSeries
13911386
s.ActiveSeries += u.Data.ActiveSeries
1387+
s.LoadedBlocks += u.Data.LoadedBlocks
13921388
perUserTotals[u.UserId] = s
13931389
}
13941390
}
13951391

13961392
// Turn aggregated map into a slice for return
1397-
response := make([]UserIDStats, 0, len(perUserTotals))
1393+
response := make([]ingester.UserIDStats, 0, len(perUserTotals))
13981394
for id, stats := range perUserTotals {
1399-
response = append(response, UserIDStats{
1395+
response = append(response, ingester.UserIDStats{
14001396
UserID: id,
1401-
UserStats: UserStats{
1397+
UserStats: ingester.UserStats{
14021398
IngestionRate: stats.IngestionRate,
14031399
APIIngestionRate: stats.APIIngestionRate,
14041400
RuleIngestionRate: stats.RuleIngestionRate,
14051401
NumSeries: stats.NumSeries,
14061402
ActiveSeries: stats.ActiveSeries,
1403+
LoadedBlocks: stats.LoadedBlocks,
14071404
},
14081405
})
14091406
}
14101407

14111408
return response, nil
14121409
}
14131410

1411+
// AllUserStatsHandler shows stats for all users.
1412+
func (d *Distributor) AllUserStatsHandler(w http.ResponseWriter, r *http.Request) {
1413+
stats, err := d.AllUserStats(r.Context())
1414+
if err != nil {
1415+
http.Error(w, err.Error(), http.StatusInternalServerError)
1416+
return
1417+
}
1418+
1419+
ingester.AllUserStatsRender(w, r, stats, d.ingestersRing.ReplicationFactor())
1420+
}
1421+
14141422
func (d *Distributor) ServeHTTP(w http.ResponseWriter, req *http.Request) {
14151423
if d.distributorsRing != nil {
14161424
d.distributorsRing.ServeHTTP(w, req)

‎pkg/distributor/http_server.go‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,6 @@ import (
66
"github.com/cortexproject/cortex/pkg/util"
77
)
88

9-
// UserStats models ingestion statistics for one user.
10-
type UserStats struct {
11-
IngestionRate float64 `json:"ingestionRate"`
12-
NumSeries uint64 `json:"numSeries"`
13-
APIIngestionRate float64 `json:"APIIngestionRate"`
14-
RuleIngestionRate float64 `json:"RuleIngestionRate"`
15-
ActiveSeries uint64 `json:"activeSeries"`
16-
}
17-
189
// UserStatsHandler handles user stats to the Distributor.
1910
func (d *Distributor) UserStatsHandler(w http.ResponseWriter, r *http.Request) {
2011
stats, err := d.UserStats(r.Context())

0 commit comments

Comments
 (0)