Skip to content

Commit 3b4cdf8

Browse files
HeavyPunkHeavyPunk
authored
feat: log readed rows count to log if timeout occured (#317)
* feat: add read rows to logs if request dropped * fix: refactoring * fix: refactoring * fix: refactoring * fix: fixed config doc * fix: fix config tests * fix: review fixes * fix: timeout should close connection also when body reading started * fix: lint
1 parent f1c1419 commit 3b4cdf8

File tree

13 files changed

+321
-127
lines changed

13 files changed

+321
-127
lines changed

‎autocomplete/autocomplete.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -367,9 +367,11 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
367367
h.config.ClickHouse.URL,
368368
sql,
369369
clickhouse.Options{
370-
TLSConfig: h.config.ClickHouse.TLSConfig,
371-
Timeout: h.config.ClickHouse.IndexTimeout,
372-
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
370+
TLSConfig: h.config.ClickHouse.TLSConfig,
371+
Timeout: h.config.ClickHouse.IndexTimeout,
372+
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
373+
CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,
374+
ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
373375
},
374376
nil,
375377
)
@@ -639,9 +641,11 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
639641
h.config.ClickHouse.URL,
640642
sql,
641643
clickhouse.Options{
642-
TLSConfig: h.config.ClickHouse.TLSConfig,
643-
Timeout: h.config.ClickHouse.IndexTimeout,
644-
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
644+
TLSConfig: h.config.ClickHouse.TLSConfig,
645+
Timeout: h.config.ClickHouse.IndexTimeout,
646+
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
647+
CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,
648+
ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
645649
},
646650
nil,
647651
)

‎config/config.go‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ type Common struct {
123123
type FeatureFlags struct {
124124
UseCarbonBehavior bool `toml:"use-carbon-behaviour" json:"use-carbon-behaviour" comment:"if true, prefers carbon's behaviour on how tags are treated"`
125125
DontMatchMissingTags bool `toml:"dont-match-missing-tags" json:"dont-match-missing-tags" comment:"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all"`
126+
LogQueryProgress bool `toml:"log-query-progress" json:"log-query-progress" comment:"if true, gch will log affected rows count by clickhouse query"`
126127
}
127128

128129
// IndexReverseRule contains rules to use direct or reversed request to index table
@@ -210,6 +211,8 @@ type ClickHouse struct {
210211
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"`
211212
QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout, limiters) for durations greater or equal"`
212213

214+
ProgressSendingInterval time.Duration `toml:"progress-sending-interval" json:"progress-sending-interval" comment:"time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header"`
215+
213216
RenderMaxQueries int `toml:"render-max-queries" json:"render-max-queries" comment:"Max queries to render queiries"`
214217
RenderConcurrentQueries int `toml:"render-concurrent-queries" json:"render-concurrent-queries" comment:"Concurrent queries to render queiries"`
215218
RenderAdaptiveQueries int `toml:"render-adaptive-queries" json:"render-adaptive-queries" comment:"Render adaptive queries (based on load average) for increase/decrease concurrent queries"`
@@ -390,24 +393,25 @@ func New() *Config {
390393
DegragedLoad: 1.0,
391394
},
392395
ClickHouse: ClickHouse{
393-
URL: "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
394-
DataTimeout: time.Minute,
395-
IndexTable: "graphite_index",
396-
IndexUseDaily: true,
397-
TaggedUseDaily: true,
398-
IndexReverse: "auto",
399-
IndexReverses: IndexReverses{},
400-
IndexTimeout: time.Minute,
401-
TaggedTable: "graphite_tagged",
402-
TaggedAutocompleDays: 7,
403-
ExtraPrefix: "",
404-
ConnectTimeout: time.Second,
405-
DataTableLegacy: "",
406-
RollupConfLegacy: "auto",
407-
MaxDataPoints: 1048576,
408-
InternalAggregation: true,
409-
FindLimiter: limiter.NoopLimiter{},
410-
TagsLimiter: limiter.NoopLimiter{},
396+
URL: "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
397+
DataTimeout: time.Minute,
398+
ProgressSendingInterval: 10 * time.Second,
399+
IndexTable: "graphite_index",
400+
IndexUseDaily: true,
401+
TaggedUseDaily: true,
402+
IndexReverse: "auto",
403+
IndexReverses: IndexReverses{},
404+
IndexTimeout: time.Minute,
405+
TaggedTable: "graphite_tagged",
406+
TaggedAutocompleDays: 7,
407+
ExtraPrefix: "",
408+
ConnectTimeout: time.Second,
409+
DataTableLegacy: "",
410+
RollupConfLegacy: "auto",
411+
MaxDataPoints: 1048576,
412+
InternalAggregation: true,
413+
FindLimiter: limiter.NoopLimiter{},
414+
TagsLimiter: limiter.NoopLimiter{},
411415
},
412416
Tags: Tags{
413417
Threads: 1,

‎config/config_test.go‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -368,26 +368,27 @@ sample-thereafter = 12
368368
Limiter: limiter.NoopLimiter{},
369369
},
370370
},
371-
FindLimiter: limiter.NoopLimiter{},
372-
TagsLimiter: limiter.NoopLimiter{},
373-
IndexTable: "graphite_index",
374-
IndexReverse: "direct",
375-
IndexReverses: make(IndexReverses, 2),
376-
IndexTimeout: 4000000000,
377-
TaggedTable: "graphite_tags",
378-
TaggedAutocompleDays: 5,
379-
TreeTable: "tree",
380-
ReverseTreeTable: "reversed_tree",
381-
DateTreeTable: "data_tree",
382-
DateTreeTableVersion: 2,
383-
TreeTimeout: 5000000000,
384-
TagTable: "tag_table",
385-
ExtraPrefix: "tum.pu-dum",
386-
ConnectTimeout: 2000000000,
387-
DataTableLegacy: "data",
388-
RollupConfLegacy: "none",
389-
MaxDataPoints: 8000,
390-
InternalAggregation: true,
371+
ProgressSendingInterval: 10 * time.Second,
372+
FindLimiter: limiter.NoopLimiter{},
373+
TagsLimiter: limiter.NoopLimiter{},
374+
IndexTable: "graphite_index",
375+
IndexReverse: "direct",
376+
IndexReverses: make(IndexReverses, 2),
377+
IndexTimeout: 4000000000,
378+
TaggedTable: "graphite_tags",
379+
TaggedAutocompleDays: 5,
380+
TreeTable: "tree",
381+
ReverseTreeTable: "reversed_tree",
382+
DateTreeTable: "data_tree",
383+
DateTreeTableVersion: 2,
384+
TreeTimeout: 5000000000,
385+
TagTable: "tag_table",
386+
ExtraPrefix: "tum.pu-dum",
387+
ConnectTimeout: 2000000000,
388+
DataTableLegacy: "data",
389+
RollupConfLegacy: "none",
390+
MaxDataPoints: 8000,
391+
InternalAggregation: true,
391392
}
392393
expected.ClickHouse.IndexReverses[0] = &IndexReverseRule{"suf", "pref", "", nil, "direct"}
393394
r, _ = regexp.Compile("^reg$")
@@ -647,6 +648,7 @@ sample-thereafter = 12
647648
Limiter: limiter.NoopLimiter{},
648649
},
649650
},
651+
ProgressSendingInterval: 10 * time.Second,
650652
RenderMaxQueries: 1000,
651653
RenderConcurrentQueries: 10,
652654
FindMaxQueries: 200,
@@ -970,6 +972,7 @@ sample-thereafter = 12
970972
AdaptiveQueries: 6,
971973
},
972974
},
975+
ProgressSendingInterval: 10 * time.Second,
973976
RenderMaxQueries: 1000,
974977
RenderConcurrentQueries: 10,
975978
RenderAdaptiveQueries: 4,

‎doc/config.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ Note that this option only works for terms with '=' operator in them.
290290
use-carbon-behaviour = false
291291
# if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all
292292
dont-match-missing-tags = false
293+
# if true, gch will log affected rows count by clickhouse query
294+
log-query-progress = false
293295

294296
[metrics]
295297
# graphite relay address
@@ -320,6 +322,8 @@ Note that this option only works for terms with '=' operator in them.
320322
url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"
321323
# default total timeout to fetch data, can be overwritten with query-params
322324
data-timeout = "1m0s"
325+
# time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header
326+
progress-sending-interval = "10s"
323327
# Max queries to render queiries
324328
render-max-queries = 0
325329
# Concurrent queries to render queiries

‎finder/finder.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@ type Finder interface {
2424

2525
func newPlainFinder(ctx context.Context, config *config.Config, query string, from int64, until int64, useCache bool) Finder {
2626
opts := clickhouse.Options{
27-
TLSConfig: config.ClickHouse.TLSConfig,
28-
Timeout: config.ClickHouse.IndexTimeout,
29-
ConnectTimeout: config.ClickHouse.ConnectTimeout,
27+
TLSConfig: config.ClickHouse.TLSConfig,
28+
Timeout: config.ClickHouse.IndexTimeout,
29+
ConnectTimeout: config.ClickHouse.ConnectTimeout,
30+
CheckRequestProgress: config.FeatureFlags.LogQueryProgress,
31+
ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,
3032
}
3133

3234
var f Finder
@@ -121,9 +123,11 @@ func Leaf(value []byte) ([]byte, bool) {
121123

122124
func FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm, from int64, until int64) (Result, error) {
123125
opts := clickhouse.Options{
124-
Timeout: config.ClickHouse.IndexTimeout,
125-
ConnectTimeout: config.ClickHouse.ConnectTimeout,
126-
TLSConfig: config.ClickHouse.TLSConfig,
126+
Timeout: config.ClickHouse.IndexTimeout,
127+
ConnectTimeout: config.ClickHouse.ConnectTimeout,
128+
TLSConfig: config.ClickHouse.TLSConfig,
129+
CheckRequestProgress: config.FeatureFlags.LogQueryProgress,
130+
ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,
127131
}
128132

129133
useCache := config.Common.FindCache != nil

‎finder/tagged_test.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -642,9 +642,11 @@ func TestParseSeriesByTagWithCostsFromCountTable(t *testing.T) {
642642
srv.AddResponce(sql, response)
643643

644644
opts := clickhouse.Options{
645-
Timeout: cfg.ClickHouse.IndexTimeout,
646-
ConnectTimeout: cfg.ClickHouse.ConnectTimeout,
647-
TLSConfig: cfg.ClickHouse.TLSConfig,
645+
Timeout: cfg.ClickHouse.IndexTimeout,
646+
ConnectTimeout: cfg.ClickHouse.ConnectTimeout,
647+
TLSConfig: cfg.ClickHouse.TLSConfig,
648+
CheckRequestProgress: cfg.FeatureFlags.LogQueryProgress,
649+
ProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval,
648650
}
649651

650652
taggedFinder := NewTagged(

0 commit comments

Comments
 (0)