feat: log readed rows count to log if timeout occured

autocomplete/autocomplete.go

@@ -367,9 +367,11 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 			h.config.ClickHouse.URL,
 			sql,
 			clickhouse.Options{
-				TLSConfig:      h.config.ClickHouse.TLSConfig,
-				Timeout:        h.config.ClickHouse.IndexTimeout,
-				ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
+				TLSConfig:               h.config.ClickHouse.TLSConfig,
+				Timeout:                 h.config.ClickHouse.IndexTimeout,
+				ConnectTimeout:          h.config.ClickHouse.ConnectTimeout,
+				CheckRequestProgress:    h.config.FeatureFlags.LogQueryProgress,
+				ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
 			},
 			nil,
 		)
@@ -639,9 +641,11 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 			h.config.ClickHouse.URL,
 			sql,
 			clickhouse.Options{
-				TLSConfig:      h.config.ClickHouse.TLSConfig,
-				Timeout:        h.config.ClickHouse.IndexTimeout,
-				ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
+				TLSConfig:               h.config.ClickHouse.TLSConfig,
+				Timeout:                 h.config.ClickHouse.IndexTimeout,
+				ConnectTimeout:          h.config.ClickHouse.ConnectTimeout,
+				CheckRequestProgress:    h.config.FeatureFlags.LogQueryProgress,
+				ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
 			},
 			nil,
 		)

config/config.go

@@ -123,6 +123,7 @@ type Common struct {
 type FeatureFlags struct {
 	UseCarbonBehavior    bool `toml:"use-carbon-behaviour" json:"use-carbon-behaviour" comment:"if true, prefers carbon's behaviour on how tags are treated"`
 	DontMatchMissingTags bool `toml:"dont-match-missing-tags" json:"dont-match-missing-tags" comment:"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all"`
+	LogQueryProgress     bool `toml:"log-query-progress" json:"log-query-progress" comment:"if true, gch will log affected rows count by clickhouse query"`
 }
 
 // IndexReverseRule contains rules to use direct or reversed request to index table
@@ -210,6 +211,8 @@ type ClickHouse struct {
 	DataTimeout time.Duration `toml:"data-timeout"             json:"data-timeout"             comment:"default total timeout to fetch data, can be overwritten with query-params"`
 	QueryParams []QueryParam  `toml:"query-params"             json:"query-params"             comment:"customized query params (url, data timeout, limiters) for durations greater or equal"`
 
+	ProgressSendingInterval time.Duration `toml:"progress-sending-interval" json:"progress-sending-interval" comment:"time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header"`
+
 	RenderMaxQueries        int `toml:"render-max-queries" json:"render-max-queries" comment:"Max queries to render queiries"`
 	RenderConcurrentQueries int `toml:"render-concurrent-queries" json:"render-concurrent-queries" comment:"Concurrent queries to render queiries"`
 	RenderAdaptiveQueries   int `toml:"render-adaptive-queries" json:"render-adaptive-queries" comment:"Render adaptive queries (based on load average) for increase/decrease concurrent queries"`
@@ -390,24 +393,25 @@ func New() *Config {
 			DegragedLoad:     1.0,
 		},
 		ClickHouse: ClickHouse{
-			URL:                  "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
-			DataTimeout:          time.Minute,
-			IndexTable:           "graphite_index",
-			IndexUseDaily:        true,
-			TaggedUseDaily:       true,
-			IndexReverse:         "auto",
-			IndexReverses:        IndexReverses{},
-			IndexTimeout:         time.Minute,
-			TaggedTable:          "graphite_tagged",
-			TaggedAutocompleDays: 7,
-			ExtraPrefix:          "",
-			ConnectTimeout:       time.Second,
-			DataTableLegacy:      "",
-			RollupConfLegacy:     "auto",
-			MaxDataPoints:        1048576,
-			InternalAggregation:  true,
-			FindLimiter:          limiter.NoopLimiter{},
-			TagsLimiter:          limiter.NoopLimiter{},
+			URL:                     "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
+			DataTimeout:             time.Minute,
+			ProgressSendingInterval: 10 * time.Second,
+			IndexTable:              "graphite_index",
+			IndexUseDaily:           true,
+			TaggedUseDaily:          true,
+			IndexReverse:            "auto",
+			IndexReverses:           IndexReverses{},
+			IndexTimeout:            time.Minute,
+			TaggedTable:             "graphite_tagged",
+			TaggedAutocompleDays:    7,
+			ExtraPrefix:             "",
+			ConnectTimeout:          time.Second,
+			DataTableLegacy:         "",
+			RollupConfLegacy:        "auto",
+			MaxDataPoints:           1048576,
+			InternalAggregation:     true,
+			FindLimiter:             limiter.NoopLimiter{},
+			TagsLimiter:             limiter.NoopLimiter{},
 		},
 		Tags: Tags{
 			Threads:     1,

config/config_test.go

@@ -368,26 +368,27 @@ sample-thereafter = 12
 				Limiter:     limiter.NoopLimiter{},
 			},
 		},
-		FindLimiter:          limiter.NoopLimiter{},
-		TagsLimiter:          limiter.NoopLimiter{},
-		IndexTable:           "graphite_index",
-		IndexReverse:         "direct",
-		IndexReverses:        make(IndexReverses, 2),
-		IndexTimeout:         4000000000,
-		TaggedTable:          "graphite_tags",
-		TaggedAutocompleDays: 5,
-		TreeTable:            "tree",
-		ReverseTreeTable:     "reversed_tree",
-		DateTreeTable:        "data_tree",
-		DateTreeTableVersion: 2,
-		TreeTimeout:          5000000000,
-		TagTable:             "tag_table",
-		ExtraPrefix:          "tum.pu-dum",
-		ConnectTimeout:       2000000000,
-		DataTableLegacy:      "data",
-		RollupConfLegacy:     "none",
-		MaxDataPoints:        8000,
-		InternalAggregation:  true,
+		ProgressSendingInterval: 10 * time.Second,
+		FindLimiter:             limiter.NoopLimiter{},
+		TagsLimiter:             limiter.NoopLimiter{},
+		IndexTable:              "graphite_index",
+		IndexReverse:            "direct",
+		IndexReverses:           make(IndexReverses, 2),
+		IndexTimeout:            4000000000,
+		TaggedTable:             "graphite_tags",
+		TaggedAutocompleDays:    5,
+		TreeTable:               "tree",
+		ReverseTreeTable:        "reversed_tree",
+		DateTreeTable:           "data_tree",
+		DateTreeTableVersion:    2,
+		TreeTimeout:             5000000000,
+		TagTable:                "tag_table",
+		ExtraPrefix:             "tum.pu-dum",
+		ConnectTimeout:          2000000000,
+		DataTableLegacy:         "data",
+		RollupConfLegacy:        "none",
+		MaxDataPoints:           8000,
+		InternalAggregation:     true,
 	}
 	expected.ClickHouse.IndexReverses[0] = &IndexReverseRule{"suf", "pref", "", nil, "direct"}
 	r, _ = regexp.Compile("^reg$")
@@ -647,6 +648,7 @@ sample-thereafter = 12
 				Limiter:     limiter.NoopLimiter{},
 			},
 		},
+		ProgressSendingInterval: 10 * time.Second,
 		RenderMaxQueries:        1000,
 		RenderConcurrentQueries: 10,
 		FindMaxQueries:          200,
@@ -970,6 +972,7 @@ sample-thereafter = 12
 				AdaptiveQueries:   6,
 			},
 		},
+		ProgressSendingInterval: 10 * time.Second,
 		RenderMaxQueries:        1000,
 		RenderConcurrentQueries: 10,
 		RenderAdaptiveQueries:   4,

doc/config.md

@@ -290,6 +290,8 @@ Note that this option only works for terms with '=' operator in them.
  use-carbon-behaviour = false
  # if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all
  dont-match-missing-tags = false
+ # if true, gch will log affected rows count by clickhouse query
+ log-query-progress = false
 
 [metrics]
  # graphite relay address
@@ -320,6 +322,8 @@ Note that this option only works for terms with '=' operator in them.
  url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"
  # default total timeout to fetch data, can be overwritten with query-params
  data-timeout = "1m0s"
+ # time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header
+ progress-sending-interval = "10s"
  # Max queries to render queiries
  render-max-queries = 0
  # Concurrent queries to render queiries

finder/finder.go

@@ -24,9 +24,11 @@ type Finder interface {
 
 func newPlainFinder(ctx context.Context, config *config.Config, query string, from int64, until int64, useCache bool) Finder {
 	opts := clickhouse.Options{
-		TLSConfig:      config.ClickHouse.TLSConfig,
-		Timeout:        config.ClickHouse.IndexTimeout,
-		ConnectTimeout: config.ClickHouse.ConnectTimeout,
+		TLSConfig:               config.ClickHouse.TLSConfig,
+		Timeout:                 config.ClickHouse.IndexTimeout,
+		ConnectTimeout:          config.ClickHouse.ConnectTimeout,
+		CheckRequestProgress:    config.FeatureFlags.LogQueryProgress,
+		ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,
 	}
 
 	var f Finder
@@ -121,9 +123,11 @@ func Leaf(value []byte) ([]byte, bool) {
 
 func FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm, from int64, until int64) (Result, error) {
 	opts := clickhouse.Options{
-		Timeout:        config.ClickHouse.IndexTimeout,
-		ConnectTimeout: config.ClickHouse.ConnectTimeout,
-		TLSConfig:      config.ClickHouse.TLSConfig,
+		Timeout:                 config.ClickHouse.IndexTimeout,
+		ConnectTimeout:          config.ClickHouse.ConnectTimeout,
+		TLSConfig:               config.ClickHouse.TLSConfig,
+		CheckRequestProgress:    config.FeatureFlags.LogQueryProgress,
+		ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,
 	}
 
 	useCache := config.Common.FindCache != nil

finder/tagged_test.go

@@ -642,9 +642,11 @@ func TestParseSeriesByTagWithCostsFromCountTable(t *testing.T) {
 		srv.AddResponce(sql, response)
 
 		opts := clickhouse.Options{
-			Timeout:        cfg.ClickHouse.IndexTimeout,
-			ConnectTimeout: cfg.ClickHouse.ConnectTimeout,
-			TLSConfig:      cfg.ClickHouse.TLSConfig,
+			Timeout:                 cfg.ClickHouse.IndexTimeout,
+			ConnectTimeout:          cfg.ClickHouse.ConnectTimeout,
+			TLSConfig:               cfg.ClickHouse.TLSConfig,
+			CheckRequestProgress:    cfg.FeatureFlags.LogQueryProgress,
+			ProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval,
 		}
 
 		taggedFinder := NewTagged(