Merge pull request #108 from yaauie/support-total_hits-as-object-in-response

Add support for Elasticsearch 7.x `total_hits` response object

Author: yaauie

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.6.0
+  - Add support for extracting hits total from Elasticsearch 7.x responses
+
 ## 3.5.0
   - Added connection check during register to avoid failures during processing
   - Changed Elasticsearch Client transport to use Manticore

lib/logstash/filters/elasticsearch.rb

@@ -94,7 +94,7 @@ def filter(event)
       results = get_client.search(params)
       raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
 
-      event.set("[@metadata][total_hits]", results['hits']['total'])
+      event.set("[@metadata][total_hits]", extract_total_from_hits(results['hits']))
 
       resultsHits = results["hits"]["hits"]
       if !resultsHits.nil? && !resultsHits.empty?
@@ -173,6 +173,21 @@ def extract_value(source, path)
     end
   end
 
+  # Given a "hits" object from an Elasticsearch response, return the total number of hits in
+  # the result set.
+  # @param hits [Hash{String=>Object}]
+  # @return [Integer]
+  def extract_total_from_hits(hits)
+    total = hits['total']
+
+    # Elasticsearch 7.x produces an object containing `value` and `relation` in order
+    # to enable unambiguous reporting when the total is only a lower bound; if we get
+    # an object back, return its `value`.
+    return total['value'] if total.kind_of?(Hash)
+
+    total
+  end
+
   def test_connection!
     get_client.client.ping
   end

logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.5.0'
+  s.version         = '3.6.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/filters/elasticsearch_spec.rb

@@ -113,6 +113,26 @@
       end
     end
 
+    context 'when Elasticsearch 7.x gives us a totals object instead of an integer' do
+      let(:config) do
+        {
+            "hosts" => ["localhost:9200"],
+            "query" => "response: 404",
+            "fields" => { "response" => "code" },
+            "result_size" => 10
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "elasticsearch_7.x_hits_total_as_object.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("[@metadata][total_hits]")).to eq(13476)
+      end
+    end
+
     context "if something wrong happen during connection" do
 
       before(:each) do

spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json

@@ -0,0 +1,70 @@
+{
+  "took": 49,
+  "timed_out": false,
+  "_shards": {
+    "total": 155,
+    "successful": 155,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 13476,
+      "relation": "eq"
+    },
+    "max_score": 1,
+    "hits": [{
+      "_index": "logstash-2014.08.26",
+      "_type": "logs",
+      "_id": "AVVY76L_AW7v0kX8KXo4",
+      "_score": 1,
+      "_source": {
+        "request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
+        "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
+        "geoip": {
+          "timezone": "America/Los_Angeles",
+          "ip": "66.249.73.185",
+          "latitude": 37.386,
+          "continent_code": "NA",
+          "city_name": "Mountain View",
+          "country_code2": "US",
+          "country_name": "United States",
+          "dma_code": 807,
+          "country_code3": "US",
+          "region_name": "California",
+          "location": [-122.0838,
+            37.386
+          ],
+          "postal_code": "94035",
+          "longitude": -122.0838,
+          "region_code": "CA"
+        },
+        "auth": "-",
+        "ident": "-",
+        "verb": "GET",
+        "useragent": {
+          "os": "Other",
+          "major": "2",
+          "minor": "1",
+          "name": "Googlebot",
+          "os_name": "Other",
+          "device": "Spider"
+        },
+        "message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
+        "referrer": "\"-\"",
+        "@timestamp": "2014-08-26T21:22:13.000Z",
+        "response": 404,
+        "bytes": 294,
+        "clientip": "66.249.73.185",
+        "@version": "1",
+        "host": "skywalker",
+        "httpversion": "1.1",
+        "timestamp": "26/Aug/2014:21:22:13 +0000"
+      }
+    }]
+  },
+  "aggregations": {
+    "bytes_avg": {
+      "value": 294
+    }
+  }
+}