Skip to content

Commit 603a32d

Browse files
committed
[encoding/googlecloudlogentry] Add support for GCP Cloud DNS Logs
1 parent ecad830 commit 603a32d

File tree

12 files changed

+1116
-0
lines changed

12 files changed

+1116
-0
lines changed

.chloggen/add-cgp-ldns-logs.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: breaking
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
7+
component: extension/googlecloudlogentry_encoding
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Parse Cloud DNS Query logs into log record attributes instead of placing it in the body as is.
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [44561]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: []

extension/encoding/googlecloudlogentryencodingextension/README.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Currently, this extension [can parse the following logs](#supported-log-types) i
2424
- [Regional External Application Load Balancer](https://docs.cloud.google.com/load-balancing/docs/https/https-reg-logging-monitoring)
2525
- [Cloud Armor logs](https://docs.cloud.google.com/armor/docs/request-logging) (embedded within load balancer logs) (extension [mapping](#cloud-armor-logs))
2626
- [Proxy Network Load Balancer logs](https://docs.cloud.google.com/load-balancing/docs/tcp/tcp-ssl-proxy-logging-monitoring#log-records) (extension [mapping](#proxy-network-load-balancer-logs))
27+
- [Cloud DNS logs](https://docs.cloud.google.com/dns/docs/monitoring#dns-log-record-format) (extension [mapping](#cloud-dns-logs))
2728

2829
For all others logs, the payload will be placed in the log record attribute. In this case, the following configuration options are supported:
2930

@@ -129,7 +130,10 @@ Examples:
129130

130131
- Audit Logs: `encoding.format: "gcp.auditlog"`
131132
- VPC Flow Logs: `encoding.format: "gcp.vpcflow"`
133+
- Application Load Balancer Logs: `encoding.format: "gcp.load_balancing"`
132134
- Proxy Network Load Balancer Logs: `encoding.format: "gcp.proxy-nlb"`
135+
- Cloud DNS Logs: `encoding.format: "gcp.dns"`
136+
133137

134138
### How encoding.format is determined
135139

@@ -142,6 +146,10 @@ For example, `projects/my-project/logs/cloudaudit.googleapis.com%2Fsystem_event`
142146
2. **Map log type to format**: The extension maps specific log types to their corresponding encoding formats (`encoding.format`):
143147
- Audit logs (activity, data access, system event, policy): `gcp.auditlog`
144148
- VPC flow logs (network management-sourced and compute-sourced VPC flow logs): `gcp.vpcflow`
149+
- Application Load Balancer logs (Global External and Regional External): `gcp.load_balancing`
150+
- Cloud Armor logs (embedded within load balancer logs): `gcp.armorlog`
151+
- Proxy Network Load Balancer logs: `gcp.proxy-nlb`
152+
- Cloud DNS logs: `gcp.dns`
145153

146154
3. **Set the attribute**: For recognized log types, the `encoding.format` attribute is set as an attribute of the `scope` field in the OTEL output log, allowing for flexible filtering and routing.
147155

@@ -155,8 +163,10 @@ The following format values are supported in the `googlecloudlogentryencodingext
155163
|------------------|------------------|-----------------|
156164
| Audit Logs | `auditlog` | Google Cloud audit logs (activity, data access, system event, policy) |
157165
| VPC Flow Logs | `vpcflow` | Virtual Private Cloud flow log records |
166+
| Application Load Balancer Logs | `load_balancing` | Global and Regional External Application Load Balancer logs |
158167
| Armor Logs | `armorlog` | Google Cloud armor logs (security policies applied) |
159168
| Proxy Network Load Balancer Logs | `proxy-nlb` | Proxy Network Load Balancer connection logs |
169+
| Cloud DNS Logs | `dns` | Cloud DNS query and response logs |
160170

161171
### Cloud Audit Logs
162172

@@ -432,7 +442,39 @@ Application Load Balancer logs (both [Global External](https://docs.cloud.google
432442
| `serverBytesReceived` | `gcp.load_balancing.proxy_nlb.server.bytes_received` |
433443
| `serverBytesSent` | `gcp.load_balancing.proxy_nlb.server.bytes_sent` |
434444

445+
### Cloud DNS logs
446+
447+
[Cloud DNS logs](https://docs.cloud.google.com/dns/docs/monitoring#dns-log-record-format) are mapped into OpenTelemetry attributes as follows:
448+
449+
| Original field | Log record attribute |
450+
|---|---|
451+
| `queryName` | `gcp.dns.query.name` |
452+
| `queryType` | `gcp.dns.query.type` |
453+
| `responseCode` | `gcp.dns.response.code` |
454+
| `alias_query_response_code` | `gcp.dns.alias_query.response.code` |
455+
| `authAnswer` | `gcp.dns.auth_answer` |
456+
| `rdata` | `gcp.dns.rdata` |
457+
| `destinationIP` | `gcp.dns.destination_ip` |
458+
| `sourceNetwork` | `gcp.dns.source.network` |
459+
| `source_type` | `gcp.dns.source.type` |
460+
| `sourceIP` | `client.address` |
461+
| `protocol` | `network.transport` |
462+
| `location` | `cloud.region` |
463+
| `target_name` | `gcp.dns.target.name` |
464+
| `target_type` | `gcp.dns.target.type` |
465+
| `serverLatency` | `gcp.dns.server_latency` |
466+
| `egressError` | `gcp.dns.egress_error` |
467+
| `healthyIps` | `gcp.dns.healthy_ips` |
468+
| `unhealthyIps` | `gcp.dns.unhealthy_ips` |
469+
| `dns64Translated` | `gcp.dns.dns64_translated` |
470+
| `vmInstanceId` | `gcp.dns.vm.instance.id` |
471+
| `vmInstanceIdString` | `gcp.dns.vm.instance.id_string` |
472+
| `vmInstanceName` | `gcp.dns.vm.instance.name` |
473+
| `vmProjectId` | `gcp.dns.vm.project_id` |
474+
| `vmZoneName` | `gcp.dns.vm.zone` |
475+
435476
**Protocol translation**: The numeric protocol field from GCP is automatically translated to human-readable protocol names using the [IANA Protocol Numbers](https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml) standard. Common values include:
477+
436478
- `6``tcp`
437479
- `17``udp`
438480
- `1``icmp`

extension/encoding/googlecloudlogentryencodingextension/extension_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,16 @@ func TestPayloads(t *testing.T) {
221221
logFilename: "testdata/proxynlb/proxynlb-basic.json",
222222
expectedFilename: "testdata/proxynlb/proxynlb-basic_expected.yaml",
223223
},
224+
{
225+
name: "dns query log - no error",
226+
logFilename: "testdata/dnslog/dns_query_no_error.json",
227+
expectedFilename: "testdata/dnslog/dns_query_no_error_expected.yaml",
228+
},
229+
{
230+
name: "dns query log - nxdomain error",
231+
logFilename: "testdata/dnslog/dns_query_no_domain_error.json",
232+
expectedFilename: "testdata/dnslog/dns_query_no_domain_error_expected.yaml",
233+
},
224234
}
225235

226236
extension := newTestExtension(t, Config{})

extension/encoding/googlecloudlogentryencodingextension/internal/constants/format.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const (
88
FormatVPCFlowLog = "vpcflow"
99
FormatLoadBalancerLog = "load-balancer"
1010
FormatProxyNLBLog = "proxy-nlb"
11+
FormatDNSQueryLog = "dns-query"
1112

1213
FormatIdentificationTag = "encoding.format"
1314

@@ -17,4 +18,5 @@ const (
1718
GCPFormatVPCFlowLog = GCPFormatPrefix + FormatVPCFlowLog
1819
GCPFormatLoadBalancerLog = GCPFormatPrefix + FormatLoadBalancerLog
1920
GCPFormatProxyNLBLog = GCPFormatPrefix + FormatProxyNLBLog
21+
GCPFormatDNSQueryLog = GCPFormatPrefix + FormatDNSQueryLog
2022
)
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Find more information about Cloud dns logs at:
5+
// https://docs.cloud.google.com/dns/docs/monitoring#dns-log-record-format
6+
package dnslog // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/googlecloudlogentryencodingextension/internal/dnslog"
7+
8+
import (
9+
"fmt"
10+
11+
gojson "github.com/goccy/go-json"
12+
"go.opentelemetry.io/collector/pdata/pcommon"
13+
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
14+
15+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/googlecloudlogentryencodingextension/internal/shared"
16+
)
17+
18+
const (
19+
CloudDNSQueryLogSuffix = "dns.googleapis.com%2Fdns_queries"
20+
21+
// Query related attributes. Ref: https://datatracker.ietf.org/doc/html/rfc1035#section-4.1.2
22+
// gcpDNSQueryName holds the DNS query name
23+
gcpDNSQueryName = "gcp.dns.query.name"
24+
// gcpDNSQueryType holds the DNS query type
25+
gcpDNSQueryType = "gcp.dns.query.type"
26+
27+
// Response related attributes
28+
// gcpDNSResponseCode holds the DNS response code
29+
gcpDNSResponseCode = "gcp.dns.response.code"
30+
// gcpDNSAliasQueryResponseCode holds the response code for alias queries
31+
gcpDNSAliasQueryResponseCode = "gcp.dns.alias_query.response.code"
32+
// gcpDNSAuthAnswer indicates whether the response is authoritative
33+
gcpDNSAuthAnswer = "gcp.dns.auth_answer" // Ref: https://datatracker.ietf.org/doc/html/rfc1035
34+
// gcpDNSRdata holds DNS answer in presentation format
35+
gcpDNSRdata = "gcp.dns.rdata"
36+
37+
// Network related attributes
38+
// gcpDNSDestinationIP holds target IP address
39+
gcpDNSDestinationIP = "gcp.dns.destination_ip"
40+
// gcpDNSSourceNetwork holds the source network name
41+
gcpDNSSourceNetwork = "gcp.dns.source.network"
42+
// gcpDNSSourceType holds the source type of the DNS query
43+
gcpDNSSourceType = "gcp.dns.source.type"
44+
45+
// Target related attributes
46+
// gcpDNSTargetName holds the target name
47+
gcpDNSTargetName = "gcp.dns.target.name"
48+
// gcpDNSTargetType holds the type of target resolving the DNS query
49+
gcpDNSTargetType = "gcp.dns.target.type"
50+
51+
// Performance and error related attributes
52+
// gcpDNSServerLatency holds the server-side latency in seconds
53+
gcpDNSServerLatency = "gcp.dns.server_latency"
54+
// gcpDNSEgressError holds Egress proxy error, the actual error as received from the on-premises DNS server
55+
gcpDNSEgressError = "gcp.dns.egress_error"
56+
// gcpDNSHealthyIPs holds addresses in the ResourceRecordSet that are known to be HEALTHY.
57+
gcpDNSHealthyIPs = "gcp.dns.healthy_ips"
58+
// gcpDNSUnhealthyIPs holds addresses in the ResourceRecordSet that are known to be UNHEALTHY.
59+
gcpDNSUnhealthyIPs = "gcp.dns.unhealthy_ips"
60+
61+
// DNS feature related attributes
62+
// gcpDNSDNS64Translated indicates whether DNS64 translation was applied
63+
gcpDNSDNS64Translated = "gcp.dns.dns64_translated"
64+
65+
// VM instance related attributes
66+
// gcpDNSVMInstanceID holds the Compute Engine VM instance ID as an integer
67+
gcpDNSVMInstanceID = "gcp.dns.vm.instance.id"
68+
// gcpDNSVMInstanceIDString holds the Compute Engine VM instance ID as a string
69+
gcpDNSVMInstanceIDString = "gcp.dns.vm.instance.id_string"
70+
// gcpDNSVMInstanceName holds the VM instance name
71+
gcpDNSVMInstanceName = "gcp.dns.vm.instance.name"
72+
// gcpDNSVMProjectID holds the Google Cloud project ID of the network from which the query was sent
73+
gcpDNSVMProjectID = "gcp.dns.vm.project_id"
74+
// gcpDNSVMZoneName holds the name of the VM zone from which the query was sent
75+
gcpDNSVMZoneName = "gcp.dns.vm.zone"
76+
)
77+
78+
type dnslog struct {
79+
AliasQueryResponseCode string `json:"alias_query_response_code"`
80+
AuthAnswer *bool `json:"authAnswer"`
81+
DestinationIP string `json:"destinationIP"`
82+
DNS64Translated *bool `json:"dns64Translated"`
83+
EgressError string `json:"egressError"`
84+
HealthyIps string `json:"healthyIps"`
85+
Location string `json:"location"`
86+
Protocol string `json:"protocol"`
87+
ProjectID string `json:"project_id"`
88+
QueryName string `json:"queryName"`
89+
QueryType string `json:"queryType"`
90+
Rdata string `json:"rdata"`
91+
ResponseCode string `json:"responseCode"`
92+
ServerLatency *float64 `json:"serverLatency"`
93+
SourceIP string `json:"sourceIP"`
94+
SourceNetwork string `json:"sourceNetwork"`
95+
SourceType string `json:"source_type"`
96+
TargetName string `json:"target_name"`
97+
TargetType string `json:"target_type"`
98+
UnhealthyIps string `json:"unhealthyIps"`
99+
VMInstanceID *int64 `json:"vmInstanceId"`
100+
VMInstanceIDStr string `json:"vmInstanceIdString"`
101+
VMInstanceName string `json:"vmInstanceName"`
102+
VMProjectID string `json:"vmProjectId"`
103+
VMZoneName string `json:"vmZoneName"`
104+
}
105+
106+
func handleQueryAttributes(log *dnslog, attr pcommon.Map) {
107+
shared.PutStr(gcpDNSQueryName, log.QueryName, attr)
108+
shared.PutStr(gcpDNSQueryType, log.QueryType, attr)
109+
}
110+
111+
func handleResponseAttributes(log *dnslog, attr pcommon.Map) {
112+
shared.PutStr(gcpDNSResponseCode, log.ResponseCode, attr)
113+
shared.PutStr(gcpDNSAliasQueryResponseCode, log.AliasQueryResponseCode, attr)
114+
shared.PutBool(gcpDNSAuthAnswer, log.AuthAnswer, attr)
115+
shared.PutStr(gcpDNSRdata, log.Rdata, attr)
116+
}
117+
118+
func handleNetworkAttributes(log *dnslog, attr pcommon.Map) {
119+
shared.PutStr(gcpDNSDestinationIP, log.DestinationIP, attr)
120+
shared.PutStr(gcpDNSSourceNetwork, log.SourceNetwork, attr)
121+
shared.PutStr(gcpDNSSourceType, log.SourceType, attr)
122+
shared.PutStr(string(semconv.ClientAddressKey), log.SourceIP, attr)
123+
shared.PutStr(string(semconv.NetworkTransportKey), log.Protocol, attr)
124+
shared.PutStr(string(semconv.CloudRegionKey), log.Location, attr)
125+
}
126+
127+
func handleTargetAttributes(log *dnslog, attr pcommon.Map) {
128+
shared.PutStr(gcpDNSTargetName, log.TargetName, attr)
129+
shared.PutStr(gcpDNSTargetType, log.TargetType, attr)
130+
}
131+
132+
func handlePerformanceAndErrorAttributes(log *dnslog, attr pcommon.Map) {
133+
shared.PutDouble(gcpDNSServerLatency, log.ServerLatency, attr)
134+
shared.PutStr(gcpDNSEgressError, log.EgressError, attr)
135+
shared.PutStr(gcpDNSHealthyIPs, log.HealthyIps, attr)
136+
shared.PutStr(gcpDNSUnhealthyIPs, log.UnhealthyIps, attr)
137+
}
138+
139+
func handleDNSFeatureAttributes(log *dnslog, attr pcommon.Map) {
140+
shared.PutBool(gcpDNSDNS64Translated, log.DNS64Translated, attr)
141+
}
142+
143+
func handleVMInstanceAttributes(log *dnslog, attr pcommon.Map) {
144+
shared.PutInt(gcpDNSVMInstanceID, log.VMInstanceID, attr)
145+
shared.PutStr(gcpDNSVMInstanceIDString, log.VMInstanceIDStr, attr)
146+
shared.PutStr(gcpDNSVMInstanceName, log.VMInstanceName, attr)
147+
shared.PutStr(gcpDNSVMProjectID, log.VMProjectID, attr)
148+
shared.PutStr(gcpDNSVMZoneName, log.VMZoneName, attr)
149+
}
150+
151+
func ParsePayloadIntoAttributes(payload []byte, attr pcommon.Map) error {
152+
var log dnslog
153+
if err := gojson.Unmarshal(payload, &log); err != nil {
154+
return fmt.Errorf("failed to unmarshal DNS log: %w", err)
155+
}
156+
157+
handleQueryAttributes(&log, attr)
158+
handleResponseAttributes(&log, attr)
159+
handleNetworkAttributes(&log, attr)
160+
handleTargetAttributes(&log, attr)
161+
handlePerformanceAndErrorAttributes(&log, attr)
162+
handleDNSFeatureAttributes(&log, attr)
163+
handleVMInstanceAttributes(&log, attr)
164+
165+
return nil
166+
}

0 commit comments

Comments
 (0)