Phantom DNS Query to GCP VM Metadata Service in My AWS Workload — Revealed by Route 53 Resolver Logging

AWSTemplateFormatVersion: '2010-09-09'
Description: >
  Route 53 Resolver DNS Firewall with VPC‑level DNS **query logging**.
  Logs **all** DNS queries for a supplied VPC to CloudWatch Logs.

Parameters:
  VpcId:
    Type: AWS::EC2::VPC::Id
    Description: VPC ID to protect and monitor.
  AssociationPriority:
    Type: Number
    Default: 150
    Description: 'Priority (100‑9900) for the VPC firewall association; must be unique within the VPC.'
    MinValue: 100
    MaxValue: 9900
  LogRetentionDays:
    Type: Number
    Default: 30
    Description: Retention (days) for the CloudWatch Logs group.

Resources:

  QueryLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      RetentionInDays: !Ref LogRetentionDays

  BlockedDomains:
    Type: AWS::Route53Resolver::FirewallDomainList
    Properties:
      Name: blocked-domains
      Domains:
        - badexample.com
        - malware.example
        - phishing.test

  FirewallRuleGroup:
    Type: AWS::Route53Resolver::FirewallRuleGroup
    Properties:
      Name: vpc-dns-firewall-rule-group
      FirewallRules:
        - FirewallDomainListId: !Ref BlockedDomains
          Priority: 10
          Action: BLOCK
          BlockResponse: NODATA

  FirewallAssociation:
    Type: AWS::Route53Resolver::FirewallRuleGroupAssociation
    Properties:
      FirewallRuleGroupId: !Ref FirewallRuleGroup
      VpcId: !Ref VpcId
      Priority: !Ref AssociationPriority
      Name: vpc-dns-firewall-association

  DNSQueryLogConfig:
    Type: AWS::Route53Resolver::ResolverQueryLoggingConfig
    Properties:
      Name: vpc-dns-query-logs
      DestinationArn: !GetAtt QueryLogGroup.Arn

  DNSQueryLogAssociation:
    Type: AWS::Route53Resolver::ResolverQueryLoggingConfigAssociation
    Properties:
      ResolverQueryLogConfigId: !Ref DNSQueryLogConfig
      ResourceId: !Ref VpcId

  DNSQueryLogsPolicy:
    Type: AWS::Logs::ResourcePolicy
    Properties:
      PolicyName: route53resolver-query-logs
      PolicyDocument: !Sub |
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Sid": "Route53ResolverQueryLogsToCloudWatch",
              "Effect": "Allow",
              "Principal": { "Service": "route53resolver.amazonaws.com" },
              "Action": [
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Resource": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${QueryLogGroup}:*"
            }
          ]
        }

Outputs:
  FirewallRuleGroupId:
    Description: ID of the DNS Firewall Rule Group
    Value: !Ref FirewallRuleGroup
  QueryLogConfigId:
    Description: ID of the Resolver Query Log Config
    Value: !Ref DNSQueryLogConfig
  QueryLogGroupName:
    Description: CloudWatch Logs group for DNS query logs
    Value: !Ref QueryLogGroup

{
    "version": "1.100000",
    "account_id": "452954105288",
    "region": "us-east-1",
    "vpc_id": "vpc-0b20bb8a5eb28cb99",
    "query_timestamp": "2025-04-27T04:45:23Z",
    "query_name": "badexample.com.",
    "query_type": "A",
    "query_class": "IN",
    "rcode": "NOERROR",
    "answers": [],
    "srcaddr": "10.0.101.123",
    "srcport": "46275",
    "transport": "UDP",
    "srcids": {
        "instance": "i-0cf6fbb4f8960f245"
    },
    "firewall_rule_action": "BLOCK",
    "firewall_rule_group_id": "rslvr-frg-69632fc987684fbd",
    "firewall_domain_list_id": "rslvr-fdl-de20dd498c14dce"
}

def get_cloud_info(self):
    provider = str(self.config.cloud_provider).lower()

    if not provider or provider == "none" or provider == "false":
        return {}
    if provider == "aws":
        # This line was not hit as I didn't explicitly configure the `cloud_provider` option in my Elastic APM setup. 
        ...
    elif provider == "gcp":
        ...
    elif provider == "azure":
        ...
    elif provider == "auto" or provider == "true":
        data = {}
        # This line returned `None` as my workload was based on AWS Fargate, which does not support the EC2 Metadata endpoint.
        data = cloud.aws_metadata()
        if data:
            return data
        # This line caused the DNS query.
        data = cloud.gcp_metadata()
        if data:
            return data
        data = cloud.azure_metadata()
        return data
    else:
        self.logger.warning("Unknown value for CLOUD_PROVIDER, skipping cloud metadata: {}".format(provider))
        return {}

if _DNS_REDIRECT_IP is None:
    badips = set()
    badnames = (
        "does-not-exist.example.com.",
        "example.invalid.",
        "__cloud_init_expected_not_found__",
    )
    badresults: dict = {}
    for iname in badnames:
        try:
            result = socket.getaddrinfo(
                iname, None, 0, 0, socket.SOCK_STREAM, socket.AI_CANONNAME
            )
            badresults[iname] = []
            for _fam, _stype, _proto, cname, sockaddr in result:
                badresults[iname].append("%s: %s" % (cname, sockaddr[0]))
                badips.add(sockaddr[0])
        except (socket.gaierror, socket.error):
            pass
    _DNS_REDIRECT_IP = badips
    if badresults:
        LOG.debug("detected dns redirection: %s", badresults)