resolved: fix DNSSEC canonical ordering logic

When applying canonical DNSSEC ordering for an RRset only order by the
wire format of the RRs' RDATA, not by the full wire formatting. The RFC
isn't particularly clear about this, but this is apparently how it is
done. This fixes validation of pentagon.gov's DS RRset.

2 files changed, 28 insertions, 8 deletions

diff --git a/src/resolve/resolved-dns-dnssec.c b/src/resolve/resolved-dns-dnssec.c
index 606d681779..f2180c3e35 100644
--- a/src/resolve/resolved-dns-dnssec.c
+++ b/src/resolve/resolved-dns-dnssec.c
@@ -116,15 +116,15 @@ static int rr_compare(const void *a, const void *b) {
         assert(*y);
         assert((*y)->wire_format);
 
-        m = MIN((*x)->wire_format_size, (*y)->wire_format_size);
+        m = MIN(DNS_RESOURCE_RECORD_RDATA_SIZE(*x), DNS_RESOURCE_RECORD_RDATA_SIZE(*y));
 
-        r = memcmp((*x)->wire_format, (*y)->wire_format, m);
+        r = memcmp(DNS_RESOURCE_RECORD_RDATA(*x), DNS_RESOURCE_RECORD_RDATA(*y), m);
         if (r != 0)
                 return r;
 
-        if ((*x)->wire_format_size < (*y)->wire_format_size)
+        if (DNS_RESOURCE_RECORD_RDATA_SIZE(*x) < DNS_RESOURCE_RECORD_RDATA_SIZE(*y))
                 return -1;
-        else if ((*x)->wire_format_size > (*y)->wire_format_size)
+        else if (DNS_RESOURCE_RECORD_RDATA_SIZE(*x) > DNS_RESOURCE_RECORD_RDATA_SIZE(*y))
                 return 1;
 
         return 0;
@@ -605,12 +605,11 @@ int dnssec_verify_rrset(
                 md_add_uint16(md, rr->key->class);
                 md_add_uint32(md, rrsig->rrsig.original_ttl);
 
-                assert(rr->wire_format_rdata_offset <= rr->wire_format_size);
-                l = rr->wire_format_size - rr->wire_format_rdata_offset;
+                l = DNS_RESOURCE_RECORD_RDATA_SIZE(rr);
                 assert(l <= 0xFFFF);
 
                 md_add_uint16(md, (uint16_t) l);
-                gcry_md_write(md, (uint8_t*) rr->wire_format + rr->wire_format_rdata_offset, l);
+                gcry_md_write(md, DNS_RESOURCE_RECORD_RDATA(rr), l);
         }
 
         hash = gcry_md_read(md, 0);
diff --git a/src/resolve/resolved-dns-rr.h b/src/resolve/resolved-dns-rr.h
index 72bded7d48..26ab36401c 100644
--- a/src/resolve/resolved-dns-rr.h
+++ b/src/resolve/resolved-dns-rr.h
@@ -236,7 +236,7 @@ struct DnsResourceRecord {
 };
 
 static inline const char* DNS_RESOURCE_KEY_NAME(const DnsResourceKey *key) {
-        if (_unlikely_(!key))
+        if (!key)
                 return NULL;
 
         if (key->_name)
@@ -245,6 +245,27 @@ static inline const char* DNS_RESOURCE_KEY_NAME(const DnsResourceKey *key) {
         return (char*) key + sizeof(DnsResourceKey);
 }
 
+static inline const void* DNS_RESOURCE_RECORD_RDATA(DnsResourceRecord *rr) {
+        if (!rr)
+                return NULL;
+
+        if (!rr->wire_format)
+                return NULL;
+
+        assert(rr->wire_format_rdata_offset <= rr->wire_format_size);
+        return (uint8_t*) rr->wire_format + rr->wire_format_rdata_offset;
+}
+
+static inline size_t DNS_RESOURCE_RECORD_RDATA_SIZE(DnsResourceRecord *rr) {
+        if (!rr)
+                return 0;
+        if (!rr->wire_format)
+                return 0;
+
+        assert(rr->wire_format_rdata_offset <= rr->wire_format_size);
+        return rr->wire_format_size - rr->wire_format_rdata_offset;
+}
+
 DnsResourceKey* dns_resource_key_new(uint16_t class, uint16_t type, const char *name);
 DnsResourceKey* dns_resource_key_new_redirect(const DnsResourceKey *key, const DnsResourceRecord *cname);
 int dns_resource_key_new_append_suffix(DnsResourceKey **ret, DnsResourceKey *key, char *name);