From 094accd3ebec17ead6c391757eaa18763b72d83f Mon Sep 17 00:00:00 2001
From: Alexander Sosedkin <asosedkin@redhat.com>
Date: Mon, 26 Jan 2026 20:16:36 +0100
Subject: [PATCH] x509/name_constraints: introduce a rich comparator

These are preparatory changes before implementing N * log N intersection
over sorted lists of constraints.

Signed-off-by: Alexander Sosedkin <asosedkin@redhat.com>

Upstream-Status: Backport [https://gitlab.com/gnutls/gnutls/-/commit/094accd3ebec17ead6c391757eaa18763b72d83f]
CVE: CVE-2025-14831
Signed-off-by: Vijay Anusuri <vanusuri@mvista.com>
---
 lib/x509/name_constraints.c | 411 ++++++++++++++++++++++++++++--------
 1 file changed, 320 insertions(+), 91 deletions(-)

--- a/lib/x509/name_constraints.c
+++ b/lib/x509/name_constraints.c
@@ -39,6 +39,9 @@
 #include "ip.h"
 #include "ip-in-cidr.h"
 #include "intprops.h"
+#include "minmax.h"
+
+#include <string.h>
 
 #define MAX_NC_CHECKS (1 << 20)
 
@@ -63,6 +66,282 @@ static struct name_constraints_node_st *
 name_constraints_node_new(gnutls_x509_name_constraints_t nc, unsigned type,
 			  const unsigned char *data, unsigned int size);
 
+/* An enum for "rich" comparisons that not only let us sort name constraints,
+ * children-before-parent, but also subsume them during intersection. */
+enum name_constraint_relation {
+	NC_SORTS_BEFORE = -2, /* unrelated constraints */
+	NC_INCLUDED_BY = -1, /* nc1 is included by nc2 / children sort first */
+	NC_EQUAL = 0, /* exact match */
+	NC_INCLUDES = 1, /* nc1 includes nc2 / parents sort last */
+	NC_SORTS_AFTER = 2 /* unrelated constraints */
+};
+
+/* A helper to compare just a pair of strings with this rich comparison */
+static enum name_constraint_relation
+compare_strings(const void *n1, size_t n1_len, const void *n2, size_t n2_len)
+{
+	int r = memcmp(n1, n2, MIN(n1_len, n2_len));
+	if (r < 0)
+		return NC_SORTS_BEFORE;
+	if (r > 0)
+		return NC_SORTS_AFTER;
+	if (n1_len < n2_len)
+		return NC_SORTS_BEFORE;
+	if (n1_len > n2_len)
+		return NC_SORTS_AFTER;
+	return NC_EQUAL;
+}
+
+/* Rich-compare DNS names. Example order/relationships:
+ * z.x.a INCLUDED_BY x.a BEFORE y.a INCLUDED_BY a BEFORE x.b BEFORE y.b */
+static enum name_constraint_relation compare_dns_names(const gnutls_datum_t *n1,
+						       const gnutls_datum_t *n2)
+{
+	enum name_constraint_relation rel;
+	unsigned int i, j, i_end, j_end;
+
+	/* start from the end of each name */
+	i = i_end = n1->size;
+	j = j_end = n2->size;
+
+	/* skip the trailing dots for the comparison */
+	while (i && n1->data[i - 1] == '.')
+		i_end = i = i - 1;
+	while (j && n2->data[j - 1] == '.')
+		j_end = j = j - 1;
+
+	while (1) {
+		// rewind back to beginning or an after-dot position
+		while (i && n1->data[i - 1] != '.')
+			i--;
+		while (j && n2->data[j - 1] != '.')
+			j--;
+
+		rel = compare_strings(&n1->data[i], i_end - i, &n2->data[j],
+				      j_end - j);
+		if (rel == NC_SORTS_BEFORE) /* x.a BEFORE y.a */
+			return NC_SORTS_BEFORE;
+		if (rel == NC_SORTS_AFTER) /* y.a AFTER x.a */
+			return NC_SORTS_AFTER;
+		if (!i && j) /* x.a INCLUDES z.x.a */
+			return NC_INCLUDES;
+		if (i && !j) /* z.x.a INCLUDED_BY x.a */
+			return NC_INCLUDED_BY;
+
+		if (!i && !j) /* r == 0, we ran out of components to compare */
+			return NC_EQUAL;
+		/* r == 0, i && j: step back past a dot and keep comparing */
+		i_end = i = i - 1;
+		j_end = j = j - 1;
+
+		/* support for non-standard ".gr INCLUDES example.gr" [1] */
+		if (!i && j) /* .a INCLUDES x.a */
+			return NC_INCLUDES;
+		if (i && !j) /* x.a INCLUDED_BY .a */
+			return NC_INCLUDED_BY;
+	}
+}
+/* [1] https://mailarchive.ietf.org/arch/msg/saag/Bw6PtreW0G7aEG7SikfzKHES4VA */
+
+/* Rich-compare email name constraints. Example order/relationships:
+ * z@x.a INCLUDED_BY x.a BEFORE y.a INCLUDED_BY a BEFORE x@b BEFORE y@b */
+static enum name_constraint_relation compare_emails(const gnutls_datum_t *n1,
+						    const gnutls_datum_t *n2)
+{
+	enum name_constraint_relation domains_rel;
+	unsigned int i, j, i_end, j_end;
+	gnutls_datum_t d1, d2; /* borrow from n1 and n2 */
+
+	/* start from the end of each name */
+	i = i_end = n1->size;
+	j = j_end = n2->size;
+
+	/* rewind to @s to look for domains */
+	while (i && n1->data[i - 1] != '@')
+		i--;
+	d1.size = i_end - i;
+	d1.data = &n1->data[i];
+	while (j && n2->data[j - 1] != '@')
+		j--;
+	d2.size = j_end - j;
+	d2.data = &n2->data[j];
+
+	domains_rel = compare_dns_names(&d1, &d2);
+
+	/* email constraint semantics differ from DNS
+	 * DNS: x.a INCLUDED_BY a
+	 * Email: x.a INCLUDED_BY .a BEFORE a */
+	if (domains_rel == NC_INCLUDED_BY || domains_rel == NC_INCLUDES) {
+		bool d1_has_dot = (d1.size > 0 && d1.data[0] == '.');
+		bool d2_has_dot = (d2.size > 0 && d2.data[0] == '.');
+		/* a constraint without a dot is exact, excluding subdomains */
+		if (!d2_has_dot && domains_rel == NC_INCLUDED_BY)
+			domains_rel = NC_SORTS_BEFORE; /* x.a BEFORE a */
+		if (!d1_has_dot && domains_rel == NC_INCLUDES)
+			domains_rel = NC_SORTS_AFTER; /* a AFTER x.a */
+	}
+
+	if (!i && !j) { /* both are domains-only */
+		return domains_rel;
+	} else if (i && !j) { /* n1 is email, n2 is domain */
+		switch (domains_rel) {
+		case NC_SORTS_AFTER:
+			return NC_SORTS_AFTER;
+		case NC_SORTS_BEFORE:
+			return NC_SORTS_BEFORE;
+		case NC_INCLUDES: /* n2 is more specific, a@x.a AFTER z.x.a */
+			return NC_SORTS_AFTER;
+		case NC_EQUAL: /* subdomains match, z@x.a INCLUDED_BY x.a */
+		case NC_INCLUDED_BY: /* n1 is more specific */
+			return NC_INCLUDED_BY;
+		}
+	} else if (!i && j) { /* n1 is domain, n2 is email */
+		switch (domains_rel) {
+		case NC_SORTS_AFTER:
+			return NC_SORTS_AFTER;
+		case NC_SORTS_BEFORE:
+			return NC_SORTS_BEFORE;
+		case NC_INCLUDES: /* n2 is more specific, a AFTER z@x.a */
+			return NC_SORTS_AFTER;
+		case NC_EQUAL: /* subdomains match, x.a INCLUDES z@x.a */
+			return NC_INCLUDES;
+		case NC_INCLUDED_BY: /* n1 is more specific, x.a BEFORE z@a */
+			return NC_SORTS_BEFORE;
+		}
+	} else if (i && j) { /* both are emails */
+		switch (domains_rel) {
+		case NC_SORTS_AFTER:
+			return NC_SORTS_AFTER;
+		case NC_SORTS_BEFORE:
+			return NC_SORTS_BEFORE;
+		case NC_INCLUDES: // n2 is more specific
+			return NC_SORTS_AFTER;
+		case NC_INCLUDED_BY: // n1 is more specific
+			return NC_SORTS_BEFORE;
+		case NC_EQUAL: // only case when we need to look before the @
+			break; // see below for readability
+		}
+	}
+
+	/* i && j, both are emails, domain names match, compare up to @ */
+	return compare_strings(n1->data, i - 1, n2->data, j - 1);
+}
+
+/* Rich-compare IP address constraints. Example order/relationships:
+ * 10.0.0.0/24 INCLUDED_BY 10.0.0.0/16 BEFORE 1::1/128 INCLUDED_BY 1::1/127 */
+static enum name_constraint_relation compare_ip_ncs(const gnutls_datum_t *n1,
+						    const gnutls_datum_t *n2)
+{
+	unsigned int len, i;
+	int r;
+	const unsigned char *ip1, *ip2, *mask1, *mask2;
+	unsigned char masked11[16], masked22[16], masked12[16], masked21[16];
+
+	if (n1->size < n2->size)
+		return NC_SORTS_BEFORE;
+	if (n1->size > n2->size)
+		return NC_SORTS_AFTER;
+	len = n1->size / 2; /* 4 for IPv4, 16 for IPv6 */
+
+	/* data is a concatenation of prefix and mask */
+	ip1 = n1->data;
+	ip2 = n2->data;
+	mask1 = n1->data + len;
+	mask2 = n2->data + len;
+	for (i = 0; i < len; i++) {
+		masked11[i] = ip1[i] & mask1[i];
+		masked22[i] = ip2[i] & mask2[i];
+		masked12[i] = ip1[i] & mask2[i];
+		masked21[i] = ip2[i] & mask1[i];
+	}
+
+	r = memcmp(mask1, mask2, len);
+	if (r < 0 && !memcmp(masked11, masked21, len)) /* prefix1 < prefix2 */
+		return NC_INCLUDES; /* ip1 & mask1 == ip2 & mask1 */
+	if (r > 0 && !memcmp(masked12, masked22, len)) /* prefix1 > prefix2 */
+		return NC_INCLUDED_BY; /* ip1 & mask2 == ip2 & mask2 */
+
+	r = memcmp(masked11, masked22, len);
+	if (r < 0)
+		return NC_SORTS_BEFORE;
+	else if (r > 0)
+		return NC_SORTS_AFTER;
+	return NC_EQUAL;
+}
+
+static inline bool is_supported_type(unsigned type)
+{
+	return type == GNUTLS_SAN_DNSNAME || type == GNUTLS_SAN_RFC822NAME ||
+	       type == GNUTLS_SAN_IPADDRESS;
+}
+
+/* Universal comparison for name constraint nodes.
+ * Unsupported types sort before supported types to allow early handling.
+ * NULL represents end-of-list and sorts after everything else. */
+static enum name_constraint_relation
+compare_name_constraint_nodes(const struct name_constraints_node_st *n1,
+			      const struct name_constraints_node_st *n2)
+{
+	bool n1_supported, n2_supported;
+
+	if (!n1 && !n2)
+		return NC_EQUAL;
+	if (!n1)
+		return NC_SORTS_AFTER;
+	if (!n2)
+		return NC_SORTS_BEFORE;
+
+	n1_supported = is_supported_type(n1->type);
+	n2_supported = is_supported_type(n2->type);
+
+	/* unsupported types bubble up (sort first). intersect relies on this */
+	if (!n1_supported && n2_supported)
+		return NC_SORTS_BEFORE;
+	if (n1_supported && !n2_supported)
+		return NC_SORTS_AFTER;
+
+	/* next, sort by type */
+	if (n1->type < n2->type)
+		return NC_SORTS_BEFORE;
+	if (n1->type > n2->type)
+		return NC_SORTS_AFTER;
+
+	/* now look deeper */
+	switch (n1->type) {
+	case GNUTLS_SAN_DNSNAME:
+		return compare_dns_names(&n1->name, &n2->name);
+	case GNUTLS_SAN_RFC822NAME:
+		return compare_emails(&n1->name, &n2->name);
+	case GNUTLS_SAN_IPADDRESS:
+		return compare_ip_ncs(&n1->name, &n2->name);
+	default:
+		/* unsupported types: stable lexicographic order */
+		return compare_strings(n1->name.data, n1->name.size,
+				       n2->name.data, n2->name.size);
+	}
+}
+
+/* qsort-compatible wrapper */
+static int compare_name_constraint_nodes_qsort(const void *a, const void *b)
+{
+	const struct name_constraints_node_st *const *n1 = a;
+	const struct name_constraints_node_st *const *n2 = b;
+	enum name_constraint_relation rel;
+
+	rel = compare_name_constraint_nodes(*n1, *n2);
+	switch (rel) {
+	case NC_SORTS_BEFORE:
+	case NC_INCLUDED_BY:
+		return -1;
+	case NC_SORTS_AFTER:
+	case NC_INCLUDES:
+		return 1;
+	case NC_EQUAL:
+	default:
+		return 0;
+	}
+}
+
 static int
 name_constraints_node_list_add(struct name_constraints_node_list_st *list,
 			       struct name_constraints_node_st *node)
@@ -420,9 +699,7 @@ static int name_constraints_node_list_in
 			}
 		}
 
-		if (found != NULL && (t->type == GNUTLS_SAN_DNSNAME ||
-				      t->type == GNUTLS_SAN_RFC822NAME ||
-				      t->type == GNUTLS_SAN_IPADDRESS)) {
+		if (found != NULL && is_supported_type(t->type)) {
 			/* move node from PERMITTED to REMOVED */
 			ret = name_constraints_node_list_add(&removed, t);
 			if (ret < 0) {
@@ -824,61 +1101,14 @@ cleanup:
 	return ret;
 }
 
-static unsigned ends_with(const gnutls_datum_t *str,
-			  const gnutls_datum_t *suffix)
-{
-	unsigned char *tree;
-	unsigned int treelen;
-
-	if (suffix->size >= str->size)
-		return 0;
-
-	tree = suffix->data;
-	treelen = suffix->size;
-	if ((treelen > 0) && (tree[0] == '.')) {
-		tree++;
-		treelen--;
-	}
-
-	if (memcmp(str->data + str->size - treelen, tree, treelen) == 0 &&
-	    str->data[str->size - treelen - 1] == '.')
-		return 1; /* match */
-
-	return 0;
-}
-
-static unsigned email_ends_with(const gnutls_datum_t *str,
-				const gnutls_datum_t *suffix)
-{
-	if (suffix->size >= str->size) {
-		return 0;
-	}
-
-	if (suffix->size > 0 && memcmp(str->data + str->size - suffix->size,
-				       suffix->data, suffix->size) != 0) {
-		return 0;
-	}
-
-	if (suffix->size > 1 && suffix->data[0] == '.') { /* .domain.com */
-		return 1; /* match */
-	} else if (str->data[str->size - suffix->size - 1] == '@') {
-		return 1; /* match */
-	}
-
-	return 0;
-}
-
 static unsigned dnsname_matches(const gnutls_datum_t *name,
 				const gnutls_datum_t *suffix)
 {
 	_gnutls_hard_log("matching %.*s with DNS constraint %.*s\n", name->size,
 			 name->data, suffix->size, suffix->data);
 
-	if (suffix->size == name->size &&
-	    memcmp(suffix->data, name->data, suffix->size) == 0)
-		return 1; /* match */
-
-	return ends_with(name, suffix);
+	enum name_constraint_relation rel = compare_dns_names(name, suffix);
+	return rel == NC_EQUAL || rel == NC_INCLUDED_BY;
 }
 
 static unsigned email_matches(const gnutls_datum_t *name,
@@ -887,11 +1117,8 @@ static unsigned email_matches(const gnut
 	_gnutls_hard_log("matching %.*s with e-mail constraint %.*s\n",
 			 name->size, name->data, suffix->size, suffix->data);
 
-	if (suffix->size == name->size &&
-	    memcmp(suffix->data, name->data, suffix->size) == 0)
-		return 1; /* match */
-
-	return email_ends_with(name, suffix);
+	enum name_constraint_relation rel = compare_emails(name, suffix);
+	return rel == NC_EQUAL || rel == NC_INCLUDED_BY;
 }
 
 /*-
@@ -915,8 +1142,7 @@ static int name_constraints_intersect_no
 	// presume empty intersection
 	struct name_constraints_node_st *intersection = NULL;
 	const struct name_constraints_node_st *to_copy = NULL;
-	unsigned iplength = 0;
-	unsigned byte;
+	enum name_constraint_relation rel;
 
 	*_intersection = NULL;
 
@@ -925,32 +1151,49 @@ static int name_constraints_intersect_no
 	}
 	switch (node1->type) {
 	case GNUTLS_SAN_DNSNAME:
-		if (!dnsname_matches(&node2->name, &node1->name))
+		rel = compare_dns_names(&node1->name, &node2->name);
+		switch (rel) {
+		case NC_EQUAL: // equal means doesn't matter which one
+		case NC_INCLUDES: // node2 is more specific
+			to_copy = node2;
+			break;
+		case NC_INCLUDED_BY: // node1 is more specific
+			to_copy = node1;
+			break;
+		case NC_SORTS_BEFORE: // no intersection
+		case NC_SORTS_AFTER: // no intersection
 			return GNUTLS_E_SUCCESS;
-		to_copy = node2;
+		}
 		break;
 	case GNUTLS_SAN_RFC822NAME:
-		if (!email_matches(&node2->name, &node1->name))
+		rel = compare_emails(&node1->name, &node2->name);
+		switch (rel) {
+		case NC_EQUAL: // equal means doesn't matter which one
+		case NC_INCLUDES: // node2 is more specific
+			to_copy = node2;
+			break;
+		case NC_INCLUDED_BY: // node1 is more specific
+			to_copy = node1;
+			break;
+		case NC_SORTS_BEFORE: // no intersection
+		case NC_SORTS_AFTER: // no intersection
 			return GNUTLS_E_SUCCESS;
-		to_copy = node2;
+		}
 		break;
 	case GNUTLS_SAN_IPADDRESS:
-		if (node1->name.size != node2->name.size)
+		rel = compare_ip_ncs(&node1->name, &node2->name);
+		switch (rel) {
+		case NC_EQUAL: // equal means doesn't matter which one
+		case NC_INCLUDES: // node2 is more specific
+			to_copy = node2;
+			break;
+		case NC_INCLUDED_BY: // node1 is more specific
+			to_copy = node1;
+			break;
+		case NC_SORTS_BEFORE: // no intersection
+		case NC_SORTS_AFTER: // no intersection
 			return GNUTLS_E_SUCCESS;
-		iplength = node1->name.size / 2;
-		for (byte = 0; byte < iplength; byte++) {
-			if (((node1->name.data[byte] ^
-			      node2->name.data[byte]) // XOR of addresses
-			     & node1->name.data[byte +
-						iplength] // AND mask from nc1
-			     & node2->name.data[byte +
-						iplength]) // AND mask from nc2
-			    != 0) {
-				// CIDRS do not intersect
-				return GNUTLS_E_SUCCESS;
-			}
 		}
-		to_copy = node2;
 		break;
 	default:
 		// for other types, we don't know how to do the intersection, assume empty
@@ -967,20 +1210,6 @@ static int name_constraints_intersect_no
 		intersection = *_intersection;
 
 		assert(intersection->name.data != NULL);
-
-		if (intersection->type == GNUTLS_SAN_IPADDRESS) {
-			// make sure both IP addresses are correctly masked
-			_gnutls_mask_ip(intersection->name.data,
-					intersection->name.data + iplength,
-					iplength);
-			_gnutls_mask_ip(node1->name.data,
-					node1->name.data + iplength, iplength);
-			// update intersection, if necessary (we already know one is subset of other)
-			for (byte = 0; byte < 2 * iplength; byte++) {
-				intersection->name.data[byte] |=
-					node1->name.data[byte];
-			}
-		}
 	}
 
 	return GNUTLS_E_SUCCESS;
