avoid attempting to lookup IP literals as hostnames

previously, __lookup_ipliteral only checked its argument against the
requested address family, so IPv4 literals passed through to
__lookup_name if the caller asked for only IPv6 results, and likewise
for IPv6 literals when the caller asked for only IPv4. this resulted
in spurious DNS lookups that reportedly even succeeded with some
nameservers.

now, __lookup_ipliteral attempts to parse its argument as both IPv4
and IPv6, and returns an error (to stop further search) rather than 0
(no results yet) if the form of the argument mismatches the requested
address family.

based on patch by Julien Ramseier.

src/network/lookup_ipliteral.c

@@ -15,38 +15,43 @@ int __lookup_ipliteral(struct address buf[static 1], const char *name, int famil
 {
 	struct in_addr a4;
 	struct in6_addr a6;
-	if (family != AF_INET6 && __inet_aton(name, &a4)>0) {
+	if (__inet_aton(name, &a4) > 0) {
+		if (family == AF_INET6) /* wrong family */
+			return EAI_NONAME;
 		memcpy(&buf[0].addr, &a4, sizeof a4);
 		buf[0].family = AF_INET;
 		buf[0].scopeid = 0;
 		return 1;
 	}
-	if (family != AF_INET) {
-		char tmp[64];
-		char *p = strchr(name, '%'), *z;
-		unsigned long long scopeid = 0;
-		if (p && p-name < 64) {
-			memcpy(tmp, name, p-name);
-			tmp[p-name] = 0;
-			name = tmp;
-		}
-		if (inet_pton(AF_INET6, name, &a6)<=0) return 0;
-		memcpy(&buf[0].addr, &a6, sizeof a6);
-		buf[0].family = AF_INET6;
-		if (p) {
-			if (isdigit(*++p)) scopeid = strtoull(p, &z, 10);
-			else z = p-1;
-			if (*z) {
-				if (!IN6_IS_ADDR_LINKLOCAL(&a6) &&
-				    !IN6_IS_ADDR_MC_LINKLOCAL(&a6))
-					return EAI_NONAME;
-				scopeid = if_nametoindex(p);
-				if (!scopeid) return EAI_NONAME;
-			}
-			if (scopeid > UINT_MAX) return EAI_NONAME;
+
+	char tmp[64];
+	char *p = strchr(name, '%'), *z;
+	unsigned long long scopeid = 0;
+	if (p && p-name < 64) {
+		memcpy(tmp, name, p-name);
+		tmp[p-name] = 0;
+		name = tmp;
+	}
+
+	if (inet_pton(AF_INET6, name, &a6) <= 0)
+		return 0;
+	if (family == AF_INET) /* wrong family */
+		return EAI_NONAME;
+
+	memcpy(&buf[0].addr, &a6, sizeof a6);
+	buf[0].family = AF_INET6;
+	if (p) {
+		if (isdigit(*++p)) scopeid = strtoull(p, &z, 10);
+		else z = p-1;
+		if (*z) {
+			if (!IN6_IS_ADDR_LINKLOCAL(&a6) &&
+			    !IN6_IS_ADDR_MC_LINKLOCAL(&a6))
+				return EAI_NONAME;
+			scopeid = if_nametoindex(p);
+			if (!scopeid) return EAI_NONAME;
 		}
-		buf[0].scopeid = scopeid;
-		return 1;
+		if (scopeid > UINT_MAX) return EAI_NONAME;
 	}
-	return 0;
+	buf[0].scopeid = scopeid;
+	return 1;
 }