fix util_lookup_group to handle large groups

I have recently been getting the above message on fc11 and
I have traced it down to a bug in util_lookup_group.

As of 145 util_lookup_group reads:

gid_t util_lookup_group(struct udev *udev, const char *group)
{
	char *endptr;
	int buflen = sysconf(_SC_GETGR_R_SIZE_MAX);
	char buf[buflen];
	struct group grbuf;
	struct group *gr;
	gid_t gid = 0;

	if (strcmp(group, "root") == 0)
		return 0;
	gid = strtoul(group, &endptr, 10);
	if (endptr[0] == '\0')
		return gid;

	errno = 0;
	getgrnam_r(group, &grbuf, buf, buflen, &gr);
	if (gr != NULL)
		return gr->gr_gid;
	if (errno == 0 || errno == ENOENT || errno == ESRCH)
		err(udev, "specified group '%s' unknown\n", group);
	else
		err(udev, "error resolving group '%s': %m\n", group);
	return 0;
}

The errno value from getgrnam_r here is ERANGE which is documented as
"Insufficient buffer space supplied".

When I call get getgrnam_r with a large enough buffer everything
works.  Indicating that the problem is that sysconf is returning
a value too small.

A quick google search tells me that sysconf(_S_GETGR_R_SIZE_MAX)
is documented as:

> sysconf(_S_GETGR_R_SIZE_MAX) returns either -1 or a good
> suggested starting value for buflen.  It does not return the
> worst case possible for buflen.

In my case I have a group with about 50 users in /etc/group
and that is what triggered the problem in udev and caused
all of the udevs group lookups to fail.

The following patch which dynamically allocates the group member buffer
should fix this problem.

Signed-off-by: "Eric W. Biederman" <ebiederm@xmission.com>

1 files changed, 27 insertions, 11 deletions

diff --git a/libudev/libudev-util-private.c b/libudev/libudev-util-private.c
index f7daa9460b..021734d772 100644
--- a/libudev/libudev-util-private.c
+++ b/libudev/libudev-util-private.c
@@ -149,8 +149,8 @@ uid_t util_lookup_user(struct udev *udev, const char *user)
 gid_t util_lookup_group(struct udev *udev, const char *group)
 {
 	char *endptr;
-	int buflen = sysconf(_SC_GETGR_R_SIZE_MAX);
-	char buf[buflen];
+	int buflen;
+	char *buf;
 	struct group grbuf;
 	struct group *gr;
 	gid_t gid = 0;
@@ -161,15 +161,31 @@ gid_t util_lookup_group(struct udev *udev, const char *group)
 	if (endptr[0] == '\0')
 		return gid;
 
-	errno = 0;
-	getgrnam_r(group, &grbuf, buf, buflen, &gr);
-	if (gr != NULL)
-		return gr->gr_gid;
-	if (errno == 0 || errno == ENOENT || errno == ESRCH)
-		err(udev, "specified group '%s' unknown\n", group);
-	else
-		err(udev, "error resolving group '%s': %m\n", group);
-	return 0;
+	buflen = sysconf(_SC_GETGR_R_SIZE_MAX);
+	if (buflen < 0)
+		buflen = 1000;
+	buf = NULL;
+	gid = 0;
+	for (;;) {
+		buf = realloc(buf, buflen);
+		if (!buf)
+			break;
+		errno = 0;
+		getgrnam_r(group, &grbuf, buf, buflen, &gr);
+		if (gr != NULL) 
+			gid = gr->gr_gid;
+		else if (errno == ERANGE) {
+			buflen *= 2;
+			continue;
+		}
+		else if (errno == 0 || errno == ENOENT || errno == ESRCH)
+			err(udev, "specified group '%s' unknown\n", group);
+		else
+			err(udev, "error resolving group '%s': %m\n", group);
+		break;
+	}
+	free(buf);
+	return gid;
 }
 
 /* handle "[<SUBSYSTEM>/<KERNEL>]<attribute>" format */