strans/tests/daemon_restart_test.c

#define _GNU_SOURCE
#include <dbus/dbus.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <poll.h>
#include <signal.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

#include "ipc.h"

enum
{
	Calltimeout = 4000,
	Starttimeout = 8000,
	Stoptimeout = 3000,
	Ctrlmask = 1<<2,
};

typedef struct Test Test;

struct Test
{
	pid_t first;
	pid_t firstpid;
	pid_t second;
	pid_t helper;
	int firsterrfd;
	int seconderrfd;
	int helpererrfd;
	int ipcfirst;
	int ipcnew;
	int busfirstfd;
	DBusConnection *busfirst;
	DBusConnection *busnew;
	char root[256];
	char runtime[320];
	char config[320];
	char ibus[384];
	char bus[448];
	char home[320];
	char socket[384];
	char addrfile[512];
	char firstaddress[512];
	char secondaddress[512];
	char firsterr[8192];
	size_t nfirsterr;
	char seconderr[8192];
	size_t nseconderr;
	char helpererr[2048];
	size_t nhelpererr;
};

static int
fail(char *fmt, ...)
{
	va_list ap;

	fprintf(stderr, "daemon_restart_test: ");
	va_start(ap, fmt);
	vfprintf(stderr, fmt, ap);
	va_end(ap);
	fputc('\n', stderr);
	return 0;
}

static int64_t
nowms(void)
{
	struct timespec ts;

	if(clock_gettime(CLOCK_MONOTONIC, &ts) < 0)
		return -1;
	return (int64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
}

static int
leftms(int64_t deadline)
{
	int64_t n;

	n = nowms();
	if(n < 0)
		return -1;
	n = deadline - n;
	if(n <= 0)
		return 0;
	if(n > 0x7fffffff)
		return 0x7fffffff;
	return n;
}

static int
makedir(char *path)
{
	if(mkdir(path, 0700) == 0)
		return 1;
	return fail("mkdir %s: %s", path, strerror(errno));
}

static void
readpipe(int fd, char *buf, size_t cap, size_t *used)
{
	ssize_t n;

	if(fd < 0 || cap == 0)
		return;
	while(*used + 1 < cap){
		n = read(fd, buf + *used, cap - *used - 1);
		if(n > 0){
			*used += n;
			continue;
		}
		if(n < 0 && errno == EINTR)
			continue;
		break;
	}
	buf[*used] = '\0';
}

static void
showerrors(Test *t)
{
	readpipe(t->firsterrfd, t->firsterr, sizeof t->firsterr,
		&t->nfirsterr);
	readpipe(t->seconderrfd, t->seconderr, sizeof t->seconderr,
		&t->nseconderr);
	readpipe(t->helpererrfd, t->helpererr, sizeof t->helpererr,
		&t->nhelpererr);
	if(t->nfirsterr != 0)
		fprintf(stderr, "daemon_restart_test: first daemon stderr:\n%s",
			t->firsterr);
	if(t->nseconderr != 0)
		fprintf(stderr, "daemon_restart_test: second daemon stderr:\n%s",
			t->seconderr);
	if(t->nhelpererr != 0)
		fprintf(stderr, "daemon_restart_test: address helper stderr:\n%s",
			t->helpererr);
}

static int
setup(Test *t)
{
	struct sigaction sa;

	memset(t, 0, sizeof *t);
	t->first = -1;
	t->firstpid = -1;
	t->second = -1;
	t->helper = -1;
	t->firsterrfd = -1;
	t->seconderrfd = -1;
	t->helpererrfd = -1;
	t->ipcfirst = -1;
	t->ipcnew = -1;
	t->busfirstfd = -1;
	memset(&sa, 0, sizeof sa);
	sa.sa_handler = SIG_DFL;
	if(sigemptyset(&sa.sa_mask) < 0 || sigaction(SIGCHLD, &sa, NULL) < 0)
		return fail("establish exact child ownership: %s", strerror(errno));
	if(nowms() < 0)
		return fail("read monotonic clock: %s", strerror(errno));
	snprintf(t->root, sizeof t->root, "/tmp/strans-restart.XXXXXX");
	if(mkdtemp(t->root) == NULL){
		t->root[0] = '\0';
		return fail("mkdtemp: %s", strerror(errno));
	}
	if(snprintf(t->runtime, sizeof t->runtime, "%s/runtime", t->root)
	   >= (int)sizeof t->runtime ||
	   snprintf(t->config, sizeof t->config, "%s/config", t->root)
	   >= (int)sizeof t->config ||
	   snprintf(t->ibus, sizeof t->ibus, "%s/ibus", t->config)
	   >= (int)sizeof t->ibus ||
	   snprintf(t->bus, sizeof t->bus, "%s/bus", t->ibus)
	   >= (int)sizeof t->bus ||
	   snprintf(t->home, sizeof t->home, "%s/home", t->root)
	   >= (int)sizeof t->home ||
	   snprintf(t->socket, sizeof t->socket, "%s/strans.sock", t->runtime)
	   >= (int)sizeof t->socket)
		return fail("temporary path is too long");
	if(!makedir(t->runtime) || !makedir(t->config) || !makedir(t->ibus) ||
	   !makedir(t->bus) || !makedir(t->home))
		return 0;
	return 1;
}

static int
childstderr(int fd)
{
	int flags;

	if(fd != STDERR_FILENO){
		if(dup2(fd, STDERR_FILENO) < 0)
			return 0;
		close(fd);
		return 1;
	}
	flags = fcntl(fd, F_GETFD);
	if(flags < 0 || fcntl(fd, F_SETFD, flags & ~FD_CLOEXEC) < 0)
		return 0;
	return 1;
}

static int
startchild(Test *t, char *program, char *mapdir, pid_t *child, int *errfd)
{
	int errpipe[2], fd;
	long maxfd;
	pid_t pid;

	if(pipe2(errpipe, O_CLOEXEC|O_NONBLOCK) < 0)
		return fail("pipe2: %s", strerror(errno));
	pid = fork();
	if(pid < 0){
		close(errpipe[0]);
		close(errpipe[1]);
		return fail("fork: %s", strerror(errno));
	}
	if(pid == 0){
		close(errpipe[0]);
		if(!childstderr(errpipe[1]))
			_exit(126);
		if(close_range(3, UINT_MAX, 0) < 0){
			maxfd = sysconf(_SC_OPEN_MAX);
			if(maxfd < 0)
				maxfd = 1024;
			for(fd = 3; fd < maxfd; fd++)
				close(fd);
		}
		if(setenv("XDG_RUNTIME_DIR", t->runtime, 1) < 0 ||
		   setenv("XDG_CONFIG_HOME", t->config, 1) < 0 ||
		   setenv("HOME", t->home, 1) < 0 || unsetenv("DISPLAY") < 0 ||
		   unsetenv("DBUS_SESSION_BUS_ADDRESS") < 0 ||
		   unsetenv("IBUS_ADDRESS") < 0 ||
		   unsetenv("IBUS_ADDRESS_FILE") < 0){
			dprintf(STDERR_FILENO, "set daemon environment: %s\n",
				strerror(errno));
			_exit(126);
		}
		execl(program, program, mapdir, (char*)0);
		dprintf(STDERR_FILENO, "exec %s: %s\n", program, strerror(errno));
		_exit(127);
	}
	close(errpipe[1]);
	*child = pid;
	*errfd = errpipe[0];
	return 1;
}

static int
findaddress(Test *t)
{
	DIR *dir;
	struct dirent *de;
	int count, ok;

	dir = opendir(t->bus);
	if(dir == NULL){
		fail("open IBus directory %s: %s", t->bus, strerror(errno));
		return -1;
	}
	count = 0;
	ok = 1;
	errno = 0;
	while((de = readdir(dir)) != NULL){
		if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0 ||
		   strstr(de->d_name, ".tmp.") != NULL)
			continue;
		count++;
		if(snprintf(t->addrfile, sizeof t->addrfile, "%s/%s", t->bus,
		   de->d_name) >= (int)sizeof t->addrfile){
			fail("IBus address path is too long");
			ok = 0;
			break;
		}
	}
	if(errno != 0){
		fail("read IBus directory %s: %s", t->bus, strerror(errno));
		ok = 0;
	}
	if(closedir(dir) < 0){
		fail("close IBus directory %s: %s", t->bus, strerror(errno));
		ok = 0;
	}
	if(!ok)
		return -1;
	if(count > 1){
		fail("found %d IBus address files", count);
		return -1;
	}
	return count;
}

static int
readfile(char *path, char *buf, size_t cap, size_t *nread)
{
	struct stat st;
	ssize_t n;
	size_t off;
	char extra;
	int fd, ok;

	fd = open(path, O_RDONLY|O_CLOEXEC);
	if(fd < 0)
		return fail("open %s: %s", path, strerror(errno));
	ok = 0;
	if(fstat(fd, &st) < 0){
		fail("stat open address file %s: %s", path, strerror(errno));
		goto out;
	}
	if(st.st_size < 0 || (uintmax_t)st.st_size >= cap){
		fail("address file %s is too large", path);
		goto out;
	}
	off = 0;
	while(off < (size_t)st.st_size){
		n = read(fd, buf + off, (size_t)st.st_size - off);
		if(n < 0 && errno == EINTR)
			continue;
		if(n <= 0){
			fail("read %s: %s", path,
				n == 0 ? "unexpected EOF" : strerror(errno));
			goto out;
		}
		off += n;
	}
	do
		n = read(fd, &extra, 1);
	while(n < 0 && errno == EINTR);
	if(n != 0){
		fail("address file %s changed while being read", path);
		goto out;
	}
	buf[off] = '\0';
	*nread = off;
	ok = 1;
out:
	if(close(fd) < 0){
		fail("close %s: %s", path, strerror(errno));
		ok = 0;
	}
	return ok;
}

static int
parseaddress(char *contents, size_t ncontents, char *address, size_t naddress,
	pid_t *declared)
{
	char found[512];
	long pid;
	int consumed;

	consumed = -1;
	if(strlen(contents) != ncontents ||
	   sscanf(contents, "IBUS_ADDRESS=%511[^\n]\nIBUS_DAEMON_PID=%ld\n%n",
	   found, &pid, &consumed) != 2 || consumed != (int)ncontents || pid <= 0)
		return fail("invalid IBus address file contents");
	if(snprintf(address, naddress, "%s", found) >= (int)naddress)
		return fail("private IBus address is too long");
	*declared = (pid_t)pid;
	return 1;
}

static int
privateaddress(char *address, pid_t pid)
{
	char prefix[96];
	size_t n;

	if(snprintf(prefix, sizeof prefix, "unix:abstract=strans-%ld",
	   (long)pid) >= (int)sizeof prefix)
		return fail("private IBus prefix is too long");
	n = strlen(prefix);
	return (strncmp(address, prefix, n) == 0 && address[n] == ',') ||
		fail("IBus address is not the daemon's private abstract address");
}

static int
childalive(pid_t *pid, char *which)
{
	int n, status;

	do
		n = waitpid(*pid, &status, WNOHANG);
	while(n < 0 && errno == EINTR);
	if(n == 0)
		return 1;
	if(n == *pid){
		fail("%s daemon exited unexpectedly with wait status %#x", which,
			status);
		*pid = -1;
		return 0;
	}
	if(n < 0){
		fail("check %s daemon %ld: %s", which, (long)*pid,
			strerror(errno));
		if(errno == ECHILD)
			*pid = -1;
		return 0;
	}
	return fail("waitpid returned the wrong %s child", which);
}

static int
waitready(Test *t, pid_t *child, int errfd, char *which)
{
	struct pollfd pfd;
	struct stat st;
	char contents[2048], *address;
	size_t ncontents;
	pid_t declared;
	int count, n, socketready, timeout;
	int64_t deadline;

	deadline = nowms();
	if(deadline < 0)
		return fail("read monotonic clock before %s startup: %s", which,
			strerror(errno));
	deadline += Starttimeout;
	address = child == &t->first ? t->firstaddress : t->secondaddress;
	for(;;){
		socketready = lstat(t->socket, &st) == 0 && S_ISSOCK(st.st_mode) &&
			(st.st_mode & 0777) == 0600;
		count = findaddress(t);
		if(count < 0)
			return 0;
		if(socketready && count == 1 &&
		   readfile(t->addrfile, contents, sizeof contents, &ncontents) &&
		   parseaddress(contents, ncontents, address, 512,
		   &declared) && declared == *child && privateaddress(address, declared))
			return 1;
		if(!childalive(child, which))
			return 0;
		timeout = leftms(deadline);
		if(timeout < 0)
			return fail("read monotonic clock during %s startup: %s", which,
				strerror(errno));
		if(timeout == 0)
			return fail("timed out waiting for both %s daemon endpoints", which);
		pfd.fd = errfd;
		pfd.events = POLLIN|POLLHUP;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout > 20 ? 20 : timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n < 0)
			return fail("poll %s daemon readiness: %s", which,
				strerror(errno));
		if(pfd.revents & (POLLERR|POLLNVAL))
			return fail("%s stderr pipe became unusable: %#x", which,
				pfd.revents);
		if(pfd.revents & (POLLIN|POLLHUP)){
			if(child == &t->first)
				readpipe(errfd, t->firsterr, sizeof t->firsterr,
					&t->nfirsterr);
			else
				readpipe(errfd, t->seconderr, sizeof t->seconderr,
					&t->nseconderr);
		}
	}
}

static int
connectsocket(char *path, int64_t deadline)
{
	struct sockaddr_un addr;
	struct pollfd pfd;
	socklen_t nerr;
	int err, fd, flags, n;

	memset(&addr, 0, sizeof addr);
	addr.sun_family = AF_UNIX;
	if(snprintf(addr.sun_path, sizeof addr.sun_path, "%s", path)
	   >= (int)sizeof addr.sun_path){
		errno = ENAMETOOLONG;
		return -1;
	}
	fd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0);
	if(fd < 0)
		return -1;
	if(connect(fd, (struct sockaddr*)&addr, sizeof addr) < 0 &&
	   errno != EINPROGRESS){
		err = errno;
		close(fd);
		errno = err;
		return -1;
	}
	for(;;){
		n = leftms(deadline);
		if(n <= 0){
			close(fd);
			errno = n < 0 ? EIO : ETIMEDOUT;
			return -1;
		}
		pfd.fd = fd;
		pfd.events = POLLOUT;
		pfd.revents = 0;
		n = poll(&pfd, 1, n);
		if(n < 0 && errno == EINTR)
			continue;
		if(n <= 0){
			err = n == 0 ? ETIMEDOUT : errno;
			close(fd);
			errno = err;
			return -1;
		}
		break;
	}
	err = 0;
	nerr = sizeof err;
	if(getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &nerr) < 0 || err != 0){
		if(err == 0)
			err = errno;
		close(fd);
		errno = err;
		return -1;
	}
	flags = fcntl(fd, F_GETFL);
	if(flags < 0 || fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0){
		err = errno;
		close(fd);
		errno = err;
		return -1;
	}
	return fd;
}

/* 1 is complete, 0 is peer closure, and -1 is a timeout or I/O error. */
static int
readuntil(int fd, void *buf, size_t n, int64_t deadline)
{
	struct pollfd pfd;
	unsigned char *p;
	ssize_t r;
	int timeout;

	p = buf;
	while(n > 0){
		timeout = leftms(deadline);
		if(timeout <= 0){
			errno = timeout < 0 ? EIO : ETIMEDOUT;
			return -1;
		}
		pfd.fd = fd;
		pfd.events = POLLIN;
		pfd.revents = 0;
		r = poll(&pfd, 1, timeout);
		if(r < 0 && errno == EINTR)
			continue;
		if(r <= 0){
			if(r == 0)
				errno = ETIMEDOUT;
			return -1;
		}
		r = recv(fd, p, n, 0);
		if(r < 0 && errno == EINTR)
			continue;
		if(r < 0)
			return -1;
		if(r == 0)
			return 0;
		p += r;
		n -= r;
	}
	return 1;
}

static int
ipcrequest(int fd, uint32_t mod, uint32_t key, unsigned char *want,
	size_t nwant, char *where)
{
	unsigned char req[Ipcreqsz], got[16];
	int rv;

	if(nwant > sizeof got)
		return fail("%s expected IPC response is too large", where);
	ipcpackreq(req, 1, mod, key);
	if(ipcsend(fd, req, sizeof req) < 0)
		return fail("send %s IPC request: %s", where, strerror(errno));
	rv = readuntil(fd, got, nwant, nowms() + Calltimeout);
	if(rv != 1)
		return fail("read %s IPC response: %s", where,
			rv == 0 ? "peer closed" : strerror(errno));
	if(memcmp(got, want, nwant) != 0)
		return fail("%s IPC response did not match", where);
	return 1;
}

static int
ipcprobe(int fd, char *where)
{
	unsigned char empty[] = {0, 0, 0, 0, 0};

	return ipcrequest(fd, 0, Kmodfirst, empty, sizeof empty, where);
}

static void
closebus(DBusConnection **conn)
{
	if(*conn == NULL)
		return;
	dbus_connection_close(*conn);
	dbus_connection_unref(*conn);
	*conn = NULL;
}

static DBusMessage*
sendcall(DBusConnection *conn, DBusMessage *m, char *where)
{
	DBusMessage *reply;
	DBusPendingCall *pending;
	int timeout;
	int64_t deadline;

	if(m == NULL){
		fail("allocate %s call", where);
		return NULL;
	}
	pending = NULL;
	if(!dbus_connection_send_with_reply(conn, m, &pending, Calltimeout) ||
	   pending == NULL){
		dbus_message_unref(m);
		fail("queue %s call", where);
		return NULL;
	}
	dbus_message_unref(m);
	deadline = nowms() + Calltimeout;
	while(!dbus_pending_call_get_completed(pending)){
		timeout = leftms(deadline);
		if(timeout <= 0){
			fail("timed out waiting for %s", where);
			goto fail;
		}
		if(!dbus_connection_read_write_dispatch(conn, timeout)){
			fail("connection closed waiting for %s", where);
			goto fail;
		}
	}
	reply = dbus_pending_call_steal_reply(pending);
	dbus_pending_call_unref(pending);
	if(reply == NULL){
		fail("%s completed without a reply", where);
		return NULL;
	}
	if(dbus_message_get_type(reply) != DBUS_MESSAGE_TYPE_METHOD_RETURN){
		fail("%s returned D-Bus message type %d", where,
			dbus_message_get_type(reply));
		dbus_message_unref(reply);
		return NULL;
	}
	return reply;
fail:
	dbus_pending_call_cancel(pending);
	dbus_pending_call_unref(pending);
	return NULL;
}

static DBusConnection*
openbus(char *address, char *where)
{
	DBusConnection *conn;
	DBusError err;

	dbus_error_init(&err);
	conn = dbus_connection_open_private(address, &err);
	if(conn == NULL){
		fail("open %s private IBus connection: %s", where,
			err.message != NULL ? err.message : "D-Bus error");
		dbus_error_free(&err);
		return NULL;
	}
	dbus_error_free(&err);
	dbus_connection_set_exit_on_disconnect(conn, FALSE);
	return conn;
}

static int
hello(DBusConnection *conn, char *name, size_t nname, char *where)
{
	DBusMessage *m, *reply;
	DBusError err;
	const char *s;

	m = dbus_message_new_method_call("org.freedesktop.DBus",
		"/org/freedesktop/DBus", "org.freedesktop.DBus", "Hello");
	reply = sendcall(conn, m, where);
	if(reply == NULL)
		return 0;
	dbus_error_init(&err);
	if(!dbus_message_has_signature(reply, "s") ||
	   !dbus_message_get_args(reply, &err, DBUS_TYPE_STRING, &s,
	   DBUS_TYPE_INVALID)){
		dbus_message_unref(reply);
		dbus_error_free(&err);
		return fail("%s returned an invalid Hello reply", where);
	}
	if(snprintf(name, nname, "%s", s) >= (int)nname){
		dbus_error_free(&err);
		dbus_message_unref(reply);
		return fail("%s Hello name is too long", where);
	}
	dbus_error_free(&err);
	dbus_message_unref(reply);
	return name[0] == ':' || fail("%s returned invalid Hello name %s",
		where, name);
}

static int
createcontext(DBusConnection *conn, char *path, size_t npath, char *where)
{
	DBusMessage *m, *reply;
	DBusError err;
	const char *client, *p;

	client = "daemon-restart-test";
	m = dbus_message_new_method_call("org.freedesktop.IBus",
		"/org/freedesktop/IBus", "org.freedesktop.IBus",
		"CreateInputContext");
	if(m == NULL || !dbus_message_append_args(m, DBUS_TYPE_STRING, &client,
	   DBUS_TYPE_INVALID)){
		if(m != NULL)
			dbus_message_unref(m);
		return fail("build %s CreateInputContext call", where);
	}
	reply = sendcall(conn, m, where);
	if(reply == NULL)
		return 0;
	dbus_error_init(&err);
	if(!dbus_message_has_signature(reply, "o") ||
	   !dbus_message_get_args(reply, &err, DBUS_TYPE_OBJECT_PATH, &p,
	   DBUS_TYPE_INVALID)){
		dbus_message_unref(reply);
		dbus_error_free(&err);
		return fail("%s returned an invalid context reply", where);
	}
	if(snprintf(path, npath, "%s", p) >= (int)npath){
		dbus_error_free(&err);
		dbus_message_unref(reply);
		return fail("%s context path is too long", where);
	}
	dbus_error_free(&err);
	dbus_message_unref(reply);
	return path[0] == '/' || fail("%s returned invalid context path %s",
		where, path);
}

static DBusMessage*
contextcall(char *path, char *member)
{
	return dbus_message_new_method_call("org.freedesktop.IBus", path,
		"org.freedesktop.IBus.InputContext", member);
}

static int
focusin(DBusConnection *conn, char *path)
{
	DBusMessage *reply;

	reply = sendcall(conn, contextcall(path, "FocusIn"),
		"replacement FocusIn");
	if(reply == NULL)
		return 0;
	if(!dbus_message_has_signature(reply, "")){
		dbus_message_unref(reply);
		return fail("replacement FocusIn returned a nonempty reply");
	}
	dbus_message_unref(reply);
	return 1;
}

static int
keycall(DBusConnection *conn, char *path, dbus_uint32_t sym,
	dbus_uint32_t state, int expected, char *where)
{
	DBusMessage *m, *reply;
	DBusError err;
	dbus_uint32_t code;
	dbus_bool_t eaten;

	code = 0;
	m = contextcall(path, "ProcessKeyEvent");
	if(m == NULL || !dbus_message_append_args(m,
	   DBUS_TYPE_UINT32, &sym, DBUS_TYPE_UINT32, &code,
	   DBUS_TYPE_UINT32, &state, DBUS_TYPE_INVALID)){
		if(m != NULL)
			dbus_message_unref(m);
		return fail("build %s ProcessKeyEvent call", where);
	}
	reply = sendcall(conn, m, where);
	if(reply == NULL)
		return 0;
	dbus_error_init(&err);
	if(!dbus_message_has_signature(reply, "b") ||
	   !dbus_message_get_args(reply, &err, DBUS_TYPE_BOOLEAN, &eaten,
	   DBUS_TYPE_INVALID)){
		dbus_message_unref(reply);
		dbus_error_free(&err);
		return fail("%s returned an invalid ProcessKeyEvent reply", where);
	}
	dbus_error_free(&err);
	dbus_message_unref(reply);
	return (eaten != FALSE) == (expected != 0) ||
		fail("%s eaten=%d, expected %d", where, eaten != FALSE, expected);
}

static int
openpersistent(Test *t)
{
	char name[32], path[96];
	int64_t deadline;

	deadline = nowms() + Calltimeout;
	t->ipcfirst = connectsocket(t->socket, deadline);
	if(t->ipcfirst < 0)
		return fail("connect persistent IPC client: %s", strerror(errno));
	if(!ipcprobe(t->ipcfirst, "persistent pre-crash"))
		return 0;
	t->busfirst = openbus(t->firstaddress, "persistent pre-crash");
	if(t->busfirst == NULL ||
	   !hello(t->busfirst, name, sizeof name, "persistent pre-crash Hello") ||
	   !createcontext(t->busfirst, path, sizeof path,
	   "persistent pre-crash context"))
		return 0;
	if(!dbus_connection_get_unix_fd(t->busfirst, &t->busfirstfd))
		return fail("persistent IBus connection has no Unix descriptor");
	return 1;
}

static int
hardcrash(Test *t)
{
	struct pollfd pfd;
	pid_t pid;
	int e, n, status, timeout;
	int64_t deadline;

	if(!childalive(&t->first, "first"))
		return 0;
	pid = t->first;
	if(kill(pid, SIGKILL) < 0){
		e = errno;
		do
			n = waitpid(pid, &status, WNOHANG);
		while(n < 0 && errno == EINTR);
		if(n == pid){
			t->first = -1;
			return fail("first daemon exited before planned SIGKILL with status %#x",
				status);
		}
		if(n < 0 && errno == ECHILD)
			t->first = -1;
		return fail("kill -9 first daemon %ld: %s", (long)pid, strerror(e));
	}
	deadline = nowms();
	if(deadline < 0)
		return fail("read monotonic clock before first daemon crash: %s",
			strerror(errno));
	deadline += Stoptimeout;
	for(;;){
		do
			n = waitpid(pid, &status, WNOHANG);
		while(n < 0 && errno == EINTR);
		if(n == pid){
			t->first = -1;
			break;
		}
		if(n < 0){
			e = errno;
			if(e == ECHILD)
				t->first = -1;
			return fail("reap first daemon %ld after SIGKILL: %s", (long)pid,
				strerror(e));
		}
		timeout = leftms(deadline);
		if(timeout < 0)
			return fail("read monotonic clock during first daemon crash: %s",
				strerror(errno));
		if(timeout == 0)
			return fail("first daemon %ld was not reaped after SIGKILL",
				(long)pid);
		pfd.fd = t->firsterrfd;
		pfd.events = POLLIN|POLLHUP;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n < 0)
			return fail("poll first daemon crash: %s", strerror(errno));
		if(pfd.revents & (POLLERR|POLLNVAL))
			return fail("first daemon stderr pipe became unusable: %#x",
				pfd.revents);
		if(pfd.revents & (POLLIN|POLLHUP))
			readpipe(t->firsterrfd, t->firsterr, sizeof t->firsterr,
				&t->nfirsterr);
	}
	if(!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL)
		return fail("first daemon hard crash had wait status %#x", status);
	readpipe(t->firsterrfd, t->firsterr, sizeof t->firsterr, &t->nfirsterr);
	return 1;
}

static int
waitipcclosed(Test *t)
{
	struct pollfd pfd;
	unsigned char byte;
	ssize_t n;
	int timeout;
	int64_t deadline;

	deadline = nowms() + Calltimeout;
	for(;;){
		timeout = leftms(deadline);
		if(timeout <= 0)
			return fail("persistent IPC connection did not disconnect promptly");
		pfd.fd = t->ipcfirst;
		pfd.events = POLLIN|POLLHUP|POLLERR;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n <= 0)
			return fail("poll persistent IPC disconnection: %s",
				n == 0 ? "timed out" : strerror(errno));
		n = recv(t->ipcfirst, &byte, 1, MSG_PEEK);
		if(n == 0 || (n < 0 && (errno == ECONNRESET || errno == ENOTCONN ||
		   errno == EPIPE)))
			return 1;
		if(n < 0 && (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK))
			continue;
		if(n < 0)
			return fail("read persistent IPC disconnection: %s",
				strerror(errno));
		return fail("persistent IPC connection retained unread data after crash");
	}
}

static int
waitbusclosed(Test *t)
{
	struct pollfd pfd;
	int n, timeout;
	int64_t deadline;

	deadline = nowms() + Calltimeout;
	for(;;){
		if(!dbus_connection_get_is_connected(t->busfirst))
			return 1;
		timeout = leftms(deadline);
		if(timeout <= 0)
			return fail("persistent IBus connection did not disconnect promptly");
		pfd.fd = t->busfirstfd;
		pfd.events = POLLIN|POLLHUP|POLLERR;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n <= 0)
			return fail("poll persistent IBus disconnection: %s",
				n == 0 ? "timed out" : strerror(errno));
		if(pfd.revents & POLLNVAL)
			return fail("persistent IBus descriptor became invalid");
		dbus_connection_read_write_dispatch(t->busfirst, 0);
	}
}

static int
checkstale(Test *t)
{
	struct stat st;
	char contents[2048], address[512];
	size_t ncontents;
	pid_t declared;

	if(lstat(t->socket, &st) < 0)
		return fail("stale IPC socket disappeared: %s", strerror(errno));
	if(!S_ISSOCK(st.st_mode))
		return fail("stale IPC endpoint is not a socket");
	if(lstat(t->addrfile, &st) < 0)
		return fail("stale IBus address file disappeared: %s", strerror(errno));
	if(!readfile(t->addrfile, contents, sizeof contents, &ncontents) ||
	   !parseaddress(contents, ncontents, address, sizeof address, &declared))
		return 0;
	if(declared != t->firstpid || strcmp(address, t->firstaddress) != 0)
		return fail("stale IBus address no longer names the dead first daemon");
	return 1;
}

static int
helpermain(char *address)
{
	DBusConnection *conn;
	DBusError err;

	dbus_error_init(&err);
	conn = dbus_connection_open_private(address, &err);
	if(conn != NULL){
		dbus_connection_set_exit_on_disconnect(conn, FALSE);
		dbus_connection_close(conn);
		dbus_connection_unref(conn);
		dbus_error_free(&err);
		fprintf(stderr, "old private address unexpectedly accepted a connection\n");
		return 1;
	}
	if(!dbus_error_is_set(&err)){
		fprintf(stderr, "old private address failed without a D-Bus error\n");
		dbus_error_free(&err);
		return 2;
	}
	dbus_error_free(&err);
	return 0;
}

static int
killreap(pid_t *child, int errfd, char *which)
{
	struct pollfd pfd;
	pid_t pid;
	int e, n, ok, status, timeout;
	int64_t deadline;

	if(*child <= 0)
		return 1;
	pid = *child;
	ok = 1;
	if(kill(pid, SIGKILL) < 0 && errno != ESRCH){
		fail("kill -9 %s %ld: %s", which, (long)pid, strerror(errno));
		ok = 0;
	}
	deadline = nowms();
	if(deadline < 0){
		fail("read monotonic clock before reaping %s %ld: %s", which,
			(long)pid, strerror(errno));
		ok = 0;
	}else
		deadline += Stoptimeout;
	while(deadline >= 0){
		do
			n = waitpid(pid, &status, WNOHANG);
		while(n < 0 && errno == EINTR);
		if(n == pid){
			*child = -1;
			return ok;
		}
		if(n < 0){
			e = errno;
			if(e == ECHILD)
				*child = -1;
			fail("reap %s %ld after SIGKILL: %s", which, (long)pid,
				strerror(e));
			return 0;
		}
		timeout = leftms(deadline);
		if(timeout < 0){
			fail("read monotonic clock while reaping %s %ld: %s", which,
				(long)pid, strerror(errno));
			ok = 0;
			break;
		}
		if(timeout == 0){
			fail("%s %ld was not reaped promptly after SIGKILL", which,
				(long)pid);
			ok = 0;
			break;
		}
		pfd.fd = errfd;
		pfd.events = 0;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n < 0){
			fail("poll %s %ld after SIGKILL: %s", which, (long)pid,
				strerror(errno));
			ok = 0;
			break;
		}
		if(pfd.revents & (POLLERR|POLLNVAL)){
			fail("%s %ld stderr pipe became unusable: %#x", which,
				(long)pid, pfd.revents);
			ok = 0;
			break;
		}
	}
	/* Keep exact child ownership until the mandatory post-SIGKILL reap. */
	do
		n = waitpid(pid, &status, 0);
	while(n < 0 && errno == EINTR);
	if(n == pid)
		*child = -1;
	else{
		e = errno;
		if(n < 0 && e == ECHILD)
			*child = -1;
		fail("final reap of %s %ld after SIGKILL: %s", which, (long)pid,
			n < 0 ? strerror(e) : "wrong child");
		ok = 0;
	}
	return ok;
}

static int
rejectold(Test *t, char *self, char *where)
{
	struct pollfd pfd;
	int errpipe[2], fd, n, status, timeout;
	long maxfd;
	pid_t pid;
	int64_t deadline;

	if(pipe2(errpipe, O_CLOEXEC|O_NONBLOCK) < 0)
		return fail("pipe old-address helper: %s", strerror(errno));
	pid = fork();
	if(pid < 0){
		close(errpipe[0]);
		close(errpipe[1]);
		return fail("fork old-address helper: %s", strerror(errno));
	}
	if(pid == 0){
		close(errpipe[0]);
		if(!childstderr(errpipe[1]))
			_exit(126);
		if(close_range(3, UINT_MAX, 0) < 0){
			maxfd = sysconf(_SC_OPEN_MAX);
			if(maxfd < 0)
				maxfd = 1024;
			for(fd = 3; fd < maxfd; fd++)
				close(fd);
		}
		execl(self, self, "--reject-address", t->firstaddress, (char*)0);
		dprintf(STDERR_FILENO, "exec %s: %s\n", self, strerror(errno));
		_exit(127);
	}
	close(errpipe[1]);
	t->helper = pid;
	t->helpererrfd = errpipe[0];
	t->nhelpererr = 0;
	t->helpererr[0] = '\0';
	deadline = nowms() + Calltimeout;
	for(;;){
		do
			n = waitpid(pid, &status, WNOHANG);
		while(n < 0 && errno == EINTR);
		if(n == pid){
			t->helper = -1;
			break;
		}
		if(n < 0){
			fail("wait old-address helper %ld: %s", (long)pid,
				strerror(errno));
			if(errno == ECHILD)
				t->helper = -1;
			else
				killreap(&t->helper, t->helpererrfd,
					"old-address helper");
			return 0;
		}
		timeout = leftms(deadline);
		if(timeout <= 0){
			fail("old private address open did not fail promptly %s", where);
			killreap(&t->helper, t->helpererrfd, "old-address helper");
			return 0;
		}
		pfd.fd = t->helpererrfd;
		pfd.events = POLLIN|POLLHUP;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n < 0){
			fail("poll old-address helper: %s", strerror(errno));
			killreap(&t->helper, t->helpererrfd, "old-address helper");
			return 0;
		}
		if(pfd.revents & (POLLERR|POLLNVAL)){
			fail("old-address helper pipe became unusable: %#x", pfd.revents);
			killreap(&t->helper, t->helpererrfd, "old-address helper");
			return 0;
		}
		if(pfd.revents & (POLLIN|POLLHUP))
			readpipe(t->helpererrfd, t->helpererr, sizeof t->helpererr,
				&t->nhelpererr);
	}
	readpipe(t->helpererrfd, t->helpererr, sizeof t->helpererr,
		&t->nhelpererr);
	if(close(t->helpererrfd) < 0){
		t->helpererrfd = -1;
		return fail("close old-address helper pipe: %s", strerror(errno));
	}
	t->helpererrfd = -1;
	if(!WIFEXITED(status) || WEXITSTATUS(status) != 0)
		return fail("old private address helper failed %s with status %#x",
			where, status);
	return 1;
}

static int
openreplacement(Test *t)
{
	unsigned char selectjp[] = {1, 0, 0, 0, 0};
	unsigned char keyk[] = {1, 0, 0, 1, 0, 'k'};
	char name[32], path[96];

	t->ipcnew = connectsocket(t->socket, nowms() + Calltimeout);
	if(t->ipcnew < 0)
		return fail("connect replacement IPC client: %s", strerror(errno));
	if(!ipcrequest(t->ipcnew, Mctrl, 'n', selectjp, sizeof selectjp,
	   "replacement select Japanese") ||
	   !ipcrequest(t->ipcnew, 0, 'k', keyk, sizeof keyk,
	   "replacement real key"))
		return 0;
	t->busnew = openbus(t->secondaddress, "replacement");
	if(t->busnew == NULL ||
	   !hello(t->busnew, name, sizeof name, "replacement Hello") ||
	   !createcontext(t->busnew, path, sizeof path, "replacement context") ||
	   !focusin(t->busnew, path) ||
	   !keycall(t->busnew, path, 'n', Ctrlmask, 1,
	   "replacement select Japanese") ||
	   !keycall(t->busnew, path, 'k', 0, 1, "replacement real key"))
		return 0;
	return childalive(&t->second, "second");
}

static int
stopsecond(Test *t)
{
	struct pollfd pfd;
	pid_t pid;
	int n, ok, reaped, status, timeout;
	int64_t deadline;

	if(t->second <= 0)
		return 1;
	if(!childalive(&t->second, "second"))
		return 0;
	pid = t->second;
	ok = 1;
	if(kill(pid, SIGTERM) < 0){
		fail("kill second daemon %ld: %s", (long)pid, strerror(errno));
		ok = 0;
	}
	reaped = 0;
	deadline = nowms() + Stoptimeout;
	while(t->second > 0){
		do
			n = waitpid(pid, &status, WNOHANG);
		while(n < 0 && errno == EINTR);
		if(n == pid){
			reaped = 1;
			t->second = -1;
			break;
		}
		if(n < 0){
			fail("waitpid second daemon %ld: %s", (long)pid,
				strerror(errno));
			if(errno == ECHILD)
				t->second = -1;
			else if(!killreap(&t->second, t->seconderrfd, "second daemon"))
				ok = 0;
			ok = 0;
			break;
		}
		timeout = leftms(deadline);
		if(timeout <= 0){
			fail("second daemon %ld did not stop after SIGTERM", (long)pid);
			ok = 0;
			if(!killreap(&t->second, t->seconderrfd, "second daemon"))
				ok = 0;
			break;
		}
		pfd.fd = t->seconderrfd;
		pfd.events = POLLIN|POLLHUP;
		pfd.revents = 0;
		n = poll(&pfd, 1, timeout);
		if(n < 0 && errno == EINTR)
			continue;
		if(n < 0 || (pfd.revents & (POLLERR|POLLNVAL))){
			fail("poll second daemon termination: %s",
				n < 0 ? strerror(errno) : "stderr pipe failure");
			ok = 0;
			if(!killreap(&t->second, t->seconderrfd, "second daemon"))
				ok = 0;
			break;
		}
		if(pfd.revents & (POLLIN|POLLHUP))
			readpipe(t->seconderrfd, t->seconderr, sizeof t->seconderr,
				&t->nseconderr);
	}
	if(reaped &&
	   !((WIFSIGNALED(status) && WTERMSIG(status) == SIGTERM) ||
	   (WIFEXITED(status) && WEXITSTATUS(status) == 1))){
		fail("second daemon exited with unexpected wait status %#x", status);
		ok = 0;
	}
	return ok;
}

static int clearfd(int);

static int
removeentry(int dirfd, char *name)
{
	struct stat st;
	int fd, ok;

	if(fstatat(dirfd, name, &st, AT_SYMLINK_NOFOLLOW) < 0)
		return fail("stat cleanup entry %s: %s", name, strerror(errno));
	if(!S_ISDIR(st.st_mode)){
		if(unlinkat(dirfd, name, 0) == 0 || errno == ENOENT)
			return 1;
		return fail("remove cleanup entry %s: %s", name, strerror(errno));
	}
	fd = openat(dirfd, name, O_RDONLY|O_CLOEXEC|O_DIRECTORY|O_NOFOLLOW);
	if(fd < 0)
		return fail("open cleanup directory %s: %s", name, strerror(errno));
	ok = clearfd(fd);
	if(close(fd) < 0){
		fail("close cleanup directory %s: %s", name, strerror(errno));
		ok = 0;
	}
	if(unlinkat(dirfd, name, AT_REMOVEDIR) < 0 && errno != ENOENT){
		fail("remove cleanup directory %s: %s", name, strerror(errno));
		ok = 0;
	}
	return ok;
}

static int
clearfd(int fd)
{
	DIR *dir;
	struct dirent *de;
	int copy, ok;

	copy = dup(fd);
	if(copy < 0)
		return fail("duplicate cleanup directory: %s", strerror(errno));
	dir = fdopendir(copy);
	if(dir == NULL){
		close(copy);
		return fail("open cleanup directory stream: %s", strerror(errno));
	}
	ok = 1;
	errno = 0;
	while((de = readdir(dir)) != NULL){
		if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
			continue;
		if(!removeentry(fd, de->d_name))
			ok = 0;
		errno = 0;
	}
	if(errno != 0){
		fail("read cleanup directory: %s", strerror(errno));
		ok = 0;
	}
	if(closedir(dir) < 0){
		fail("close cleanup directory stream: %s", strerror(errno));
		ok = 0;
	}
	return ok;
}

static int
removeroot(Test *t)
{
	struct stat st;
	int fd, ok;

	if(t->root[0] == '\0')
		return 1;
	fd = open(t->root, O_RDONLY|O_CLOEXEC|O_DIRECTORY|O_NOFOLLOW);
	if(fd < 0)
		return errno == ENOENT || fail("open cleanup root %s: %s", t->root,
			strerror(errno));
	ok = clearfd(fd);
	if(close(fd) < 0){
		fail("close cleanup root %s: %s", t->root, strerror(errno));
		ok = 0;
	}
	if(rmdir(t->root) < 0 && errno != ENOENT){
		fail("rmdir %s: %s", t->root, strerror(errno));
		ok = 0;
	}
	errno = 0;
	if(lstat(t->root, &st) == 0 || errno != ENOENT){
		fail("temporary root still exists after cleanup");
		ok = 0;
	}
	return ok;
}

static int
cleanup(Test *t)
{
	int ok;

	ok = 1;
	closebus(&t->busnew);
	closebus(&t->busfirst);
	if(t->ipcnew >= 0){
		if(close(t->ipcnew) < 0){
			fail("close replacement IPC client: %s", strerror(errno));
			ok = 0;
		}
		t->ipcnew = -1;
	}
	if(t->ipcfirst >= 0){
		if(close(t->ipcfirst) < 0){
			fail("close persistent IPC client: %s", strerror(errno));
			ok = 0;
		}
		t->ipcfirst = -1;
	}
	if(t->helper > 0 && !killreap(&t->helper, t->helpererrfd,
	   "old-address helper"))
		ok = 0;
	if(t->first > 0 && !killreap(&t->first, t->firsterrfd, "first daemon"))
		ok = 0;
	if(!stopsecond(t))
		ok = 0;
	readpipe(t->firsterrfd, t->firsterr, sizeof t->firsterr, &t->nfirsterr);
	readpipe(t->seconderrfd, t->seconderr, sizeof t->seconderr,
		&t->nseconderr);
	readpipe(t->helpererrfd, t->helpererr, sizeof t->helpererr,
		&t->nhelpererr);
	if(t->firsterrfd >= 0){
		if(close(t->firsterrfd) < 0){
			fail("close first daemon stderr: %s", strerror(errno));
			ok = 0;
		}
		t->firsterrfd = -1;
	}
	if(t->seconderrfd >= 0){
		if(close(t->seconderrfd) < 0){
			fail("close second daemon stderr: %s", strerror(errno));
			ok = 0;
		}
		t->seconderrfd = -1;
	}
	if(t->helpererrfd >= 0){
		if(close(t->helpererrfd) < 0){
			fail("close old-address helper stderr: %s", strerror(errno));
			ok = 0;
		}
		t->helpererrfd = -1;
	}
	if(!removeroot(t))
		ok = 0;
	return ok;
}

static int
runrestart(Test *t, char *self, char *program, char *mapdir)
{
	if(!startchild(t, program, mapdir, &t->first, &t->firsterrfd))
		return 0;
	t->firstpid = t->first;
	if(!waitready(t, &t->first, t->firsterrfd, "first") ||
	   !openpersistent(t))
		return 0;
	if(!hardcrash(t) || !waitipcclosed(t) || !waitbusclosed(t) ||
	   !checkstale(t) || !rejectold(t, self, "after hard crash"))
		return 0;
	closebus(&t->busfirst);
	if(close(t->ipcfirst) < 0)
		return fail("close disconnected persistent IPC client: %s",
			strerror(errno));
	t->ipcfirst = -1;
	if(!startchild(t, program, mapdir, &t->second, &t->seconderrfd) ||
	   !waitready(t, &t->second, t->seconderrfd, "second") ||
	   !openreplacement(t))
		return 0;
	return childalive(&t->second, "second");
}

int
main(int argc, char **argv)
{
	Test test;
	int ok;

	if(argc == 3 && strcmp(argv[1], "--reject-address") == 0)
		return helpermain(argv[2]);
	if(argc != 3){
		fprintf(stderr, "usage: daemon_restart_test strans mapdir\n");
		return 2;
	}
	ok = setup(&test);
	if(ok)
		ok = runrestart(&test, argv[0], argv[1], argv[2]);
	if(!cleanup(&test))
		ok = 0;
	if(!ok){
		showerrors(&test);
		return 1;
	}
	printf("daemon hard-crash endpoint recovery: ok\n");
	return 0;
}