add new masked cancellation mode

this is a new extension which is presently intended only for
experimental and internal libc use. interface and behavior details may
change subject to feedback and experience from using it internally.

the basic concept for the new PTHREAD_CANCEL_MASKED state is that the
first cancellation point to observe the cancellation request fails
with an errno value of ECANCELED rather than acting on cancellation,
allowing the caller to process the status and choose whether/how to
act upon it.

include/pthread.h

@@ -63,6 +63,7 @@ extern "C" {
 
 #define PTHREAD_CANCEL_ENABLE 0
 #define PTHREAD_CANCEL_DISABLE 1
+#define PTHREAD_CANCEL_MASKED 2
 
 #define PTHREAD_CANCEL_DEFERRED 0
 #define PTHREAD_CANCEL_ASYNCHRONOUS 1

src/thread/pthread_cancel.c

@@ -2,9 +2,13 @@
 #include "syscall.h"
 #include "libc.h"
 
-void __cancel()
+long __cancel()
 {
-	pthread_exit(PTHREAD_CANCELED);
+	pthread_t self = __pthread_self();
+	if (self->canceldisable == PTHREAD_CANCEL_ENABLE || self->cancelasync)
+		pthread_exit(PTHREAD_CANCELED);
+	self->canceldisable = PTHREAD_CANCEL_DISABLE;
+	return -ECANCELED;
 }
 
 /* If __syscall_cp_asm has adjusted the stack pointer, it must provide a
@@ -23,13 +27,16 @@ long __syscall_cp_c(syscall_arg_t nr,
 {
 	pthread_t self;
 	long r;
+	int st;
 
-	if (!libc.has_thread_pointer || (self = __pthread_self())->canceldisable)
+	if (!libc.has_thread_pointer || (st=(self=__pthread_self())->canceldisable)
+	    && (st==PTHREAD_CANCEL_DISABLE || nr==SYS_close))
 		return __syscall(nr, u, v, w, x, y, z);
 
 	r = __syscall_cp_asm(&self->cancel, nr, u, v, w, x, y, z);
-	if (r==-EINTR && nr!=SYS_close && self->cancel && !self->canceldisable)
-		__cancel();
+	if (r==-EINTR && nr!=SYS_close && self->cancel &&
+	    self->canceldisable != PTHREAD_CANCEL_DISABLE)
+		r = __cancel();
 	return r;
 }
 
@@ -47,14 +54,13 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx)
 	extern const char __cp_begin[1], __cp_end[1];
 
 	a_barrier();
-	if (!self->cancel || self->canceldisable) return;
+	if (!self->cancel || self->canceldisable == PTHREAD_CANCEL_DISABLE) return;
 
 	_sigaddset(&uc->uc_sigmask, SIGCANCEL);
 
 	if (self->cancelasync || ip >= __cp_begin && ip < __cp_end) {
-		self->canceldisable = 1;
-		pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0);
-		__cancel();
+		((char **)&uc->uc_mcontext)[CANCEL_REG_IP] = (char *)__cp_cancel;
+		return;
 	}
 
 	__syscall(SYS_tkill, self->tid, SIGCANCEL);

src/thread/pthread_setcancelstate.c

@@ -2,7 +2,7 @@
 
 int __pthread_setcancelstate(int new, int *old)
 {
-	if (new > 1U) return EINVAL;
+	if (new > 2U) return EINVAL;
 	if (!libc.has_thread_pointer) return ENOSYS;
 	struct pthread *self = __pthread_self();
 	if (old) *old = self->canceldisable;