Fix handling of close-on-exec bit (#860)

This commit's main purpose is to robustify ksh93's handling of
the close-on-exec bit. Much of this code is based on groundwork
laid out in ksh93v-, though many other changes were made beyond
that since the 93v- syscall intercepts were not backported.

- Avoid needlessly calling fcntl twice if F_DUPFD_CLOEXEC
  is available.
- Added support for F_DUPFD_CLOEXEC to sh_fcntl.
- Incidentally fixed file descriptor leakage in histexceptf().
- Added support for close-on-exec to sh_pipe and sh_rpipe,
  which is implemented via SOCK_CLOEXEC, pipe2(2) and (as a
  fallback) plain fcntl F_SETFD.
  - For process substitutions, rather than use another fcntl to
    undo close-on-exec (like in 93v-), instead add a second argument
    to sh_pipe and sh_rpipe that can be used to specify whether
    close-on-exec is wanted.
- Added support for O_CLOEXEC to sh_open().
- Made it such that sh_iomovefd preserves the presence or absence
  of the close-on-exec bit.
  - The ksh93v- change to make sh_iomovefd return fds >9 broke one
    of the regression tests, so that was changed to dup to 3 or
    higher.

src/lib/libast/features/lib:
- When ksh is compiled on illumos with clang, the libast
  _lib_select test will fail and subsequently many io.sh test
  failures then occur. This is caused by the FD_ZERO macro calling
  memset implicitly, which is invalid in C99. To fix this, a
  string.h include directive has been added to the feature test.

Author: JohnoKing

src/cmd/ksh93/bltins/hist.c

@@ -206,7 +206,7 @@ int	b_hist(int argc,char *argv[], Shbltin_t *context)
 			errormsg(SH_DICT,ERROR_exit(1),e_create,"");
 			UNREACHABLE();
 		}
-		if((fdo=open(fname,O_CREAT|O_RDWR,S_IRUSR|S_IWUSR)) < 0)
+		if((fdo=open(fname,O_CREAT|O_RDWR|O_cloexec,S_IRUSR|S_IWUSR)) < 0)
 		{
 			errormsg(SH_DICT,ERROR_system(1),e_create,fname);
 			UNREACHABLE();

src/cmd/ksh93/edit/history.c

@@ -123,9 +123,7 @@ static History_t *hist_ptr;
 			cp = "unknown";
 	}
 	logname = sh_strdup(cp);
-	if((acctfd=sh_open(acctfile,
-		O_BINARY|O_WRONLY|O_APPEND|O_CREAT,S_IRUSR|S_IWUSR))>=0 &&
-	    (unsigned)acctfd < 10)
+	if((acctfd=sh_open(acctfile,O_BINARY|O_WRONLY|O_APPEND|O_CREAT|O_cloexec,S_IRUSR|S_IWUSR))>=0 && acctfd < 10)
 	{
 		int n;
 		if((n = sh_fcntl(acctfd, F_dupfd_cloexec, 10)) >= 0)
@@ -145,8 +143,8 @@ static History_t *hist_ptr;
 		sfsprintf(newfile,sizeof(newfile),"%.8s%d\0",e_devfdNN,acctfd);
 		nv_putval(np,newfile,NV_RDONLY);
 	}
-	else
-		fcntl(acctfd,F_SETFD,FD_CLOEXEC);
+	else if(!(sh.fdstatus[acctfd]&IOCLEX))
+		sh_fcntl(acctfd,F_SETFD,FD_CLOEXEC);
 	return 1;
     }
 #endif /* SHOPT_ACCTFILE */
@@ -156,7 +154,7 @@ static int sh_checkaudit(const char *name, char *logbuf, size_t len)
 {
 	char	*cp, *last;
 	int	id1, id2, r=0, n, fd;
-	if((fd=open(name, O_RDONLY,O_cloexec)) < 0)
+	if((fd=open(name, O_RDONLY|O_cloexec)) < 0)
 		return 0;
 	if((n = read(fd, logbuf,len-1)) < 0)
 		goto done;
@@ -205,7 +203,7 @@ int  sh_histinit(void)
 	History_t *hp;
 	char *histname;
 	char *fname=0;
-	int histmask, maxlines, hist_start=0;
+	int histmask, maxlines, hist_start=0, n;
 	char *cp;
 	off_t hsize = 0;
 
@@ -224,13 +222,10 @@ int  sh_histinit(void)
 	cp = path_relative(histname);
 	if(!histinit)
 		histmode = S_IRUSR|S_IWUSR;
-	if((fd=open(cp,O_BINARY|O_APPEND|O_RDWR|O_CREAT|O_cloexec,histmode))>=0)
-	{
+	if((fd=sh_open(cp,O_BINARY|O_APPEND|O_RDWR|O_CREAT|O_cloexec,histmode))>=0)
 		hsize=lseek(fd,0,SEEK_END);
-	}
-	if((unsigned)fd < 10)
+	if(fd > 0 && fd < 10)
 	{
-		int n;
 		if((n=sh_fcntl(fd,F_dupfd_cloexec,10))>=0)
 		{
 			sh_close(fd);
@@ -253,13 +248,13 @@ int  sh_histinit(void)
 		{
 			if(!(fname = pathtmp(NULL,0,0,NULL)))
 				return 0;
-			fd = open(fname,O_BINARY|O_APPEND|O_CREAT|O_RDWR,S_IRUSR|S_IWUSR|O_cloexec);
+			fd = sh_open(fname,O_BINARY|O_APPEND|O_CREAT|O_RDWR|O_cloexec,S_IRUSR|S_IWUSR);
 		}
 	}
 	if(fd<0)
 		return 0;
-	/* set the file to close-on-exec */
-	fcntl(fd,F_SETFD,FD_CLOEXEC);
+	if(!(sh.fdstatus[fd]&IOCLEX))
+		sh_fcntl(fd,F_SETFD,FD_CLOEXEC);  /* set the file to close-on-exec */
 	if(cp=nv_getval(HISTSIZE))
 	{
 		intmax_t m = strtoll(cp, NULL, 10);
@@ -344,7 +339,6 @@ int  sh_histinit(void)
 		{
 			if((fd=sh_open(buff,O_BINARY|O_WRONLY|O_APPEND|O_CREAT|O_cloexec,S_IRUSR|S_IWUSR))>=0 && fd < 10)
 			{
-				int n;
 				if((n = sh_fcntl(fd,F_dupfd_cloexec, 10)) >= 0)
 				{
 					sh_close(fd);
@@ -354,7 +348,8 @@ int  sh_histinit(void)
 			if(fd>=0)
 			{
 				const char *tty;
-				fcntl(fd,F_SETFD,FD_CLOEXEC);
+				if(!(sh.fdstatus[fd]&IOCLEX))
+					sh_fcntl(fd,F_SETFD,FD_CLOEXEC);
 				tty = ttyname(2);
 				hp->tty = sh_strdup(tty?tty:"notty");
 				hp->auditfp = sfnew(NULL,NULL,-1,fd,SFIO_WRITE);
@@ -1125,12 +1120,21 @@ static int hist_exceptf(Sfio_t* fp, int type, void *data, Sfdisc_t *handle)
 			return 0;
 		/* write failure could be NFS problem, try to reopen */
 		sh_close(oldfd=sffileno(fp));
-		if((newfd=open(hp->histname,O_BINARY|O_APPEND|O_CREAT|O_RDWR|O_cloexec,S_IRUSR|S_IWUSR)) >= 0)
+		if((newfd=sh_open(hp->histname,O_BINARY|O_APPEND|O_CREAT|O_RDWR|O_cloexec,S_IRUSR|S_IWUSR)) >= 0)
 		{
-			if(sh_fcntl(newfd, F_dupfd_cloexec, oldfd) != oldfd)
-				return -1;
-			fcntl(oldfd,F_SETFD,FD_CLOEXEC);
-			close(newfd);
+			if(newfd != oldfd)
+			{
+				int dupfd = sh_fcntl(newfd, F_dupfd_cloexec, oldfd);
+				sh_close(newfd);
+				if(dupfd != oldfd)
+				{
+					if(dupfd > -1)
+						sh_close(dupfd);
+					return -1;
+				}
+			}
+			if(!(sh.fdstatus[oldfd]&IOCLEX))
+				sh_fcntl(oldfd,F_SETFD,FD_CLOEXEC);
 			if(lseek(oldfd,0,SEEK_END) < hp->histcnt)
 			{
 				int index = hp->histind;

src/cmd/ksh93/features/externs

@@ -219,7 +219,7 @@ tst	execve_dir_enoexec note{ does execve(3) set errno to ENOEXEC when trying to
 	{
 		char	dirname[64];
 		int	e;
-		sprintf(dirname,".dir.%u",(unsigned)getpid());
+		sprintf(dirname,".dir.%u",(unsigned int)getpid());
 		mkdir(dirname,0777);
 		execve(dirname,argv,environ);
 		e = errno;

src/cmd/ksh93/include/io.h

@@ -79,7 +79,7 @@ extern void 	sh_ioinit(void);
 extern int 	sh_iomovefd(int);
 extern int	sh_iorenumber(int,int);
 extern void 	sh_pclose(int[]);
-extern int	sh_rpipe(int[]);
+extern int	sh_rpipe(int[],int);
 extern void 	sh_iorestore(int,int);
 extern Sfio_t 	*sh_iostream(int);
 extern int	sh_redirect(struct ionod*,int);

src/cmd/ksh93/include/shell.h

@@ -468,7 +468,7 @@ extern Sfio_t		*sh_pathopen(const char*);
 extern ssize_t 		sh_read(int, void*, size_t);
 extern ssize_t 		sh_write(int, const void*, size_t);
 extern off_t		sh_seek(int, off_t, int);
-extern int 		sh_pipe(int[]);
+extern int 		sh_pipe(int[],int);
 extern mode_t 		sh_umask(mode_t);
 extern Shwait_f		sh_waitnotify(Shwait_f);
 extern Shscope_t	*sh_getscope(int,int);
@@ -493,7 +493,6 @@ extern Shell_t		sh;
 #   define close(a)	sh_close(a)
 #   define exit(a)	sh_exit(a)
 #   define fcntl(a,b,c)	sh_fcntl(a,b,c)
-#   define pipe(a)	sh_pipe(a)
 #   define read(a,b,c)	sh_read(a,b,c)
 #   define write(a,b,c)	sh_write(a,b,c)
 #   define umask(a)	sh_umask(a)

src/cmd/ksh93/sh/args.c

@@ -714,7 +714,7 @@ char **sh_argbuild(int *nargs, const struct comnod *comptr,int flag)
 }
 
 #if _pipe_socketpair && !_socketpair_devfd
-#   define sh_pipe(a)	sh_rpipe(a)
+#   define sh_pipe(a,b)	sh_rpipe(a,b)
 #endif
 
 struct argnod *sh_argprocsub(struct argnod *argp)
@@ -734,7 +734,7 @@ struct argnod *sh_argprocsub(struct argnod *argp)
 #if SHOPT_DEVFD
 	sfwrite(sh.stk,e_devfdNN,8);
 	pv[2] = 0;
-	sh_pipe(pv);
+	sh_pipe(pv,0);
 #else
 	pv[0] = -1;
 	while(sh.fifo = pathtemp(0,0,0,"ksh.fifo",0), sh.fifo && mkfifo(sh.fifo,0)<0)