tpax_archive_append(): directory recursion: initial implementation.

3 files changed, 495 insertions, 20 deletions

diff --git a/include/tpax/tpax.h b/include/tpax/tpax.h
index e12a44b..85cb5cf 100644
--- a/include/tpax/tpax.h
+++ b/include/tpax/tpax.h
@@ -150,7 +150,9 @@ tpax_api int  tpax_get_driver_fdctx     (const struct tpax_driver_ctx *, struct
 tpax_api int  tpax_set_driver_fdctx     (struct tpax_driver_ctx *, const struct tpax_fd_ctx *);
 
 /* core api */
-tpax_api int  tpax_archive_append       (const struct tpax_driver_ctx *, const struct tpax_unit_ctx *);
+tpax_api int  tpax_archive_append       (const struct tpax_driver_ctx *, const struct tpax_unit_ctx *,
+                                         const char *);
+
 tpax_api int  tpax_archive_seal         (const struct tpax_driver_ctx *);
 
 /* helper api */
diff --git a/src/driver/tpax_amain.c b/src/driver/tpax_amain.c
index 9d87e18..3d138ce 100644
--- a/src/driver/tpax_amain.c
+++ b/src/driver/tpax_amain.c
@@ -54,7 +54,7 @@ static void tpax_perform_unit_actions(
 	struct tpax_unit_ctx *		uctx)
 {
 	if (dctx->cctx->drvflags & TPAX_DRIVER_EXEC_MODE_WRITE)
-		tpax_archive_append(dctx,uctx);
+		tpax_archive_append(dctx,uctx,0);
 }
 
 static int tpax_exit(struct tpax_driver_ctx * dctx, int ret)
diff --git a/src/logic/tpax_archive_append.c b/src/logic/tpax_archive_append.c
index be9b2fa..4b81cb1 100644
--- a/src/logic/tpax_archive_append.c
+++ b/src/logic/tpax_archive_append.c
@@ -1,4 +1,3 @@
-
 /******************************************************/
 /*  tpax: a topological pax implementation            */
 /*  Copyright (C) 2020  Z. Gilboa                     */
@@ -13,11 +12,13 @@
 #include <errno.h>
 #include <grp.h>
 #include <pwd.h>
+#include <sys/mman.h>
 #include <sys/stat.h>
 
 #include <tpax/tpax.h>
 #include <tpax/tpax_specs.h>
 #include "tpax_driver_impl.h"
+#include "tpax_getdents_impl.h"
 #include "tpax_tmpfile_impl.h"
 #include "tpax_errinfo_impl.h"
 
@@ -75,10 +76,80 @@ static int tpax_archive_append_pad(
 	return ret;
 }
 
-int tpax_archive_append(
+static struct tpax_dirent_buffer * tpax_dirent_buf_first_alloc(
+	const struct tpax_driver_ctx * dctx)
+{
+	void * addr;
+	struct tpax_driver_ctx_impl * ictx;
+
+	addr = (struct tpax_dirent_buffer *)mmap(
+		0,TPAX_DIRENT_BUFLEN,
+		PROT_READ|PROT_WRITE,
+		MAP_PRIVATE|MAP_ANONYMOUS,
+		-1,0);
+
+	if (addr == MAP_FAILED)
+		return 0;
+
+	ictx                  = tpax_get_driver_ictx(dctx);
+	ictx->dirents         = (struct tpax_dirent_buffer *)addr;
+	ictx->dirents->cdent  = ictx->dirents->dbuf;
+
+	ictx->dirents->size   = TPAX_DIRENT_BUFLEN;
+	ictx->dirents->next   = 0;
+
+	ictx->dirents->nfree  = TPAX_DIRENT_BUFLEN;
+	ictx->dirents->nfree -= offsetof(struct tpax_dirent_buffer,dbuf);
+
+	return ictx->dirents;
+}
+
+static struct tpax_dirent_buffer * tpax_dirent_buf_next_alloc(
+	struct tpax_dirent_buffer * current)
+{
+	void * addr;
+
+	addr = (struct tpax_dirent_buffer *)mmap(
+		0,TPAX_DIRENT_BUFLEN,
+		PROT_READ|PROT_WRITE,
+		MAP_PRIVATE|MAP_ANONYMOUS,
+		-1,0);
+
+	if (addr == MAP_FAILED)
+		return 0;
+
+	current->next         = (struct tpax_dirent_buffer *)addr;
+	current->next->cdent  = current->next->dbuf;
+
+	current->next->size   = TPAX_DIRENT_BUFLEN;
+	current->next->next   = 0;
+
+	current->next->nfree  = TPAX_DIRENT_BUFLEN;
+	current->next->nfree -= offsetof(struct tpax_dirent_buffer,dbuf);
+
+	return current->next;
+}
+
+static int tpax_archive_append_ret(
+	int                     ret,
+	struct tpax_unit_ctx *  unit)
+{
+	if (unit)
+		tpax_free_unit_ctx(unit);
+
+	return ret;
+}
+
+static int tpax_archive_append_one(
 	const struct tpax_driver_ctx *  dctx,
-	const struct tpax_unit_ctx *    uctx)
+	const struct tpax_unit_ctx *    uctx,
+	const struct dirent *           dent,
+	int                             fdat,
+	const char *                    prefix,
+	const struct tpax_dirent *      parent,
+	const char *                    pdir)
 {
+	struct tpax_unit_ctx *          unit;
 	struct tpax_ustar_header        uhdr;
 	off_t                           hpos;
 	off_t                           dpos;
@@ -90,12 +161,77 @@ int tpax_archive_append(
 	size_t                          buflen;
 	size_t                          cmplen;
 	void *				membuf;
+	size_t                          nlen;
+	const char *			path;
+	const char *                    src;
+	char *                          dst;
+	char				pbuf[1024];
 	char                            sbuf[2048];
 
+	/* fake uctx for recursion items */
+	unit = 0;
+
+	if (dent && tpax_get_unit_ctx(
+			dctx,fdat,dent->d_name,
+			&unit) < 0)
+		return TPAX_NESTED_ERROR(dctx);
+
+	uctx = dent ? unit : uctx;
+
+	/* prefixed path */
+	if (!prefix && !parent && !pdir) {
+		path = *uctx->path;
+
+	} else {
+		nlen  = strlen(*uctx->path);
+		nlen += prefix ? strlen(prefix) + 1 : 0;
+		nlen += parent ? strlen(parent->dirent.d_name) + 1 : 0;
+
+		if (nlen >= sizeof(pbuf))
+			return TPAX_BUFFER_ERROR(dctx);
+
+		dst = pbuf;
+
+		if (prefix) {
+			src = prefix;
+
+			for (; *src; )
+				*dst++ = *src++;
+
+			if (dst[-1] != '/')
+				*dst++ = '/';
+		}
+
+		if (parent) {
+			src = parent->dirent.d_name;
+
+			for (; *src; )
+				*dst++ = *src++;
+
+			*dst++ = '/';
+		}
+
+		if (pdir) {
+			src = pdir;
+
+			for (; *src; )
+				*dst++ = *src++;
+
+			*dst++ = '/';
+		}
+
+		src = *uctx->path;
+
+		for (; *src; )
+			*dst++ = *src++;
+
+		*dst = 0;
+		path = pbuf;
+	}
+
 	/* record errors */
 	tpax_driver_set_ectx(
-		dctx,0,
-		*uctx->path);
+		dctx,0,path);
 
 	/* driver */
 	fdout = tpax_driver_fdout(dctx);
@@ -106,9 +242,11 @@ int tpax_archive_append(
 
 	/* header */
 	if (tpax_init_ustar_header(
-			dctx,*uctx->path,uctx->st,
+			dctx,path,uctx->st,
 			*uctx->link,&uhdr) < 0)
-		return TPAX_NESTED_ERROR(dctx);
+		return tpax_archive_append_ret(
+			TPAX_NESTED_ERROR(dctx),
+			unit);
 
 	/* buffer  */
 	membuf = 0;
@@ -128,15 +266,21 @@ int tpax_archive_append(
 			if (tpax_file_create_memory_snapshot(
 					dctx,fdat,*uctx->path,
 					uctx->st,membuf) < 0)
-				return TPAX_NESTED_ERROR(dctx);
+				return tpax_archive_append_ret(
+					TPAX_NESTED_ERROR(dctx),
+					unit);
 		} else {
 			if ((fdtmp = tpax_file_create_tmpfs_snapshot(
 					dctx,fdat,*uctx->path,
 					uctx->st)) < 0)
-				return TPAX_NESTED_ERROR(dctx);
+				return tpax_archive_append_ret(
+					TPAX_NESTED_ERROR(dctx),
+					unit);
 
 			if (lseek(fdtmp,0,SEEK_SET) < 0)
-				return TPAX_SYSTEM_ERROR(dctx);
+				return tpax_archive_append_ret(
+					TPAX_SYSTEM_ERROR(dctx),
+					unit);
 		}
 	}
 
@@ -145,14 +289,18 @@ int tpax_archive_append(
 		if (fdtmp >= 0)
 			close(fdtmp);
 
-		return TPAX_SYSTEM_ERROR(dctx);
+		return tpax_archive_append_ret(
+			TPAX_SYSTEM_ERROR(dctx),
+			unit);
 	}
 
 	tpax_set_driver_cpos(dctx,dpos);
 
 	/* all done? */
-	if (!(S_ISREG(uctx->st->st_mode)))
+	if (!(S_ISREG(uctx->st->st_mode))) {
+		tpax_archive_append_ret(0,unit);
 		return 0;
+	}
 
 	/* append data from snapshot */
 	if (fdtmp >= 0) {
@@ -169,11 +317,15 @@ int tpax_archive_append(
 
 			if (nbytes < 0) {
 				close(fdtmp);
-				return TPAX_SYSTEM_ERROR(dctx);
+				return tpax_archive_append_ret(
+					TPAX_SYSTEM_ERROR(dctx),
+					unit);
 
 			} else if (nbytes == 0) {
 				close(fdtmp);
-				return TPAX_CUSTOM_ERROR(dctx,TPAX_ERR_FLOW_ERROR);
+				return tpax_archive_append_ret(
+					TPAX_CUSTOM_ERROR(dctx,TPAX_ERR_FLOW_ERROR),
+					unit);
 
 			} else {
 				nread += nbytes;
@@ -181,7 +333,9 @@ int tpax_archive_append(
 
 			if (tpax_archive_append_memory_data(fdout,buf,nbytes) < 0) {
 				close(fdtmp);
-				return TPAX_SYSTEM_ERROR(dctx);
+				return tpax_archive_append_ret(
+					TPAX_SYSTEM_ERROR(dctx),
+					unit);
 			}
 		}
 
@@ -190,11 +344,330 @@ int tpax_archive_append(
 		if (tpax_archive_append_memory_data(
 				fdout,membuf,
 				uctx->st->st_size) < 0)
-			return TPAX_SYSTEM_ERROR(dctx);
+			return tpax_archive_append_ret(
+				TPAX_SYSTEM_ERROR(dctx),
+				unit);
+	}
+
+	return tpax_archive_append_ret(
+		tpax_archive_append_pad(dctx,fdout,uctx->st),
+		unit);
+}
+
+static int tpax_archive_append_dir(
+	const struct tpax_driver_ctx *  dctx,
+	const struct tpax_unit_ctx *    uctx,
+	struct tpax_dirent *            dent,
+	int                             fdat,
+	int                             depth,
+	const char *                    prefix,
+	const struct tpax_dirent *      parent)
+{
+	int                             fd;
+	bool                            fkeep;
+	long                            nbytes;
+	size_t                          needed;
+	struct dirent *                 dirent;
+	struct dirent *                 dirents;
+	struct tpax_dirent_buffer *     dentbuf;
+	struct tpax_dirent *            cdent;
+	struct tpax_unit_ctx *          unit;
+	struct stat                     st;
+	uintptr_t                       addr;
+	char *                          src;
+	char *                          dst;
+	char *                          cap;
+
+	/* fake uctx for recursion items */
+	unit = 0;
+
+	if (dent && tpax_get_unit_ctx(
+			dctx,dent->fdat,dent->dirent.d_name,
+			&unit) < 0)
+		return TPAX_NESTED_ERROR(dctx);
+
+	uctx = dent ? unit : uctx;
+
+	/* verify that recursion item is still a directory */
+	if (unit && !S_ISDIR(unit->st->st_mode))
+		return tpax_archive_append_ret(
+			TPAX_CUSTOM_ERROR(dctx,TPAX_ERR_FLOW_ERROR),
+			unit);
+
+	/* append directory entry to archive */
+	if (tpax_archive_append_one(
+			dctx,uctx,0,
+			tpax_driver_fdcwd(dctx),
+			prefix,0,0) < 0)
+		return tpax_archive_append_ret(
+			TPAX_NESTED_ERROR(dctx),
+			unit);
+
+	/* obtain buffer for file-system directory entries */
+	dirents = tpax_get_driver_getdents_buffer(dctx);
+	dirent  = dirents;
+	fkeep   = false;
+	nbytes  = 0;
+	depth++;
+
+	/* open directory and obtain first directory entries */
+	if ((fd = openat(fdat,*uctx->path,O_RDONLY|O_DIRECTORY|O_CLOEXEC,0)) < 0)
+		return tpax_archive_append_ret(
+			TPAX_SYSTEM_ERROR(dctx),
+			unit);
+
+	nbytes = tpax_getdents(fd,dirents,TPAX_DIRENT_BUFLEN);
+
+	while ((nbytes == -EINTR) || ((nbytes < 0) && (errno == EINTR)))
+		nbytes = tpax_getdents(fd,dirents,TPAX_DIRENT_BUFLEN);
+
+	if (nbytes < 0)
+		return tpax_archive_append_ret(
+			TPAX_SYSTEM_ERROR(dctx),
+			unit);
+
+	/* iterate */
+	for (; nbytes>0; ) {
+		if (!strcmp(dirent->d_name,".")) {
+			(void)0;
+
+		} else if (!strcmp(dirent->d_name,"..")) {
+			(void)0;
+
+		} else {
+			if (dirent->d_type == DT_UNKNOWN) {
+				if (fstatat(fd,dirent->d_name,&st,AT_SYMLINK_NOFOLLOW))
+					return tpax_archive_append_ret(
+						TPAX_SYSTEM_ERROR(dctx),
+						unit);
+
+				if (S_ISDIR(st.st_mode)) {
+					dirent->d_type = DT_DIR;
+				}
+			}
+
+			if (dirent->d_type == DT_DIR) {
+				if (!(dentbuf = tpax_get_driver_dirents(dctx)))
+					if (!(dentbuf = tpax_dirent_buf_first_alloc(dctx)))
+						return tpax_archive_append_ret(
+							TPAX_SYSTEM_ERROR(dctx),
+							unit);
+
+				needed  = dirent->d_reclen;
+				needed += offsetof(struct tpax_dirent,dirent);
+				needed += 0x7;
+				needed |= 0x7;
+				needed ^= 0x7;
+
+				for (; dentbuf->next && (dentbuf->nfree < needed); )
+					dentbuf = dentbuf->next;
+
+				if (dentbuf->nfree < needed)
+					if (!(dentbuf = tpax_dirent_buf_next_alloc(dentbuf)))
+						return tpax_archive_append_ret(
+							TPAX_SYSTEM_ERROR(dctx),
+							unit);
+
+				fkeep = true;
+				cdent = dentbuf->cdent;
+
+				cdent->fdat   = fd;
+				cdent->depth  = depth;
+				cdent->nsize  = needed;
+				cdent->parent = parent;
+
+				memset(&cdent->dirent,0,offsetof(struct dirent,d_name));
+
+				cdent->dirent.d_type   = dirent->d_type;
+				cdent->dirent.d_reclen = dirent->d_reclen;
+
+				src  = dirent->d_name;
+				dst  = cdent->dirent.d_name;
+
+				cap  = dst - offsetof(struct dirent,d_name);
+				cap -= offsetof(struct tpax_dirent,dirent);
+				cap += needed;
+
+				for (; *src; )
+					*dst++ = *src++;
+
+				for (; dst<cap; )
+					*dst++ = 0;
+
+				dentbuf->cdent  = (struct tpax_dirent *)cap;
+				dentbuf->nfree -= needed;
+			} else {
+				if (tpax_archive_append_one(
+						dctx,0,dirent,fd,prefix,
+						0,*uctx->path) < 0)
+					return tpax_archive_append_ret(
+						TPAX_NESTED_ERROR(dctx),
+						unit);
+			}
+		}
+
+		addr    = (uintptr_t)dirent;
+		addr   += dirent->d_reclen;
+		nbytes -= dirent->d_reclen;
+		dirent  = (struct dirent *)addr;
+
+		if (nbytes == 0) {
+			nbytes = tpax_getdents(fd,dirents,TPAX_DIRENT_BUFLEN);
+
+			while ((nbytes == -EINTR) || ((nbytes < 0) && (errno == EINTR)))
+				nbytes = tpax_getdents(fd,dirents,TPAX_DIRENT_BUFLEN);
+
+			if (nbytes < 0)
+				tpax_archive_append_ret(
+					TPAX_SYSTEM_ERROR(dctx),
+					unit);
+
+			dirent = dirents;
+		}
 	}
 
-	return tpax_archive_append_pad(
-		dctx,fdout,uctx->st);
+	/* all done */
+	return tpax_archive_append_ret(
+		fkeep ? 0 : close(fd),
+		unit);
+}
+
+static int tpax_archive_append_impl(
+	const struct tpax_driver_ctx *  dctx,
+	const struct tpax_unit_ctx *    uctx,
+	int                             depth,
+	const char *                    prefix,
+	struct tpax_dirent *            parent)
+{
+	if (S_ISDIR(uctx->st->st_mode))
+		if (dctx->cctx->drvflags & TPAX_DRIVER_DIR_MEMBER_RECURSE)
+			return tpax_archive_append_dir(
+				dctx,uctx,0,
+				tpax_driver_fdcwd(dctx),
+				depth,prefix,parent);
+
+	return tpax_archive_append_one(
+			dctx,uctx,0,
+			tpax_driver_fdcwd(dctx),
+			prefix,0,0);
+}
+
+int tpax_archive_append(
+	const struct tpax_driver_ctx *  dctx,
+	const struct tpax_unit_ctx *    uctx,
+	const char *                    prefix)
+{
+	struct tpax_dirent_buffer *     dentbuf;
+	struct tpax_dirent *            cdent;
+	struct tpax_dirent *            cnext;
+	const struct tpax_dirent *      parent;
+	uintptr_t                       addr;
+	const char *                    rdir;
+	const char *                    src;
+	char *                          dst;
+	char *                          cap;
+	char *                          mark;
+	int                             idx;
+	const char *                    dirv[256];
+	char                            pbuf[2048];
+
+	/* normalized prefix */
+	if (prefix && (prefix[0] == '.') && (!prefix[1]))
+		prefix = 0;
+
+	if (prefix && (prefix[0] == '.') && (prefix[1] == '/'))
+		for (++prefix; *prefix=='/'; prefix++)
+			(void)0;
+
+	/* append explicit item */
+	tpax_archive_append_impl(dctx,uctx,0,prefix,0);
+
+	/* iterate through queued items */
+	dentbuf = tpax_get_driver_dirents(dctx);
+	cdent   = dentbuf ? dentbuf->dbuf : 0;
+
+	if (cdent) {
+		dst = pbuf;
+		cap = &pbuf[sizeof(pbuf)];
+
+		if (prefix && prefix[0]) {
+			for (; *src && dst<cap; )
+				*dst++ = *src++;
+
+			if (dst == cap)
+				return TPAX_BUFFER_ERROR(dctx);
+
+			if (dst[-1] != '/')
+				*dst++ = '/';
+		}
+
+		src = *uctx->path;
+
+		for (; *src && dst<cap; )
+			*dst++ = *src++;
+
+		if (dst == cap)
+			return TPAX_BUFFER_ERROR(dctx);
+
+		if (dst[-1] != '/')
+			*dst++ = '/';
+
+		*dst = 0;
+		mark = dst;
+		rdir = pbuf;
+
+		(void)mark;
+	}
+
+	for (; cdent; ) {
+		switch (cdent->dirent.d_type) {
+			case DT_DIR:
+				if (tpax_archive_append_dir(
+						dctx,0,cdent,cdent->fdat,
+						cdent->depth,rdir,cdent) < 0)
+					return TPAX_NESTED_ERROR(dctx);
+
+				break;
+
+			default:
+				if (tpax_archive_append_one(
+						dctx,0,&cdent->dirent,
+						cdent->fdat,rdir,cdent,0) < 0)
+					return TPAX_NESTED_ERROR(dctx);
+		}
+
+		addr  = (uintptr_t)cdent;
+		addr += cdent->nsize;
+		cnext = (struct tpax_dirent *)addr;
+
+		if (cnext == dentbuf->cdent) {
+			dentbuf = dentbuf->next;
+			cnext   = dentbuf ? dentbuf->dbuf : 0;
+		}
+
+		if (cnext && (cnext->parent != cdent->parent)) {
+			if (cnext->depth > 256)
+				return TPAX_BUFFER_ERROR(dctx);
+
+			for (parent=cnext->parent; parent; parent=parent->parent)
+				dirv[parent->depth - 1] = parent->dirent.d_name;
+
+			for (idx=0,dst=mark; idx<cnext->parent->depth; idx++) {
+				src    = dirv[idx];
+
+				for (; *src; )
+					*dst++ = *src++;
+
+				*dst++ = '/';
+			}
+
+			*--dst = 0;
+		}
+
+		cdent = cnext;
+	}
+
+	return 0;
 }
 
 int tpax_archive_seal(const struct tpax_driver_ctx * dctx)