Merge pull request #300 from Tarsnap/libcperciva-import

Libcperciva import

Makefile.inc

@@ -14,11 +14,22 @@ IDIRS	:=	${IDIRS}
 # Default is no man pages
 MAN	=
 
+# The SUBDIR_DEPTH variable (defined above) initializes tracking the
+# subdirectory depth, but it's not fully set at this stage in the build
+# process.  Instead, we jump back to the shell and print the (finalized) value
+# of SUBDIR_DEPTH.  This avoids an infinite loop because we query the Makefile
+# (which does not include ../Makefile.inc), rather than Makefile.BSD (which
+# would end up including this file again).
+FINALIZED_SUBDIR_DEPTH	!=	${MAKE} -v SUBDIR_DEPTH
+
 .if !defined(NOLIBALL)
 # Link everything to liball.a, unless they specifically ask not to use it.
-# If appropriate, metabuild.sh will do:
+LIBALL	=	${SUBDIR_DEPTH}/liball/liball.a
+.if exists(${FINALIZED_SUBDIR_DEPTH}/liball/optional_mutex_normal)
+# If we have optional_mutex, add it. If appropriate, metabuild.sh will do:
 #     s/optional_mutex_normal/optional_mutex_pthread/g
-LIBALL	=	${SUBDIR_DEPTH}/liball/liball.a ${SUBDIR_DEPTH}/liball/optional_mutex_normal/liball_optional_mutex_normal.a
+LIBALL	+=	${SUBDIR_DEPTH}/liball/optional_mutex_normal/liball_optional_mutex_normal.a
+.endif
 LDADD	+=	${LIBALL}
 DPADD	+=	${LIBALL}
 .endif

liball/Makefile

@@ -1,7 +1,7 @@
 .POSIX:
 # AUTOGENERATED FILE, DO NOT EDIT
 LIB=liball.a
-SRCS=crc32c.c crc32c_arm.c crc32c_sse42.c md5.c sha1.c sha256.c sha256_arm.c sha256_shani.c sha256_sse2.c aws_readkeys.c aws_sign.c cpusupport_arm_crc32_64.c cpusupport_arm_sha256.c cpusupport_x86_shani.c cpusupport_x86_sse2.c cpusupport_x86_sse42.c cpusupport_x86_ssse3.c elasticarray.c elasticqueue.c ptrheap.c seqptrmap.c timerqueue.c events.c events_immediate.c events_network.c events_network_selectstats.c events_timer.c http.c https.c netbuf_read.c netbuf_ssl.c netbuf_write.c network_accept.c network_connect.c network_read.c network_write.c network_ssl.c network_ssl_compat.c asprintf.c b64encode.c daemonize.c entropy.c getopt.c hexify.c humansize.c insecure_memzero.c json.c monoclock.c noeintr.c sock.c sock_util.c warnp.c bench.c mkpair.c doubleheap.c kvldskey.c kvhash.c kvpair.c onlinequantile.c pool.c dynamodb_kv.c dynamodb_request.c dynamodb_request_queue.c logging.c proto_dynamodb_kv_client.c proto_dynamodb_kv_server.c proto_kvlds_client.c proto_kvlds_server.c proto_lbs_client.c proto_lbs_server.c proto_s3_client.c proto_s3_server.c s3_request.c s3_request_queue.c s3_serverpool.c s3_verifyetag.c serverpool.c wire_packet.c wire_readpacket.c wire_requestqueue.c wire_writepacket.c kivaloo.c kvlds.c
+SRCS=crc32c.c crc32c_arm.c crc32c_sse42.c md5.c sha1.c sha256.c sha256_arm.c sha256_shani.c sha256_sse2.c aws_readkeys.c aws_sign.c cpusupport_arm_crc32_64.c cpusupport_arm_sha256.c cpusupport_x86_shani.c cpusupport_x86_sse2.c cpusupport_x86_sse42.c cpusupport_x86_ssse3.c elasticarray.c elasticqueue.c ptrheap.c seqptrmap.c timerqueue.c events.c events_immediate.c events_network.c events_network_selectstats.c events_timer.c http.c https.c netbuf_read.c netbuf_ssl.c netbuf_write.c network_accept.c network_connect.c network_read.c network_write.c network_ssl.c network_ssl_compat.c asprintf.c b64encode.c daemonize.c entropy.c getopt.c hexify.c humansize.c insecure_memzero.c ipc_sync.c json.c monoclock.c noeintr.c sock.c sock_util.c warnp.c bench.c mkpair.c doubleheap.c kvldskey.c kvhash.c kvpair.c onlinequantile.c pool.c dynamodb_kv.c dynamodb_request.c dynamodb_request_queue.c logging.c proto_dynamodb_kv_client.c proto_dynamodb_kv_server.c proto_kvlds_client.c proto_kvlds_server.c proto_lbs_client.c proto_lbs_server.c proto_s3_client.c proto_s3_server.c s3_request.c s3_request_queue.c s3_serverpool.c s3_verifyetag.c serverpool.c wire_packet.c wire_readpacket.c wire_requestqueue.c wire_writepacket.c kivaloo.c kvlds.c
 IDIRS=-I../libcperciva/alg -I../libcperciva/aws -I../libcperciva/cpusupport -I../libcperciva/datastruct -I../libcperciva/events -I ../libcperciva/http -I ../libcperciva/netbuf -I../libcperciva/network -I ../libcperciva/network_ssl -I../libcperciva/util -I../libcperciva/external/queue -I ../lib/bench -I ../lib/datastruct -I ../lib/dynamodb -I ../lib/logging -I ../lib/proto_dynamodb_kv -I ../lib/proto_kvlds -I ../lib/proto_lbs -I ../lib/proto_s3 -I ../lib/s3 -I ../lib/serverpool -I ../lib/wire -I ../lib/util
 SUBDIR_DEPTH=..
 RELATIVE_DIR=liball
@@ -101,7 +101,7 @@ asprintf.o: ../libcperciva/util/asprintf.c ../libcperciva/util/asprintf.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/asprintf.c -o asprintf.o
 b64encode.o: ../libcperciva/util/b64encode.c ../libcperciva/util/b64encode.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/b64encode.c -o b64encode.o
-daemonize.o: ../libcperciva/util/daemonize.c ../libcperciva/util/noeintr.h ../libcperciva/util/warnp.h ../libcperciva/util/daemonize.h
+daemonize.o: ../libcperciva/util/daemonize.c ../libcperciva/util/ipc_sync.h ../libcperciva/util/warnp.h ../libcperciva/util/daemonize.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/daemonize.c -o daemonize.o
 entropy.o: ../libcperciva/util/entropy.c ../libcperciva/util/warnp.h ../libcperciva/util/entropy.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/entropy.c -o entropy.o
@@ -113,6 +113,8 @@ humansize.o: ../libcperciva/util/humansize.c ../libcperciva/util/asprintf.h ../l
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/humansize.c -o humansize.o
 insecure_memzero.o: ../libcperciva/util/insecure_memzero.c ../libcperciva/util/insecure_memzero.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/insecure_memzero.c -o insecure_memzero.o
+ipc_sync.o: ../libcperciva/util/ipc_sync.c ../libcperciva/util/noeintr.h ../libcperciva/util/warnp.h ../libcperciva/util/ipc_sync.h
+	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/ipc_sync.c -o ipc_sync.o
 json.o: ../libcperciva/util/json.c ../libcperciva/util/json.h
 	${CC} ${CFLAGS_POSIX} -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -DCPUSUPPORT_CONFIG_FILE=\"cpusupport-config.h\" -DAPISUPPORT_CONFIG_FILE=\"apisupport-config.h\" -I.. ${IDIRS} ${CPPFLAGS} ${CFLAGS} -c ../libcperciva/util/json.c -o json.o
 monoclock.o: ../libcperciva/util/monoclock.c ../libcperciva/util/warnp.h ../libcperciva/util/monoclock.h

liball/Makefile.BSD

@@ -89,6 +89,7 @@ SRCS	+=	getopt.c
 SRCS	+=	hexify.c
 SRCS	+=	humansize.c
 SRCS	+=	insecure_memzero.c
+SRCS	+=	ipc_sync.c
 SRCS	+=	json.c
 SRCS	+=	monoclock.c
 SRCS	+=	noeintr.c

libcperciva/alg/sha256_arm.c

@@ -49,12 +49,11 @@ static const uint32_t Krnd[64] = {
 	X0 = vsha256su1q_u32(vsha256su0q_u32(X0, X1), X2, X3)
 
 /**
- * SHA256_Transform_arm(state, block, W, S):
+ * SHA256_Transform_arm(state, block):
  * Compute the SHA256 block compression function, transforming ${state} using
  * the data in ${block}.  This implementation uses ARM SHA256 instructions,
  * and should only be used if _SHA256 is defined and cpusupport_arm_sha256()
- * returns nonzero.  The arrays W and S may be filled with sensitive data, and
- * should be cleared by the callee.
+ * returns nonzero.
  */
 #ifdef POSIXFAIL_ABSTRACT_DECLARATOR
 void

libcperciva/alg/sha256_arm.h

@@ -4,12 +4,11 @@
 #include <stdint.h>
 
 /**
- * SHA256_Transform_arm(state, block, W, S):
+ * SHA256_Transform_arm(state, block):
  * Compute the SHA256 block compression function, transforming ${state} using
  * the data in ${block}.  This implementation uses ARM SHA256 instructions,
  * and should only be used if _SHA256 is defined and cpusupport_arm_sha256()
- * returns nonzero.  The arrays W and S may be filled with sensitive data, and
- * should be cleared by the callee.
+ * returns nonzero.
  */
 #ifdef POSIXFAIL_ABSTRACT_DECLARATOR
 void SHA256_Transform_arm(uint32_t state[8], const uint8_t block[64]);

libcperciva/alg/sha256_sse2.c

@@ -167,7 +167,7 @@ MSG4(__m128i X0, __m128i X1, __m128i X2, __m128i X3)
 }
 
 /**
- * SHA256_Transform_sse2(state, block):
+ * SHA256_Transform_sse2(state, block, W, S):
  * Compute the SHA256 block compression function, transforming ${state} using
  * the data in ${block}.  This implementation uses x86 SSE2 instructions, and
  * should only be used if _SSE2 is defined and cpusupport_x86_sse2() returns

libcperciva/alg/sha256_sse2.h

@@ -4,7 +4,7 @@
 #include <stdint.h>
 
 /**
- * SHA256_Transform_sse2(state, block):
+ * SHA256_Transform_sse2(state, block, W, S):
  * Compute the SHA256 block compression function, transforming ${state} using
  * the data in ${block}.  This implementation uses x86 SSE2 instructions, and
  * should only be used if _SSE2 is defined and cpusupport_x86_sse2() returns

libcperciva/apisupport/Build/apisupport-LIBSSL-HOST_NAME.c

@@ -1,3 +1,5 @@
+#include <stddef.h>
+
 #include <openssl/ssl.h>
 
 int

libcperciva/apisupport/Build/apisupport.sh

@@ -75,6 +75,8 @@ feature NONPOSIX SETGROUPS "" ""			\
 	"-U_POSIX_C_SOURCE -U_XOPEN_SOURCE"		\
 	"-U_POSIX_C_SOURCE -U_XOPEN_SOURCE -Wno-reserved-id-macro"
 
-# Detect how to compile libssl code.
+# Detect how to compile libssl and libcrypto code.
 feature LIBSSL HOST_NAME "-lssl" ""			\
 	"-Wno-cast-qual"
+feature LIBCRYPTO LOW_LEVEL_AES "-lcrypto" ""		\
+	"-Wno-deprecated-declarations"

libcperciva/util/daemonize.c

@@ -1,47 +1,12 @@
-#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <unistd.h>
 
-#include "noeintr.h"
+#include "ipc_sync.h"
 #include "warnp.h"
 
 #include "daemonize.h"
 
-/* Read and discard one byte from ${fd}, looping upon EINTR. */
-static int
-readbyte(int fd)
-{
-	char dummy;
-	char done = 0;
-
-	do {
-		switch (read(fd, &dummy, 1)) {
-		case -1:
-			/* Anything other than EINTR is bad. */
-			if (errno != EINTR) {
-				warnp("read");
-				goto err0;
-			}
-
-			/* Otherwise, loop and read again. */
-			break;
-		case 0:
-			warn0("Unexpected EOF in pipe");
-			goto err0;
-		case 1:
-			/* Expected value; quit the loop. */
-			done = 1;
-		}
-	} while (!done);
-
-	/* Success! */
-	return (0);
-
-err0:
-	/* Failure! */
-	return (-1);
-}
-
 /* Write the process' pid to a file called ${spid}. */
 static int
 writepid(const char * spid)
@@ -53,7 +18,7 @@ writepid(const char * spid)
 		warnp("fopen(%s)", spid);
 		goto err0;
 	}
-	if (fprintf(f, "%d", getpid()) < 0) {
+	if (fprintf(f, "%jd", (intmax_t)getpid()) < 0) {
 		warnp("fprintf");
 		goto err1;
 	}
@@ -83,17 +48,14 @@ writepid(const char * spid)
 int
 daemonize(const char * spid)
 {
-	int fd[2];
-	char dummy = 0;
+	struct ipc_sync * IS;
 
 	/*
 	 * Create a pipe for the child to notify the parent when it has
 	 * finished daemonizing.
 	 */
-	if (pipe(fd)) {
-		warnp("pipe");
+	if ((IS = ipc_sync_init()) == NULL)
 		goto err0;
-	}
 
 	/*
 	 * Fork into the parent process (which waits for a poke and exits)
@@ -103,35 +65,22 @@ daemonize(const char * spid)
 	case -1:
 		/* Fork failed. */
 		warnp("fork");
-		goto err2;
+		goto err1;
 	case 0:
 		/* In child process. */
 		break;
 	default:
-		/*
-		 * In parent process.  Close write end of pipe so that if the
-		 * client dies we will notice the pipe being reset.
-		 */
-		while (close(fd[1])) {
-			if (errno == EINTR)
-				continue;
-			warnp("close");
-			goto err1;
-		}
-
-		/* Read and discard a byte from the read end of the pipe. */
-		if (readbyte(fd[0]))
+		/* In parent process; wait for the child to start. */
+		if (ipc_sync_wait(IS))
 			goto err1;
 
 		/*
-		 * We don't need to read from this any more.  The pipe would
-		 * be implicitly closed by the following _exit(), but we're
-		 * explicitly closing it for the benefit of leak checkers.
+		 * Free resources associated with ${IS}.  These would be
+		 * released by the following _exit(), but we're explicitly
+		 * releasing them for the benefit of leak checkers.
 		 */
-		if (close(fd[0])) {
-			warnp("close");
-			goto err0;
-		}
+		if (ipc_sync_done(IS))
+			goto err1;
 
 		/* We have been poked by the child.  Exit. */
 		_exit(0);
@@ -147,35 +96,19 @@ daemonize(const char * spid)
 	if (writepid(spid))
 		goto die;
 
-	/* Tell the parent to suicide. */
-	if (noeintr_write(fd[1], &dummy, 1) == -1) {
-		warnp("write");
+	/* Tell the parent that we've started. */
+	if (ipc_sync_signal(IS))
 		goto die;
-	}
 
-	/* Close the pipe. */
-	while (close(fd[0])) {
-		if (errno == EINTR)
-			continue;
-		warnp("close");
+	/* Clean up. */
+	if (ipc_sync_done(IS))
 		goto die;
-	}
-	while (close(fd[1])) {
-		if (errno == EINTR)
-			continue;
-		warnp("close");
-		goto die;
-	}
 
 	/* Success! */
 	return (0);
 
-err2:
-	if (close(fd[1]))
-		warnp("close");
 err1:
-	if (close(fd[0]))
-		warnp("close");
+	ipc_sync_done(IS);
 err0:
 	/* Failure! */
 	return (-1);

libcperciva/util/entropy.h

@@ -25,7 +25,7 @@ int entropy_read_fill(struct entropy_read_cookie *, uint8_t *, size_t);
  * entropy_read_done(er):
  * Release any resources used by ${er}.
  */
-int entropy_read_done(struct entropy_read_cookie * er);
+int entropy_read_done(struct entropy_read_cookie *);
 
 /**
  * entropy_read(buf, buflen):