[klibc] [klibc:master] tests: Add test for malloc size arithmetic

[klibc-bot for Ben Hutchings]

Commit-ID:  8e88e0aafb402e11c61b9e2e377406afdb42f69e
Gitweb:     http://git.kernel.org/?p=libs/klibc/klibc.git;a=commit;h=8e88e0aafb402e11c61b9e2e377406afdb42f69e
Author:     Ben Hutchings <ben at decadent.org.uk>
AuthorDate: Wed, 28 Apr 2021 03:46:52 +0200
Committer:  Ben Hutchings <ben at decadent.org.uk>
CommitDate: Wed, 28 Apr 2021 04:43:03 +0200

[klibc] tests: Add test for malloc size arithmetic

It has been reported that klibc's malloc() and calloc() are
vulnerable to integer overflows.  Add test cases demonstrating
some of these.

Signed-off-by: Ben Hutchings <ben at decadent.org.uk>

---
 usr/klibc/tests/Kbuild        |  6 +++++
 usr/klibc/tests/malloctest3.c | 57 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/usr/klibc/tests/Kbuild b/usr/klibc/tests/Kbuild
index 00b701fc..44229c70 100644
--- a/usr/klibc/tests/Kbuild
+++ b/usr/klibc/tests/Kbuild
@@ -10,6 +10,11 @@ test-files := $(notdir $(test-files))
 # of useless warnings unless we tell it not to.
 KLIBCCFLAGS_testvsnp.o := -Wno-format
 
+# This deliberately calls malloc() with unreasonably large values.  We
+# can't use cc-disable-warning here as the option to *enable* this
+# warning requires a value.
+KLIBCCFLAGS_malloctest3.o := $(call cc-option,-Wno-alloc-size-larger-than)
+
 static-y := $(test-files:.c=)
 shared-y := $(addsuffix .shared, $(static-y))
 
@@ -24,6 +29,7 @@ idtest.shared-y		:= idtest.o
 lseek.shared-y		:= lseek.o
 malloctest.shared-y	:= malloctest.o
 malloctest2.shared-y	:= malloctest2.o
+malloctest3.shared-y	:= malloctest3.o
 memstrtest.shared-y	:= memstrtest.o
 microhello.shared-y	:= microhello.o
 minihello.shared-y	:= minihello.o
diff --git a/usr/klibc/tests/malloctest3.c b/usr/klibc/tests/malloctest3.c
new file mode 100644
index 00000000..d9d2ca9c
--- /dev/null
+++ b/usr/klibc/tests/malloctest3.c
@@ -0,0 +1,57 @@
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	void *p;
+
+	/* Our implementation should always return NULL */
+	errno = 0;
+	p = malloc(0);
+	assert(p == NULL);
+	assert(errno == 0);
+
+	/* These sizes won't fit in memory, so should always fail */
+	errno = 0;
+	p = malloc(SIZE_MAX);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = malloc(SIZE_MAX - 0x10000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+
+#if SIZE_MAX > 0x100000000
+	/* We should be able to allocate 4 GB + 1 */
+	p = malloc(0x100000001);
+	assert(p != NULL);
+	((volatile char *)p)[0x100000000] = 1;
+	free(p);
+
+	/* calloc() should detect multiplication overflow */
+	errno = 0;
+	p = calloc(0x100000000, 0x100000000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = calloc(0x100000001, 0x100000001);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+#else
+	/* calloc() should detect multiplication overflow */
+	errno = 0;
+	p = calloc(0x10000, 0x10000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = calloc(0x10001, 0x10001);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+#endif
+
+	return 0;
+}