diff --git a/tests/libtest/lib518.c b/tests/libtest/lib518.c
index 3f5ac11a6..ffe38961e 100644
--- a/tests/libtest/lib518.c
+++ b/tests/libtest/lib518.c
@@ -251,7 +251,7 @@ static int rlimit(int keep_open)
 
   /* verify that we won't overflow size_t in malloc() */
 
-  if (num_open.rlim_max > ((size_t)-1) / sizeof(*fd)) {
+  if ((size_t)(num_open.rlim_max) > ((size_t)-1) / sizeof(*fd)) {
     sprintf(strbuff1, fmt, num_open.rlim_max);
     sprintf(strbuff, "unable to allocate an array for %s "
             "file descriptors, would overflow size_t", strbuff1);
diff --git a/tests/libtest/lib537.c b/tests/libtest/lib537.c
index fd030dd10..ab24b4242 100644
--- a/tests/libtest/lib537.c
+++ b/tests/libtest/lib537.c
@@ -233,7 +233,7 @@ static int rlimit(int keep_open)
 
   /* verify that we won't overflow size_t in malloc() */
 
-  if (num_open.rlim_max > ((size_t)-1) / sizeof(*fd)) {
+  if ((size_t)(num_open.rlim_max) > ((size_t)-1) / sizeof(*fd)) {
     sprintf(strbuff1, fmt, num_open.rlim_max);
     sprintf(strbuff, "unable to allocate an array for %s "
             "file descriptors, would overflow size_t", strbuff1);