syscall: cap RLIMIT_NOFILE soft limit in TestRlimit on darwin

On some machines, kern.maxfilesperproc is 4096. If Rlimit.Cur is larger
than that, Setrlimit will get an errEINVAL.

Fixes #40564.

Change-Id: Ib94303c790a489ff0559c88d41a021e514d18f8d
Reviewed-on: https://go-review.googlesource.com/c/go/+/246658
Reviewed-by: Tobias Klauser <tobias.klauser@gmail.com>
Run-TryBot: Tobias Klauser <tobias.klauser@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
diff --git a/src/syscall/syscall_unix_test.go b/src/syscall/syscall_unix_test.go
index 13b79ca..3c7982e 100644
--- a/src/syscall/syscall_unix_test.go
+++ b/src/syscall/syscall_unix_test.go
@@ -336,11 +336,11 @@
 	}
 	set := rlimit
 	set.Cur = set.Max - 1
-	if runtime.GOOS == "darwin" && set.Cur > 10240 {
-		// The max file limit is 10240, even though
-		// the max returned by Getrlimit is 1<<63-1.
-		// This is OPEN_MAX in sys/syslimits.h.
-		set.Cur = 10240
+	if runtime.GOOS == "darwin" && set.Cur > 4096 {
+		// rlim_min for RLIMIT_NOFILE should be equal to
+		// or lower than kern.maxfilesperproc, which on
+		// some machines are 4096. See #40564.
+		set.Cur = 4096
 	}
 	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &set)
 	if err != nil {
@@ -353,8 +353,8 @@
 	}
 	set = rlimit
 	set.Cur = set.Max - 1
-	if runtime.GOOS == "darwin" && set.Cur > 10240 {
-		set.Cur = 10240
+	if runtime.GOOS == "darwin" && set.Cur > 4096 {
+		set.Cur = 4096
 	}
 	if set != get {
 		t.Fatalf("Rlimit: change failed: wanted %#v got %#v", set, get)