cmd/macos-roots-test: new package
The darwin API to extract the trusted roots is so complex that we have
no chance of getting it right by just reading the "docs".
So, make a stand-alone tool people can go get to crowdsource testing.
Change-Id: Iaf6da6cec5b86e6b27f9fccba106b8d1e2944c97
Reviewed-on: https://go-review.googlesource.com/128118
Reviewed-by: Ian Lance Taylor <iant@golang.org>
diff --git a/cmd/macos-roots-test/main.go b/cmd/macos-roots-test/main.go
new file mode 100644
index 0000000..d2cf21a
--- /dev/null
+++ b/cmd/macos-roots-test/main.go
@@ -0,0 +1,119 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command macOS-roots-test runs crypto/x509.TestSystemRoots as a
+// stand-alone binary for crowdsourced testing.
+package main
+
+import (
+ "crypto/x509"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "time"
+ "unsafe"
+)
+
+type CertPool struct {
+ bySubjectKeyId map[string][]int
+ byName map[string][]int
+ certs []*x509.Certificate
+}
+
+func (s *CertPool) contains(cert *x509.Certificate) bool {
+ if s == nil {
+ return false
+ }
+
+ candidates := s.byName[string(cert.RawSubject)]
+ for _, c := range candidates {
+ if s.certs[c].Equal(cert) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func main() {
+ var failed bool
+
+ t0 := time.Now()
+ sysRootsExt, err := loadSystemRoots() // actual system roots
+ sysRootsDuration := time.Since(t0)
+
+ if err != nil {
+ log.Fatalf("failed to read system roots (cgo): %v", err)
+ }
+ sysRoots := (*CertPool)(unsafe.Pointer(sysRootsExt))
+
+ t1 := time.Now()
+ execRootsExt, err := execSecurityRoots() // non-cgo roots
+ execSysRootsDuration := time.Since(t1)
+
+ if err != nil {
+ log.Fatalf("failed to read system roots (nocgo): %v", err)
+ }
+ execRoots := (*CertPool)(unsafe.Pointer(execRootsExt))
+
+ fmt.Printf(" cgo sys roots: %v\n", sysRootsDuration)
+ fmt.Printf("non-cgo sys roots: %v\n", execSysRootsDuration)
+
+ // On Mavericks, there are 212 bundled certs, at least there was at
+ // one point in time on one machine. (Maybe it was a corp laptop
+ // with extra certs?) Other OS X users report 135, 142, 145...
+ // Let's try requiring at least 100, since this is just a sanity
+ // check.
+ if want, have := 100, len(sysRoots.certs); have < want {
+ failed = true
+ fmt.Printf("want at least %d system roots, have %d\n", want, have)
+ }
+
+ // Check that the two cert pools are the same.
+ sysPool := make(map[string]*x509.Certificate, len(sysRoots.certs))
+ for _, c := range sysRoots.certs {
+ sysPool[string(c.Raw)] = c
+ }
+ for _, c := range execRoots.certs {
+ if _, ok := sysPool[string(c.Raw)]; ok {
+ delete(sysPool, string(c.Raw))
+ } else {
+ // verify-cert lets in certificates that are not trusted roots, but are
+ // signed by trusted roots. This should not be a problem, so confirm that's
+ // the case and skip them.
+ if _, err := c.Verify(x509.VerifyOptions{
+ Roots: sysRootsExt,
+ Intermediates: execRootsExt, // the intermediates for EAP certs are stored in the keychain
+ KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
+ }); err != nil {
+ failed = true
+ fmt.Printf("certificate only present in non-cgo pool: %v (verify error: %v)\n", c.Subject, err)
+ } else {
+ fmt.Printf("signed certificate only present in non-cgo pool (acceptable): %v\n", c.Subject)
+ }
+ }
+ }
+ for _, c := range sysPool {
+ failed = true
+ fmt.Printf("certificate only present in cgo pool: %v\n", c.Subject)
+ }
+
+ if failed && debugDarwinRoots {
+ cmd := exec.Command("security", "dump-trust-settings")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ cmd.Run()
+ cmd = exec.Command("security", "dump-trust-settings", "-d")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ cmd.Run()
+ }
+
+ if failed {
+ fmt.Printf("\n\n!!! The test failed!\n\nPlease report *the whole output* at https://github.com/golang/go/issues/24652 wrapping it in ``` a code block ```\nThank you!\n")
+ } else {
+ fmt.Printf("\n\nThe test passed, no need to report the output. Thank you.\n")
+ }
+}
diff --git a/cmd/macos-roots-test/root_cgo_darwin.go b/cmd/macos-roots-test/root_cgo_darwin.go
new file mode 100644
index 0000000..591098f
--- /dev/null
+++ b/cmd/macos-roots-test/root_cgo_darwin.go
@@ -0,0 +1,290 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+/*
+#cgo CFLAGS: -mmacosx-version-min=10.10 -D__MAC_OS_X_VERSION_MAX_ALLOWED=101300
+#cgo LDFLAGS: -framework CoreFoundation -framework Security
+
+#include <errno.h>
+#include <sys/sysctl.h>
+
+#include <CoreFoundation/CoreFoundation.h>
+#include <Security/Security.h>
+
+void CFReleaseIfNotNULL(CFTypeRef cf) {
+ if (cf != NULL) CFRelease(cf);
+}
+
+static bool isSSLPolicy(SecPolicyRef policyRef) {
+ if (!policyRef) {
+ return false;
+ }
+ CFDictionaryRef properties = SecPolicyCopyProperties(policyRef);
+ if (properties == NULL) {
+ return false;
+ }
+ CFTypeRef value = NULL;
+ if (CFDictionaryGetValueIfPresent(properties, kSecPolicyOid, (const void **)&value)) {
+ CFRelease(properties);
+ return CFEqual(value, kSecPolicyAppleSSL);
+ }
+ CFRelease(properties);
+ return false;
+}
+
+// sslTrustSettingsResult obtains the final kSecTrustSettingsResult value
+// for a certificate in the user or admin domain, combining usage constraints
+// for the SSL SecTrustSettingsPolicy, ignoring SecTrustSettingsKeyUsage,
+// kSecTrustSettingsAllowedError and kSecTrustSettingsPolicyString.
+// https://developer.apple.com/documentation/security/1400261-sectrustsettingscopytrustsetting
+static SInt32 sslTrustSettingsResult(SecCertificateRef cert) {
+ CFArrayRef trustSettings = NULL;
+ OSStatus err = SecTrustSettingsCopyTrustSettings(cert, kSecTrustSettingsDomainUser, &trustSettings);
+
+ // According to Apple's SecTrustServer.c, "user trust settings overrule admin trust settings",
+ // but the rules of the override are unclear. Let's assume admin trust settings are applicable
+ // if and only if user trust settings fail to load or are NULL.
+ if (err != errSecSuccess || trustSettings == NULL) {
+ CFReleaseIfNotNULL(trustSettings);
+ err = SecTrustSettingsCopyTrustSettings(cert, kSecTrustSettingsDomainAdmin, &trustSettings);
+ }
+
+ // > no trust settings [...] means "this certificate must be verified to a known trusted certificate”
+ if (err != errSecSuccess || trustSettings == NULL) {
+ CFReleaseIfNotNULL(trustSettings);
+ return kSecTrustSettingsResultUnspecified;
+ }
+
+ // > An empty trust settings array means "always trust this certificate” with an
+ // > overall trust setting for the certificate of kSecTrustSettingsResultTrustRoot.
+ if (CFArrayGetCount(trustSettings) == 0) {
+ CFReleaseIfNotNULL(trustSettings);
+ return kSecTrustSettingsResultTrustRoot;
+ }
+
+ // kSecTrustSettingsResult is defined as CFSTR("kSecTrustSettingsResult"),
+ // but the Go linker's internal linking mode can't handle CFSTR relocations.
+ // Create our own dynamic string instead and release it below.
+ CFStringRef _kSecTrustSettingsResult = CFStringCreateWithCString(
+ NULL, "kSecTrustSettingsResult", kCFStringEncodingUTF8);
+ CFStringRef _kSecTrustSettingsPolicy = CFStringCreateWithCString(
+ NULL, "kSecTrustSettingsPolicy", kCFStringEncodingUTF8);
+
+ CFIndex m; SInt32 result = 0;
+ for (m = 0; m < CFArrayGetCount(trustSettings); m++) {
+ CFDictionaryRef tSetting = (CFDictionaryRef)CFArrayGetValueAtIndex(trustSettings, m);
+
+ // First, check if this trust setting applies to our policy. We assume
+ // only one will. The docs suggest that there might be multiple applying
+ // but don't explain how to combine them.
+ SecPolicyRef policyRef;
+ if (CFDictionaryGetValueIfPresent(tSetting, _kSecTrustSettingsPolicy, (const void**)&policyRef)) {
+ if (!isSSLPolicy(policyRef)) {
+ continue;
+ }
+ } else {
+ continue;
+ }
+
+ CFNumberRef cfNum;
+ if (CFDictionaryGetValueIfPresent(tSetting, _kSecTrustSettingsResult, (const void**)&cfNum)) {
+ CFNumberGetValue(cfNum, kCFNumberSInt32Type, &result);
+ } else {
+ // > If the value of the kSecTrustSettingsResult component is not
+ // > kSecTrustSettingsResultUnspecified for a usage constraints dictionary that has
+ // > no constraints, the default value kSecTrustSettingsResultTrustRoot is assumed.
+ result = kSecTrustSettingsResultTrustRoot;
+ }
+
+ break;
+ }
+
+ // If trust settings are present, but none of them match the policy...
+ // the docs don't tell us what to do.
+ //
+ // "Trust settings for a given use apply if any of the dictionaries in the
+ // certificate’s trust settings array satisfies the specified use." suggests
+ // that it's as if there were no trust settings at all, so we should probably
+ // fallback to the admin trust settings. TODO.
+ if (result == 0) {
+ result = kSecTrustSettingsResultUnspecified;
+ }
+
+ CFRelease(_kSecTrustSettingsResult);
+ CFRelease(trustSettings);
+
+ return result;
+}
+
+// FetchPEMRoots fetches the system's list of trusted X.509 root certificates
+// for the kSecTrustSettingsPolicy SSL.
+//
+// On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root
+// certificates of the system. On failure, the function returns -1.
+// Additionally, it fills untrustedPemRoots with certs that must be removed from pemRoots.
+//
+// Note: The CFDataRef returned in pemRoots and untrustedPemRoots must
+// be released (using CFRelease) after we've consumed its content.
+int _FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots, bool debugDarwinRoots) {
+ int i;
+
+ if (debugDarwinRoots) {
+ printf("crypto/x509: kSecTrustSettingsResultInvalid = %d\n", kSecTrustSettingsResultInvalid);
+ printf("crypto/x509: kSecTrustSettingsResultTrustRoot = %d\n", kSecTrustSettingsResultTrustRoot);
+ printf("crypto/x509: kSecTrustSettingsResultTrustAsRoot = %d\n", kSecTrustSettingsResultTrustAsRoot);
+ printf("crypto/x509: kSecTrustSettingsResultDeny = %d\n", kSecTrustSettingsResultDeny);
+ printf("crypto/x509: kSecTrustSettingsResultUnspecified = %d\n", kSecTrustSettingsResultUnspecified);
+ }
+
+ // Get certificates from all domains, not just System, this lets
+ // the user add CAs to their "login" keychain, and Admins to add
+ // to the "System" keychain
+ SecTrustSettingsDomain domains[] = { kSecTrustSettingsDomainSystem,
+ kSecTrustSettingsDomainAdmin,
+ kSecTrustSettingsDomainUser };
+
+ int numDomains = sizeof(domains)/sizeof(SecTrustSettingsDomain);
+ if (pemRoots == NULL) {
+ return -1;
+ }
+
+ CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
+ CFMutableDataRef combinedUntrustedData = CFDataCreateMutable(kCFAllocatorDefault, 0);
+ for (i = 0; i < numDomains; i++) {
+ int j;
+ CFArrayRef certs = NULL;
+ OSStatus err = SecTrustSettingsCopyCertificates(domains[i], &certs);
+ if (err != noErr) {
+ continue;
+ }
+
+ CFIndex numCerts = CFArrayGetCount(certs);
+ for (j = 0; j < numCerts; j++) {
+ CFDataRef data = NULL;
+ CFArrayRef trustSettings = NULL;
+ SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, j);
+ if (cert == NULL) {
+ continue;
+ }
+
+ SInt32 result;
+ if (domains[i] == kSecTrustSettingsDomainSystem) {
+ // Certs found in the system domain are always trusted. If the user
+ // configures "Never Trust" on such a cert, it will also be found in the
+ // admin or user domain, causing it to be added to untrustedPemRoots. The
+ // Go code will then clean this up.
+ result = kSecTrustSettingsResultTrustAsRoot;
+ } else {
+ result = sslTrustSettingsResult(cert);
+ if (debugDarwinRoots) {
+ CFErrorRef errRef = NULL;
+ CFStringRef summary = SecCertificateCopyShortDescription(NULL, cert, &errRef);
+ if (errRef != NULL) {
+ printf("crypto/x509: SecCertificateCopyShortDescription failed\n");
+ CFRelease(errRef);
+ continue;
+ }
+
+ CFIndex length = CFStringGetLength(summary);
+ CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8) + 1;
+ char *buffer = malloc(maxSize);
+ if (CFStringGetCString(summary, buffer, maxSize, kCFStringEncodingUTF8)) {
+ printf("crypto/x509: %s returned %d\n", buffer, result);
+ }
+ free(buffer);
+ CFRelease(summary);
+ }
+ }
+
+ CFMutableDataRef appendTo;
+ if (result == kSecTrustSettingsResultTrustRoot) {
+ // "can only be applied to root (self-signed) certificates", so
+ // make sure Subject and Issuer Name match.
+ CFErrorRef errRef = NULL;
+ CFDataRef subjectName = SecCertificateCopyNormalizedSubjectContent(cert, &errRef);
+ if (errRef != NULL) {
+ CFRelease(errRef);
+ continue;
+ }
+ CFDataRef issuerName = SecCertificateCopyNormalizedIssuerContent(cert, &errRef);
+ if (errRef != NULL) {
+ CFRelease(subjectName);
+ CFRelease(errRef);
+ continue;
+ }
+ Boolean equal = CFEqual(subjectName, issuerName);
+ CFRelease(subjectName);
+ CFRelease(issuerName);
+ if (!equal) {
+ continue;
+ }
+
+ appendTo = combinedData;
+ } else if (result == kSecTrustSettingsResultTrustAsRoot) {
+ // In theory "can only be applied to non-root certificates", but ignore
+ // this for now, also because it's the state we assume for the system domain.
+ appendTo = combinedData;
+ } else if (result == kSecTrustSettingsResultDeny) {
+ appendTo = combinedUntrustedData;
+ } else if (result == kSecTrustSettingsResultUnspecified) {
+ continue;
+ } else {
+ continue;
+ }
+
+ err = SecItemExport(cert, kSecFormatX509Cert, kSecItemPemArmour, NULL, &data);
+ if (err != noErr) {
+ continue;
+ }
+ if (data != NULL) {
+ CFDataAppendBytes(appendTo, CFDataGetBytePtr(data), CFDataGetLength(data));
+ CFRelease(data);
+ }
+ }
+ CFRelease(certs);
+ }
+ *pemRoots = combinedData;
+ *untrustedPemRoots = combinedUntrustedData;
+ return 0;
+}
+*/
+import "C"
+import (
+ "crypto/x509"
+ "errors"
+ "unsafe"
+)
+
+func loadSystemRoots() (*x509.CertPool, error) {
+ roots := x509.NewCertPool()
+
+ var data C.CFDataRef = 0
+ var untrustedData C.CFDataRef = 0
+ err := C._FetchPEMRoots(&data, &untrustedData, C.bool(debugDarwinRoots))
+ if err == -1 {
+ // TODO: better error message
+ return nil, errors.New("crypto/x509: failed to load darwin system roots with cgo")
+ }
+
+ defer C.CFRelease(C.CFTypeRef(data))
+ buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data)))
+ roots.AppendCertsFromPEM(buf)
+ if untrustedData == 0 {
+ return roots, nil
+ }
+ defer C.CFRelease(C.CFTypeRef(untrustedData))
+ buf = C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(untrustedData)), C.int(C.CFDataGetLength(untrustedData)))
+ untrustedRoots := x509.NewCertPool()
+ untrustedRoots.AppendCertsFromPEM(buf)
+
+ trustedRoots := x509.NewCertPool()
+ for _, c := range (*CertPool)(unsafe.Pointer(roots)).certs {
+ if !(*CertPool)(unsafe.Pointer(untrustedRoots)).contains(c) {
+ trustedRoots.AddCert(c)
+ }
+ }
+ return trustedRoots, nil
+}
diff --git a/cmd/macos-roots-test/root_darwin.go b/cmd/macos-roots-test/root_darwin.go
new file mode 100644
index 0000000..25cbcd1
--- /dev/null
+++ b/cmd/macos-roots-test/root_darwin.go
@@ -0,0 +1,174 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "crypto/x509"
+ "encoding/pem"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "os/user"
+ "path/filepath"
+ "sync"
+)
+
+var debugDarwinRoots = true
+
+// This code is only used when compiling without cgo.
+// It is here, instead of root_nocgo_darwin.go, so that tests can check it
+// even if the tests are run with cgo enabled.
+// The linker will not include these unused functions in binaries built with cgo enabled.
+
+// execSecurityRoots finds the macOS list of trusted root certificates
+// using only command-line tools. This is our fallback path when cgo isn't available.
+//
+// The strategy is as follows:
+//
+// 1. Run "security find-certificate" to dump the list of system root
+// CAs in PEM format.
+//
+// 2. For each dumped cert, conditionally verify it with "security
+// verify-cert" if that cert was not in the SystemRootCertificates
+// keychain, which can't have custom trust policies.
+//
+// We need to run "verify-cert" for all certificates not in SystemRootCertificates
+// because there might be certificates in the keychains without a corresponding
+// trust entry, in which case the logic is complicated (see root_cgo_darwin.go).
+//
+// TODO: actually parse the "trust-settings-export" output and apply the full
+// logic. See Issue 26830.
+func execSecurityRoots() (*x509.CertPool, error) {
+ keychains := []string{"/Library/Keychains/System.keychain"}
+
+ // Note that this results in trusting roots from $HOME/... (the environment
+ // variable), which might not be expected.
+ u, err := user.Current()
+ if err != nil {
+ if debugDarwinRoots {
+ fmt.Printf("crypto/x509: get current user: %v\n", err)
+ }
+ } else {
+ keychains = append(keychains,
+ filepath.Join(u.HomeDir, "/Library/Keychains/login.keychain"),
+
+ // Fresh installs of Sierra use a slightly different path for the login keychain
+ filepath.Join(u.HomeDir, "/Library/Keychains/login.keychain-db"),
+ )
+ }
+
+ var (
+ mu sync.Mutex
+ roots = x509.NewCertPool()
+ numVerified int // number of execs of 'security verify-cert', for debug stats
+ wg sync.WaitGroup
+ verifyCh = make(chan *x509.Certificate)
+ )
+
+ // Using 4 goroutines to pipe into verify-cert seems to be
+ // about the best we can do. The verify-cert binary seems to
+ // just RPC to another server with coarse locking anyway, so
+ // running 16 at a time for instance doesn't help at all.
+ for i := 0; i < 4; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for cert := range verifyCh {
+ valid := verifyCertWithSystem(cert)
+
+ mu.Lock()
+ numVerified++
+ if valid {
+ roots.AddCert(cert)
+ }
+ mu.Unlock()
+ }
+ }()
+ }
+ err = forEachCertInKeychains(keychains, func(cert *x509.Certificate) {
+ verifyCh <- cert
+ })
+ if err != nil {
+ return nil, err
+ }
+ close(verifyCh)
+ wg.Wait()
+
+ if debugDarwinRoots {
+ fmt.Printf("crypto/x509: ran security verify-cert %d times\n", numVerified)
+ }
+
+ err = forEachCertInKeychains([]string{
+ "/System/Library/Keychains/SystemRootCertificates.keychain",
+ }, roots.AddCert)
+ if err != nil {
+ return nil, err
+ }
+
+ return roots, nil
+}
+
+func forEachCertInKeychains(paths []string, f func(*x509.Certificate)) error {
+ args := append([]string{"find-certificate", "-a", "-p"}, paths...)
+ cmd := exec.Command("/usr/bin/security", args...)
+ data, err := cmd.Output()
+ if err != nil {
+ return err
+ }
+ for len(data) > 0 {
+ var block *pem.Block
+ block, data = pem.Decode(data)
+ if block == nil {
+ break
+ }
+ if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+ continue
+ }
+ cert, err := x509.ParseCertificate(block.Bytes)
+ if err != nil {
+ continue
+ }
+ f(cert)
+ }
+ return nil
+}
+
+func verifyCertWithSystem(cert *x509.Certificate) bool {
+ data := pem.EncodeToMemory(&pem.Block{
+ Type: "CERTIFICATE", Bytes: cert.Raw,
+ })
+
+ f, err := ioutil.TempFile("", "cert")
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "can't create temporary file for cert: %v", err)
+ return false
+ }
+ defer os.Remove(f.Name())
+ if _, err := f.Write(data); err != nil {
+ fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+ return false
+ }
+ if err := f.Close(); err != nil {
+ fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)
+ return false
+ }
+ cmd := exec.Command("/usr/bin/security", "verify-cert", "-p", "ssl", "-c", f.Name(), "-l", "-L")
+ var stderr bytes.Buffer
+ if debugDarwinRoots {
+ cmd.Stderr = &stderr
+ }
+ if err := cmd.Run(); err != nil {
+ if debugDarwinRoots {
+ fmt.Printf("crypto/x509: verify-cert rejected %s: %q\n", cert.Subject, bytes.TrimSpace(stderr.Bytes()))
+ }
+ return false
+ }
+ if debugDarwinRoots {
+ fmt.Printf("crypto/x509: verify-cert approved %s\n", cert.Subject)
+ }
+ return true
+}
