acme/autocert: support ACME RFC 8555

The Manager now uses RFC 8555 implementation of Let's Encrypt by default.
Existing users need not do any manual upgrades. If you vendor
acme/autocert, it is enough to just rebuild your binaries at this CL.

If there's an account key stored in Manager's cache which has been used
with an earlier Let's Encrypt implementation (aka v1 or draft-02),
it will be automatically re-registered with the new endpoint.

One notable change is the CAServer from internal/acmetest was amended
to simulate a CA implementing RFC 8555, replacing draft-02.
Support for both RFC and draft-02 seemed too complicated and not worth
the benefits: the old pre-RFC bits will be removed from both acme and
acme/autocert packages at some point.

Fixes golang/go#21081

Change-Id: Id530758ac612b1c20f9df51c4d10f770e5f41ecf
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/199520
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: x1ddos

acme/autocert/autocert.go

@@ -35,6 +35,9 @@ import (
 	"golang.org/x/net/idna"
 )
 
+// DefaultACMEDirectory is the default ACME Directory URL used when the Manager's Client is nil.
+const DefaultACMEDirectory = "https://acme-v02.api.letsencrypt.org/directory"
+
 // createCertRetryAfter is how much time to wait before removing a failed state
 // entry due to an unsuccessful createCert call.
 // This is a variable instead of a const for testing.
@@ -135,9 +138,10 @@ type Manager struct {
 	// Client is used to perform low-level operations, such as account registration
 	// and requesting new certificates.
 	//
-	// If Client is nil, a zero-value acme.Client is used with acme.LetsEncryptURL
-	// as directory endpoint. If the Client.Key is nil, a new ECDSA P-256 key is
-	// generated and, if Cache is not nil, stored in cache.
+	// If Client is nil, a zero-value acme.Client is used with DefaultACMEDirectory
+	// as the directory endpoint.
+	// If the Client.Key is nil, a new ECDSA P-256 key is generated and,
+	// if Cache is not nil, stored in cache.
 	//
 	// Mutating the field after the first call of GetCertificate method will have no effect.
 	Client *acme.Client
@@ -640,78 +644,72 @@ func (m *Manager) certState(ck certKey) (*certState, error) {
 // authorizedCert starts the domain ownership verification process and requests a new cert upon success.
 // The key argument is the certificate private key.
 func (m *Manager) authorizedCert(ctx context.Context, key crypto.Signer, ck certKey) (der [][]byte, leaf *x509.Certificate, err error) {
-	client, err := m.acmeClient(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if err := m.verify(ctx, client, ck.domain); err != nil {
-		return nil, nil, err
-	}
 	csr, err := certRequest(key, ck.domain, m.ExtraExtensions)
 	if err != nil {
 		return nil, nil, err
 	}
-	der, _, err = client.CreateCert(ctx, csr, 0, true)
+
+	client, err := m.acmeClient(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
-	leaf, err = validCert(ck, der, key, m.now())
+	dir, err := client.Discover(ctx)
 	if err != nil {
 		return nil, nil, err
 	}
-	return der, leaf, nil
-}
 
-// revokePendingAuthz revokes all authorizations idenfied by the elements of uri slice.
-// It ignores revocation errors.
-func (m *Manager) revokePendingAuthz(ctx context.Context, uri []string) {
-	client, err := m.acmeClient(ctx)
-	if err != nil {
-		return
+	var chain [][]byte
+	switch {
+	// Pre-RFC legacy CA.
+	case dir.OrderURL == "":
+		if err := m.verify(ctx, client, ck.domain); err != nil {
+			return nil, nil, err
+		}
+		der, _, err := client.CreateCert(ctx, csr, 0, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		chain = der
+	// RFC 8555 compliant CA.
+	default:
+		o, err := m.verifyRFC(ctx, client, ck.domain)
+		if err != nil {
+			return nil, nil, err
+		}
+		der, _, err := client.CreateOrderCert(ctx, o.FinalizeURL, csr, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		chain = der
 	}
-	for _, u := range uri {
-		client.RevokeAuthorization(ctx, u)
+	leaf, err = validCert(ck, chain, key, m.now())
+	if err != nil {
+		return nil, nil, err
 	}
+	return chain, leaf, nil
 }
 
-// verify runs the identifier (domain) authorization flow
+// verify runs the identifier (domain) pre-authorization flow for legacy CAs
 // using each applicable ACME challenge type.
 func (m *Manager) verify(ctx context.Context, client *acme.Client, domain string) error {
-	// The list of challenge types we'll try to fulfill
-	// in this specific order.
-	challengeTypes := []string{"tls-alpn-01"}
-	m.tokensMu.RLock()
-	if m.tryHTTP01 {
-		challengeTypes = append(challengeTypes, "http-01")
-	}
-	m.tokensMu.RUnlock()
-
-	// Keep track of pending authzs and revoke the ones that did not validate.
-	pendingAuthzs := make(map[string]bool)
+	// Remove all hanging authorizations to reduce rate limit quotas
+	// after we're done.
+	var authzURLs []string
 	defer func() {
-		var uri []string
-		for k, pending := range pendingAuthzs {
-			if pending {
-				uri = append(uri, k)
-			}
-		}
-		if len(uri) > 0 {
-			// Use "detached" background context.
-			// The revocations need not happen in the current verification flow.
-			go m.revokePendingAuthz(context.Background(), uri)
-		}
+		go m.deactivatePendingAuthz(authzURLs)
 	}()
 
 	// errs accumulates challenge failure errors, printed if all fail
 	errs := make(map[*acme.Challenge]error)
+	challengeTypes := m.supportedChallengeTypes()
 	var nextTyp int // challengeType index of the next challenge type to try
 	for {
 		// Start domain authorization and get the challenge.
 		authz, err := client.Authorize(ctx, domain)
 		if err != nil {
 			return err
 		}
+		authzURLs = append(authzURLs, authz.URI)
 		// No point in accepting challenges if the authorization status
 		// is in a final state.
 		switch authz.Status {
@@ -721,8 +719,6 @@ func (m *Manager) verify(ctx context.Context, client *acme.Client, domain string
 			return fmt.Errorf("acme/autocert: invalid authorization %q", authz.URI)
 		}
 
-		pendingAuthzs[authz.URI] = true
-
 		// Pick the next preferred challenge.
 		var chal *acme.Challenge
 		for chal == nil && nextTyp < len(challengeTypes) {
@@ -752,11 +748,126 @@ func (m *Manager) verify(ctx context.Context, client *acme.Client, domain string
 			errs[chal] = err
 			continue
 		}
-		delete(pendingAuthzs, authz.URI)
 		return nil
 	}
 }
 
+// verifyRFC runs the identifier (domain) order-based authorization flow for RFC compliant CAs
+// using each applicable ACME challenge type.
+func (m *Manager) verifyRFC(ctx context.Context, client *acme.Client, domain string) (*acme.Order, error) {
+	// Try each supported challenge type starting with a new order each time.
+	// The nextTyp index of the next challenge type to try is shared across
+	// all order authorizations: if we've tried a challenge type once and it didn't work,
+	// it will most likely not work on another order's authorization either.
+	challengeTypes := m.supportedChallengeTypes()
+	nextTyp := 0 // challengeTypes index
+AuthorizeOrderLoop:
+	for {
+		o, err := client.AuthorizeOrder(ctx, acme.DomainIDs(domain))
+		if err != nil {
+			return nil, err
+		}
+		// Remove all hanging authorizations to reduce rate limit quotas
+		// after we're done.
+		defer func() {
+			go m.deactivatePendingAuthz(o.AuthzURLs)
+		}()
+
+		// Check if there's actually anything we need to do.
+		switch o.Status {
+		case acme.StatusReady:
+			// Already authorized.
+			return o, nil
+		case acme.StatusPending:
+			// Continue normal Order-based flow.
+		default:
+			return nil, fmt.Errorf("acme/autocert: invalid new order status %q; order URL: %q", o.Status, o.URI)
+		}
+
+		// Satisfy all pending authorizations.
+		for _, zurl := range o.AuthzURLs {
+			z, err := client.GetAuthorization(ctx, zurl)
+			if err != nil {
+				return nil, err
+			}
+			if z.Status != acme.StatusPending {
+				// We are interested only in pending authorizations.
+				continue
+			}
+			// Pick the next preferred challenge.
+			var chal *acme.Challenge
+			for chal == nil && nextTyp < len(challengeTypes) {
+				chal = pickChallenge(challengeTypes[nextTyp], z.Challenges)
+				nextTyp++
+			}
+			if chal == nil {
+				return nil, fmt.Errorf("acme/autocert: unable to satisfy %q for domain %q: no viable challenge type found", z.URI, domain)
+			}
+			// Respond to the challenge and wait for validation result.
+			cleanup, err := m.fulfill(ctx, client, chal, domain)
+			if err != nil {
+				continue AuthorizeOrderLoop
+			}
+			defer cleanup()
+			if _, err := client.Accept(ctx, chal); err != nil {
+				continue AuthorizeOrderLoop
+			}
+			if _, err := client.WaitAuthorization(ctx, z.URI); err != nil {
+				continue AuthorizeOrderLoop
+			}
+		}
+
+		// All authorizations are satisfied.
+		// Wait for the CA to update the order status.
+		o, err = client.WaitOrder(ctx, o.URI)
+		if err != nil {
+			continue AuthorizeOrderLoop
+		}
+		return o, nil
+	}
+}
+
+func pickChallenge(typ string, chal []*acme.Challenge) *acme.Challenge {
+	for _, c := range chal {
+		if c.Type == typ {
+			return c
+		}
+	}
+	return nil
+}
+
+func (m *Manager) supportedChallengeTypes() []string {
+	m.tokensMu.RLock()
+	defer m.tokensMu.RUnlock()
+	typ := []string{"tls-alpn-01"}
+	if m.tryHTTP01 {
+		typ = append(typ, "http-01")
+	}
+	return typ
+}
+
+// deactivatePendingAuthz relinquishes all authorizations identified by the elements
+// of the provided uri slice which are in "pending" state.
+// It ignores revocation errors.
+//
+// deactivatePendingAuthz takes no context argument and instead runs with its own
+// "detached" context because deactivations are done in a goroutine separate from
+// that of the main issuance or renewal flow.
+func (m *Manager) deactivatePendingAuthz(uri []string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	client, err := m.acmeClient(ctx)
+	if err != nil {
+		return
+	}
+	for _, u := range uri {
+		z, err := client.GetAuthorization(ctx, u)
+		if err == nil && z.Status == acme.StatusPending {
+			client.RevokeAuthorization(ctx, u)
+		}
+	}
+}
+
 // fulfill provisions a response to the challenge chal.
 // The cleanup is non-nil only if provisioning succeeded.
 func (m *Manager) fulfill(ctx context.Context, client *acme.Client, chal *acme.Challenge, domain string) (cleanup func(), err error) {
@@ -780,15 +891,6 @@ func (m *Manager) fulfill(ctx context.Context, client *acme.Client, chal *acme.C
 	return nil, fmt.Errorf("acme/autocert: unknown challenge type %q", chal.Type)
 }
 
-func pickChallenge(typ string, chal []*acme.Challenge) *acme.Challenge {
-	for _, c := range chal {
-		if c.Type == typ {
-			return c
-		}
-	}
-	return nil
-}
-
 // putCertToken stores the token certificate with the specified name
 // in both m.certTokens map and m.Cache.
 func (m *Manager) putCertToken(ctx context.Context, name string, cert *tls.Certificate) {
@@ -949,7 +1051,7 @@ func (m *Manager) acmeClient(ctx context.Context) (*acme.Client, error) {
 
 	client := m.Client
 	if client == nil {
-		client = &acme.Client{DirectoryURL: acme.LetsEncryptURL}
+		client = &acme.Client{DirectoryURL: DefaultACMEDirectory}
 	}
 	if client.Key == nil {
 		var err error
@@ -967,14 +1069,23 @@ func (m *Manager) acmeClient(ctx context.Context) (*acme.Client, error) {
 	}
 	a := &acme.Account{Contact: contact}
 	_, err := client.Register(ctx, a, m.Prompt)
-	if ae, ok := err.(*acme.Error); err == nil || ok && ae.StatusCode == http.StatusConflict {
-		// conflict indicates the key is already registered
+	if err == nil || isAccountAlreadyExist(err) {
 		m.client = client
 		err = nil
 	}
 	return m.client, err
 }
 
+// isAccountAlreadyExist reports whether the err, as returned from acme.Client.Register,
+// indicates the account has already been registered.
+func isAccountAlreadyExist(err error) bool {
+	if err == acme.ErrAccountAlreadyExists {
+		return true
+	}
+	ae, ok := err.(*acme.Error)
+	return ok && ae.StatusCode == http.StatusConflict
+}
+
 func (m *Manager) hostPolicy() HostPolicy {
 	if m.HostPolicy != nil {
 		return m.HostPolicy

acme/autocert/autocert_test.go

@@ -372,6 +372,7 @@ func testGetCertificate_tokenCache(t *testing.T, tokenAlg algorithmSupport) {
 	url, finish := startACMEServerStub(t, tokenCertFn(man2, tokenAlg), "example.org")
 	defer finish()
 	man1.Client = &acme.Client{DirectoryURL: url}
+	man2.Client = &acme.Client{DirectoryURL: url}
 	hello := clientHelloInfo("example.org", algECDSA)
 	if _, err := man1.GetCertificate(hello); err != nil {
 		t.Error(err)
@@ -400,15 +401,15 @@ func TestGetCertificate_ecdsaVsRSA(t *testing.T) {
 
 	cert, err := man.GetCertificate(clientHelloInfo("example.org", algECDSA))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 	if _, ok := cert.Leaf.PublicKey.(*ecdsa.PublicKey); !ok {
 		t.Error("an ECDSA client was served a non-ECDSA certificate")
 	}
 
 	cert, err = man.GetCertificate(clientHelloInfo("example.org", algRSA))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 	if _, ok := cert.Leaf.PublicKey.(*rsa.PublicKey); !ok {
 		t.Error("a RSA client was served a non-RSA certificate")
@@ -458,7 +459,7 @@ func TestGetCertificate_wrongCacheKeyType(t *testing.T) {
 	// The RSA cached cert should be silently ignored and replaced.
 	cert, err := man.GetCertificate(clientHelloInfo(exampleDomain, algECDSA))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 	if _, ok := cert.Leaf.PublicKey.(*ecdsa.PublicKey); !ok {
 		t.Error("an ECDSA client was served a non-ECDSA certificate")
@@ -778,8 +779,9 @@ func TestRevokeFailedAuthz(t *testing.T) {
 			http.Error(w, "won't accept tls-alpn-01 challenge", http.StatusBadRequest)
 		// http-01 challenge "accept" request.
 		case "/challenge/http-01":
-			// Accept but the authorization will be "expired".
-			w.Write([]byte("{}"))
+			// Refuse.
+			w.WriteHeader(http.StatusBadRequest)
+			w.Write([]byte(`{"status":"invalid"}`))
 		// Authorization requests.
 		case "/authz/0", "/authz/1", "/authz/2":
 			// Revocation requests.
@@ -803,8 +805,7 @@ func TestRevokeFailedAuthz(t *testing.T) {
 				return
 			}
 			// Authorization status requests.
-			// Simulate abandoned authorization, deleted by the CA.
-			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(`{"status":"pending"}`))
 		default:
 			http.NotFound(w, r)
 			t.Errorf("unrecognized r.URL.Path: %s", r.URL.Path)
@@ -1219,7 +1220,6 @@ func TestEndToEnd(t *testing.T) {
 	client := &http.Client{Transport: tr}
 	res, err := client.Get(us.URL)
 	if err != nil {
-		t.Logf("CA errors: %v", ca.Errors())
 		t.Fatal(err)
 	}
 	defer res.Body.Close()