enable retries and timeouts on GCP KMS calls (#2548)

bobcallaway · web-flow · commit 455d37ba5e51 · 2025-08-01T07:47:33.000-04:00
* enable retries and timeouts on GCP KMS calls

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;

* fix lint

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;

* fix more lint issues in e2e tests

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;

---------

Signed-off-by: Bob Callaway &lt;bcallaway@google.com&gt;
diff --git a/cmd/rekor-server/app/root.go b/cmd/rekor-server/app/root.go
@@ -104,6 +104,8 @@ func init() {
 	rootCmd.PersistentFlags().String("rekor_server.signer", "memory",
 		`Rekor signer to use. Valid options are: [awskms://keyname, azurekms://keyname, gcpkms://keyname, hashivault://keyname, memory, tink, <filename containing PEM-encoded private key>].
 Memory and file-based signers should only be used for testing.`)
+	rootCmd.PersistentFlags().Uint("rekor_server.signer.gcpkms.retries", 0, "Number of retries for GCP KMS requests")
+	rootCmd.PersistentFlags().Uint("rekor_server.signer.gcpkms.timeout", 0, "sets the RPC timeout per call for GCP KMS requests in seconds, defaults to 0 (no timeout)")
 	rootCmd.PersistentFlags().String("rekor_server.signer-passwd", "", "Password to decrypt signer private key")
 	rootCmd.PersistentFlags().String("rekor_server.tink_kek_uri", "", "Key encryption key for decrypting Tink keyset. Valid options are [aws-kms://keyname, gcp-kms://keyname]")
 	rootCmd.PersistentFlags().String("rekor_server.tink_keyset_path", "", "Path to encrypted Tink keyset, containing private key to sign log checkpoints")
diff --git a/cmd/rekor-server/e2e_test.go b/cmd/rekor-server/e2e_test.go
@@ -218,7 +218,7 @@ func getTreeID(t *testing.T) int64 {
 	tidStr := strings.TrimSpace(strings.Split(out, "TreeID: ")[1])
 	tid, err := strconv.ParseInt(tidStr, 10, 64)
 	if err != nil {
-		t.Errorf(err.Error())
+		t.Error(err)
 	}
 	t.Log("Tree ID:", tid)
 	return tid
diff --git a/go.mod b/go.mod
@@ -1,8 +1,8 @@
 module github.com/sigstore/rekor
 
-go 1.23.0
+go 1.24
 
-toolchain go1.24.1
+toolchain go1.24.5
 
 require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
@@ -60,6 +60,7 @@ require (
 	github.com/go-redis/redismock/v9 v9.2.0
 	github.com/go-sql-driver/mysql v1.9.3
 	github.com/golang/mock v1.7.0-rc.1
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-retryablehttp v0.7.8
 	github.com/jmoiron/sqlx v1.4.0
@@ -68,7 +69,7 @@ require (
 	github.com/sigstore/protobuf-specs v0.5.0
 	github.com/sigstore/sigstore/pkg/signature/kms/aws v1.9.5
 	github.com/sigstore/sigstore/pkg/signature/kms/azure v1.9.5
-	github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.9.5
+	github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.9.6-0.20250729224751-181c5d3339b3
 	github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.9.5
 	github.com/stretchr/testify v1.10.0
 	github.com/tink-crypto/tink-go-awskms/v2 v2.1.0
@@ -126,7 +127,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
@@ -139,7 +140,7 @@ require (
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/vault/api v1.16.0 // indirect
-	github.com/jellydator/ttlcache/v3 v3.3.0 // indirect
+	github.com/jellydator/ttlcache/v3 v3.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
@@ -188,7 +189,7 @@ require (
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/google/go-containerregistry v0.20.3 // indirect
+	github.com/google/go-containerregistry v0.20.6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/google/wire v0.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
diff --git a/go.sum b/go.sum
diff --git a/pkg/api/api.go b/pkg/api/api.go
@@ -178,6 +178,8 @@ func NewAPI(treeID uint) (*API, error) {
 		FileSignerPassword:     viper.GetString("rekor_server.signer-passwd"),
 		TinkKEKURI:             viper.GetString("rekor_server.tink_kek_uri"),
 		TinkKeysetPath:         viper.GetString("rekor_server.tink_keyset_path"),
+		GCPKMSRetries:          viper.GetUint("rekor_server.signer.gcpkms.retries"),
+		GCPKMSTimeout:          viper.GetUint("rekor_server.signer.gcpkms.timeout"),
 	}
 	ranges, err := sharding.NewLogRanges(ctx, logClient, shardingConfig, tid, signingConfig)
 	if err != nil {
diff --git a/pkg/generated/restapi/configure_rekor_server.go b/pkg/generated/restapi/configure_rekor_server.go
@@ -351,7 +351,7 @@ func logAndServeError(w http.ResponseWriter, r *http.Request, err error) {
 		for _, embeddedErr := range compErr.Errors {
 			var maxBytesError *http.MaxBytesError
 			if parseErr, ok := embeddedErr.(*errors.ParseError); ok && go_errors.As(parseErr.Reason, &maxBytesError) {
-				err = errors.New(http.StatusRequestEntityTooLarge, http.StatusText(http.StatusRequestEntityTooLarge)) //nolint:govet
+				err = errors.New(http.StatusRequestEntityTooLarge, "Request Entity Too Large")
 				break
 			}
 		}
diff --git a/pkg/pki/x509/x509_test.go b/pkg/pki/x509/x509_test.go
@@ -40,22 +40,30 @@ import (
 // openssl genrsa -out myprivate.pem 512
 // openssl pkcs8 -topk8 -in myprivate.pem  -nocrypt'
 const pkcs1v15Priv = `-----BEGIN PRIVATE KEY-----
-MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEAoLEL57Kd5w8b5LCl
-SM+5mJbVYj4GoFXP/Gynfk6mDj7aANYWAkU74xkjz0BX2Nq0IT9DyxWI8aXZ8B6R
-YtbsPwIDAQABAkA2WgwTz5eXKsYdgR421YQKN6JvO1mUa9IQqFOy5jlGgbR+W5HG
-JfQVJKhCGMYYmByHgR0QDk/6gvJjhuszTHuJAiEA0siY/vE20zC1UHpPgDXXVSNN
-dKtM6YKBKSo47oTKQHsCIQDDKZgal50Cd3W+lOWpNO23QGZgBhJrJ70TpcPWGEsS
-DQIhAIDIMLnq1G1Z4B2IbRRPUP3icMtscbRlmNZ2xovsM8oLAiBluZh+w+gjEQFe
-hV3wBJajnf2+r2uKTvxO8WhSf/chQQIhAKzYjX2chfvPN6hRqeGeoPpRLXS8cdxC
-A4hZJRvZgkO3
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOhrZXYKydvbMvE2
+9/1BLMUqLfZj0hADX1H4cynJH6+l3Ax3t8BP4S/g0ZFEH81CD83MVYj5FBfNGaj/
+GYm16enH/ZmdY7tI9EVVXjF7H+NK0WZp04BFJiUqra8QsMNrz1WOEekzdlWNeHpx
+4EPNsO61msIizaV4ly01HX58OXvHAgMBAAECgYBBL6X0VpBJDpCaIM2rBTWWUv8z
+JMoM3bVFW0aJiLRPYlh2UrmBwaWp9QcyFAZLXmTqVo4C7cEZ79drk6jI+/GPrEjL
+jlE5H1cSqnfJV3wjDFBp+tQK2H5fX+BLJL0Jf0w/zpujZ1fwyIeeUK4AiEcc3po6
+BOf246ENiYSk7Osx4QJBAPQTK4gg9+fRmT6LiKCqfiZVjAdYqd3QIz1JZWlJQfI/
+HSNGeVeYaX7bElJsLGd24WqFOVs4KJOUqFdJ3rRbnhECQQDzxnGa6VQ+om2sPnJS
+3lEfXbIzYIFa9yLAM65D4+4apsgGW5l8+XuSmXepZnQYq30e3Pr0bjOShLz1Tkbc
+NoRXAkBEOleg5hZmpyC/ayH2R7Kb5K4QH6jcaKJxL2M521Cj9yCeC8U/x0s2OucU
+Q0jmY0UAEd3GshwlpRipzeyDXlkBAkEA622id+aR+u+plai1hny4wd8eY+n246A7
+yn3e9igh41Fhamp6gKz8/+cBHvQYeV7dNrpaD0iCvCU/zHUtkC2CfwJAbwgIMm4B
+MUSwy+0Fzm+LHmdl+MbXWLNebW/DBRE+ZgHEnl2+4Mg8CEh+fl1fbJrqWtMxz89h
+w80MgPfqGZbHFw==
 -----END PRIVATE KEY-----
 `
 
 // Extracted from above with:
 // openssl rsa -in myprivate.pem -pubout
 const pkcs1v15Pub = `-----BEGIN PUBLIC KEY-----
-MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKCxC+eynecPG+SwpUjPuZiW1WI+BqBV
-z/xsp35Opg4+2gDWFgJFO+MZI89AV9jatCE/Q8sViPGl2fAekWLW7D8CAwEAAQ==
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDoa2V2Csnb2zLxNvf9QSzFKi32
+Y9IQA19R+HMpyR+vpdwMd7fAT+Ev4NGRRB/NQg/NzFWI+RQXzRmo/xmJtenpx/2Z
+nWO7SPRFVV4xex/jStFmadOARSYlKq2vELDDa89VjhHpM3ZVjXh6ceBDzbDutZrC
+Is2leJctNR1+fDl7xwIDAQAB
 -----END PUBLIC KEY-----
 `
 
diff --git a/pkg/sharding/ranges.go b/pkg/sharding/ranges.go
@@ -147,7 +147,7 @@ func updateRange(ctx context.Context, logClient trillian.TrillianLogClient, r Lo
 
 	// Initialize shard signer
 	s, err := signer.New(ctx, r.SigningConfig.SigningSchemeOrKeyPath, r.SigningConfig.FileSignerPassword,
-		r.SigningConfig.TinkKEKURI, r.SigningConfig.TinkKeysetPath)
+		r.SigningConfig.TinkKEKURI, r.SigningConfig.TinkKeysetPath, r.SigningConfig.GCPKMSRetries, r.SigningConfig.GCPKMSTimeout)
 	if err != nil {
 		return LogRange{}, err
 	}
diff --git a/pkg/signer/memory_test.go b/pkg/signer/memory_test.go
@@ -24,7 +24,7 @@ import (
 func TestMemory(t *testing.T) {
 	ctx := context.Background()
 
-	m, err := New(ctx, "memory", "", "", "")
+	m, err := New(ctx, "memory", "", "", "", 0, 0)
 	if err != nil {
 		t.Fatalf("new memory: %v", err)
 	}
diff --git a/pkg/signer/signer.go b/pkg/signer/signer.go
@@ -20,15 +20,21 @@ import (
 	"context"
 	"crypto"
 	"strings"
+	"time"
 
+	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"github.com/sigstore/sigstore/pkg/signature"
 	"github.com/sigstore/sigstore/pkg/signature/kms"
 	"golang.org/x/exp/slices"
 
+	"google.golang.org/api/option"
+	"google.golang.org/grpc"
+
+	"github.com/sigstore/sigstore/pkg/signature/kms/gcp"
+
 	// these are imported to load the providers via init() calls
 	_ "github.com/sigstore/sigstore/pkg/signature/kms/aws"
 	_ "github.com/sigstore/sigstore/pkg/signature/kms/azure"
-	_ "github.com/sigstore/sigstore/pkg/signature/kms/gcp"
 	_ "github.com/sigstore/sigstore/pkg/signature/kms/hashivault"
 )
 
@@ -38,20 +44,27 @@ type SigningConfig struct {
 	FileSignerPassword     string `json:"fileSignerPassword" yaml:"fileSignerPassword"`
 	TinkKEKURI             string `json:"tinkKEKURI" yaml:"tinkKEKURI"`
 	TinkKeysetPath         string `json:"tinkKeysetPath" yaml:"tinkKeysetPath"`
+	GCPKMSRetries          uint   `json:"gcpkmsRetries" yaml:"gcpkmsRetries"`
+	GCPKMSTimeout          uint   `json:"gcpkmsTimeout" yaml:"gcpkmsTimeout"`
 }
 
 func (sc SigningConfig) IsUnset() bool {
 	return sc.SigningSchemeOrKeyPath == "" && sc.FileSignerPassword == "" &&
 		sc.TinkKEKURI == "" && sc.TinkKeysetPath == ""
 }
 
-func New(ctx context.Context, signer, pass, tinkKEKURI, tinkKeysetPath string) (signature.Signer, error) {
+func New(ctx context.Context, signer, pass, tinkKEKURI, tinkKeysetPath string, gcpkmsretries, gcpkmstimeout uint) (signature.Signer, error) {
 	switch {
 	case slices.ContainsFunc(kms.SupportedProviders(),
 		func(s string) bool {
 			return strings.HasPrefix(signer, s)
 		}):
-		return kms.Get(ctx, signer, crypto.SHA256)
+		opts := make([]signature.RPCOption, 0)
+		if strings.HasPrefix(signer, gcp.ReferenceScheme) {
+			callOpts := []grpc_retry.CallOption{grpc_retry.WithMax(gcpkmsretries), grpc_retry.WithPerRetryTimeout(time.Duration(gcpkmstimeout) * time.Second)}
+			opts = append(opts, gcp.WithGoogleAPIClientOption(option.WithGRPCDialOption(grpc.WithUnaryInterceptor(grpc_retry.UnaryClientInterceptor(callOpts...)))))
+		}
+		return kms.Get(ctx, signer, crypto.SHA256, opts...)
 	case signer == MemoryScheme:
 		return NewMemory()
 	case signer == TinkScheme:
diff --git a/pkg/types/dsse/v0.0.1/e2e_test.go b/pkg/types/dsse/v0.0.1/e2e_test.go
@@ -439,12 +439,12 @@ func TestSendingEntryWithClientComputedHashes(t *testing.T) {
 	sha.Write(b)
 	payloadHashBytes := sha.Sum(nil)
 	payloadHashStr := hex.EncodeToString(payloadHashBytes)
-	t.Logf(payloadHashStr)
+	t.Log(payloadHashStr)
 	sha.Reset()
 	sha.Write(eb)
 	envelopeHashBytes := sha.Sum(nil)
 	envelopeHashStr := hex.EncodeToString(envelopeHashBytes)
-	t.Logf(envelopeHashStr)
+	t.Log(envelopeHashStr)
 
 	ap := types.ArtifactProperties{
 		ArtifactBytes:  eb,
diff --git a/tests/e2e_test.go b/tests/e2e_test.go
@@ -710,7 +710,7 @@ func getTreeID(t *testing.T) int64 {
 	tidStr := strings.TrimSpace(strings.Split(out, "TreeID: ")[1])
 	tid, err := strconv.ParseInt(tidStr, 10, 64)
 	if err != nil {
-		t.Errorf(err.Error())
+		t.Error(err)
 	}
 	t.Log("Tree ID:", tid)
 	return tid
@@ -721,7 +721,7 @@ func getTotalTreeSize(t *testing.T) int64 {
 	sizeStr := strings.Fields(strings.Split(out, "Total Tree Size: ")[1])[0]
 	size, err := strconv.ParseInt(sizeStr, 10, 64)
 	if err != nil {
-		t.Errorf(err.Error())
+		t.Error(err)
 	}
 	t.Log("Total Tree Size:", size)
 	return size
