diff --git a/.gitignore b/.gitignore index a323398e..06652228 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,12 @@ local-test.sh !vendor/**/zz_generated.* +# Plan documents +docs/plans + +# Claude Code +CLAUDE.md + # editor and IDE paraphernalia .idea *.swp diff --git a/go.mod b/go.mod index 6b416f2b..6f6ff812 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,8 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/samber/lo v1.47.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/zoumo/golib v0.2.0 // indirect ) require ( diff --git a/go.sum b/go.sum index a9fa4c2e..169b0cd6 100644 --- a/go.sum +++ b/go.sum @@ -546,6 +546,8 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= @@ -594,6 +596,8 @@ github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zoumo/golib v0.2.0 h1:K6W8WWrgnl2bXRvUaiXjAaiFKsCTHwnrBkBHZoFr8lE= +github.com/zoumo/golib v0.2.0/go.mod h1:gOMPRvDgn9m49tfHoKUb2RO0NqplNoe/qj5/ZrczjgQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/main.go b/main.go index 46855fb3..9f2118f6 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,6 @@ import ( "context" "flag" "os" - "path/filepath" "github.com/spf13/pflag" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -62,8 +61,8 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&certDir, "cert-dir", webhookTempCertDir(), "The directory that contains the server key and certificate. If not set, webhook server would look up the server key and certificate in {TempDir}/k8s-webhook-server/serving-certs") - flag.StringVar(&dnsName, "dns-name", "kusionstack-controller-manager.kusionstack-system.svc", "The DNS name of the webhook server.") + flag.StringVar(&certDir, "cert-dir", "/webhook-certs", "The directory that contains the server key and certificate for webhook.") + flag.StringVar(&dnsName, "dns-name", "controller-manager.kusionstack-system.svc", "The DNS name of the webhook server.") klog.InitFlags(nil) defer klog.Flush() @@ -122,9 +121,9 @@ func main() { } // +kubebuilder:scaffold:builder - setupLog.Info("initialize webhook") - if err := webhook.Initialize(context.Background(), config, dnsName, certDir); err != nil { - setupLog.Error(err, "unable to initialize webhook") + setupLog.Info("initialize webhook cert manager") + if err := webhook.Initialize(context.Background(), mgr, dnsName); err != nil { + setupLog.Error(err, "unable to initialize webhook cert manager") os.Exit(1) } @@ -143,7 +142,3 @@ func main() { os.Exit(1) } } - -func webhookTempCertDir() string { - return filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs") -} diff --git a/pkg/utils/pki_helpers.go b/pkg/utils/pki_helpers.go deleted file mode 100644 index 9836b632..00000000 --- a/pkg/utils/pki_helpers.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "crypto" - cryptorand "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math" - "math/big" - "time" - - "github.com/pkg/errors" - certutil "k8s.io/client-go/util/cert" -) - -const ( - certificateBlockType = "CERTIFICATE" - rsaKeySize = 2048 - duration365d = time.Hour * 24 * 365 -) - -// NewPrivateKey creates an RSA private key -func NewPrivateKey() (*rsa.PrivateKey, error) { - return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) -} - -// EncodeCertPEM returns PEM-encoded certificate data -func EncodeCertPEM(cert *x509.Certificate) []byte { - block := pem.Block{ - Type: certificateBlockType, - Bytes: cert.Raw, - } - return pem.EncodeToMemory(&block) -} - -// NewSignedCert creates a signed certificate using the given CA certificate and key -func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { - serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - if len(cfg.CommonName) == 0 { - return nil, errors.New("must specify a CommonName") - } - if len(cfg.Usages) == 0 { - return nil, errors.New("must specify at least one ExtKeyUsage") - } - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - DNSNames: cfg.AltNames.DNSNames, - IPAddresses: cfg.AltNames.IPs, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: time.Now().Add(duration365d).UTC(), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: cfg.Usages, - } - certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 3d93a971..93482308 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -18,31 +18,17 @@ package webhook import ( "context" - "crypto" - "crypto/rsa" - "crypto/x509" "os" - "path/filepath" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - "k8s.io/client-go/util/retry" "k8s.io/klog/v2" + certmanager "kusionstack.io/kube-utils/webhook/certmanager" "sigs.k8s.io/controller-runtime/pkg/manager" - - "kusionstack.io/kuperator/pkg/utils" ) const ( mutatingWebhookConfigurationName = "kusionstack-controller-manager-mutating" validatingWebhookConfigurationName = "kusionstack-controller-manager-validating" - webhookCertsSecretName = "kusionstack-webhook-certs" + webhookCertsSecretName = "webhook-certs" ) // AddToManagerFuncs is a list of functions to add all Webhook Servers to the Manager @@ -61,192 +47,30 @@ func AddToManager(m manager.Manager) error { return nil } -func Initialize(ctx context.Context, config *rest.Config, dnsName, certDir string) error { - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return err - } - return ensureWebhookCABundleAndCert(ctx, clientset, dnsName, certDir) -} - -func ensureWebhookCABundleAndCert(ctx context.Context, clientset *kubernetes.Clientset, dnsName, certDir string) error { - secret, err := ensureWebhookSecret(ctx, clientset, dnsName) - if err != nil { - return err - } - klog.Infof("webhook secret ensured, secret: %s", secret.Name) - - mwhc, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, mutatingWebhookConfigurationName, metav1.GetOptions{}) - if err != nil { - return err - } - - for i := range mwhc.Webhooks { - if mwhc.Webhooks[i].ClientConfig.CABundle == nil { - mwhc.Webhooks[i].ClientConfig.CABundle = secret.Data["ca.crt"] - } - } - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - _, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, mwhc, metav1.UpdateOptions{}) - return err - }) - if err != nil { +// Initialize sets up the webhook certificate manager with auto-rotation support. +// It uses kube-utils/certmanager which: +// - Automatically generates and rotates certificates when expired +// - Syncs certs from Secret to local filesystem +// - Updates CABundle in WebhookConfigurations +func Initialize(ctx context.Context, mgr manager.Manager, dnsName string) error { + cfg := certmanager.CertConfig{ + Host: dnsName, + Namespace: getNamespace(), + SecretName: webhookCertsSecretName, + MutatingWebhookNames: []string{mutatingWebhookConfigurationName}, + ValidatingWebhookNames: []string{validatingWebhookConfigurationName}, + } + + certMgr := certmanager.New(mgr, cfg) + if err := certMgr.SetupWithManager(mgr); err != nil { return err } - vwhc, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, validatingWebhookConfigurationName, metav1.GetOptions{}) - if err != nil { - return err - } + klog.InfoS("webhook cert manager initialized", + "secret", webhookCertsSecretName, + "namespace", getNamespace(), + "host", dnsName) - for i := range vwhc.Webhooks { - if vwhc.Webhooks[i].ClientConfig.CABundle == nil { - vwhc.Webhooks[i].ClientConfig.CABundle = secret.Data["ca.crt"] - } - } - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - _, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, vwhc, metav1.UpdateOptions{}) - return err - }) - if err != nil { - return err - } - klog.Infof("webhook ca bundle ensured, mutatingwebhookconfiguration: %s, validatingwebhookconfiguration: %s", mutatingWebhookConfigurationName, validatingWebhookConfigurationName) - - var tlsKey, tlsCert []byte - tlsKey, ok := secret.Data["tls.key"] - if !ok { - return errors.New("tls.key not found in secret") - } - tlsCert, ok = secret.Data["tls.crt"] - if !ok { - return errors.New("tls.crt not found in secret") - } - - err = ensureWebhookCert(certDir, tlsKey, tlsCert) - if err != nil { - return err - } - klog.Infof("webhook cert ensured, cert dir: %s", certDir) - - return nil -} - -func ensureWebhookSecret(ctx context.Context, clientset *kubernetes.Clientset, dnsName string) (secret *corev1.Secret, err error) { - var ( - found = true - dirty = false - ) - secret, err = clientset.CoreV1().Secrets(getNamespace()).Get(ctx, webhookCertsSecretName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - found = false - } else { - return - } - } - if found { - if secret.Data == nil || len(secret.Data) != 4 || - secret.Data["ca.key"] == nil || secret.Data["ca.crt"] == nil || - secret.Data["tls.key"] == nil || secret.Data["tls.crt"] == nil { - dirty = true - } - if !dirty { - return - } - } - - caKey, caCert, err := generateSelfSignedCACert() - if err != nil { - return - } - caKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(caKey) - if err != nil { - return - } - caCertPEM := utils.EncodeCertPEM(caCert) - - privateKey, signedCert, err := generateSelfSignedCert(caCert, caKey, dnsName) - if err != nil { - return - } - privateKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(privateKey) - if err != nil { - return - } - signedCertPEM := utils.EncodeCertPEM(signedCert) - - data := map[string][]byte{ - "ca.key": caKeyPEM, "ca.crt": caCertPEM, - "tls.key": privateKeyPEM, "tls.crt": signedCertPEM, - } - secret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: webhookCertsSecretName, - Namespace: getNamespace(), - }, - Data: data, - } - - var updatedSecret *corev1.Secret - err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - if dirty { - updatedSecret, err = clientset.CoreV1().Secrets(getNamespace()).Update(ctx, secret, metav1.UpdateOptions{}) - } else { - updatedSecret, err = clientset.CoreV1().Secrets(getNamespace()).Create(ctx, secret, metav1.CreateOptions{}) - } - return err - }) - return updatedSecret, err -} - -func generateSelfSignedCACert() (caKey *rsa.PrivateKey, caCert *x509.Certificate, err error) { - caKey, err = utils.NewPrivateKey() - if err != nil { - return - } - - caCert, err = cert.NewSelfSignedCACert(cert.Config{CommonName: "self-signed-k8s-cert"}, caKey) - - return -} - -func generateSelfSignedCert(caCert *x509.Certificate, caKey crypto.Signer, dnsName string) (privateKey *rsa.PrivateKey, signedCert *x509.Certificate, err error) { - privateKey, err = utils.NewPrivateKey() - if err != nil { - return - } - - signedCert, err = utils.NewSignedCert( - &cert.Config{ - CommonName: dnsName, - AltNames: cert.AltNames{DNSNames: []string{dnsName}}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - privateKey, caCert, caKey, - ) - - return -} - -func ensureWebhookCert(certDir string, tlsKey, tlsCert []byte) error { - if _, err := os.Stat(certDir); os.IsNotExist(err) { - err := os.MkdirAll(certDir, 0o777) - if err != nil { - return err - } - klog.Infof("cert dir is created: %s", certDir) - } - - keyFile := filepath.Join(certDir, "tls.key") - certFile := filepath.Join(certDir, "tls.crt") - - if err := os.WriteFile(keyFile, tlsKey, 0o644); err != nil { - return err - } - if err := os.WriteFile(certFile, tlsCert, 0o644); err != nil { - return err - } return nil }