package tls import ( "context" "crypto/tls" "fmt" "io/ioutil" "net/http" "net/url" "os" "strconv" "strings" "time" "github.com/pkg/errors" "github.com/rancher/dynamiclistener" "github.com/rancher/dynamiclistener/cert" "github.com/rancher/dynamiclistener/server" "github.com/rancher/dynamiclistener/storage/kubernetes" "github.com/rancher/norman/types/convert" "github.com/rancher/rancher/pkg/namespace" "github.com/rancher/rancher/pkg/settings" "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps" appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1" "github.com/rancher/wrangler/v3/pkg/generated/controllers/core" corev1controllers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ) const ( rancherCertFile = "/etc/rancher/ssl/cert.pem" rancherKeyFile = "/etc/rancher/ssl/key.pem" rancherCACertsFile = "/etc/rancher/ssl/cacerts.pem" commonName = "rancher" ) type internalAPI struct{} var ( InternalAPI = internalAPI{} ) func ListenAndServe(ctx context.Context, restConfig *rest.Config, handler http.Handler, bindHost string, httpsPort, httpPort int, acmeDomains []string, noCACerts bool) error { restConfig = rest.CopyConfig(restConfig) restConfig.Timeout = 10 * time.Minute opts := &server.ListenOpts{} var err error core, err := core.NewFactoryFromConfig(restConfig) if err != nil { return err } apps, err := apps.NewFactoryFromConfig(restConfig) if err != nil { return err } if httpsPort != 0 { opts, err = SetupListener(core.Core().V1().Secret(), acmeDomains, noCACerts) if err != nil { return errors.Wrap(err, "failed to setup TLS listener") } } opts.BindHost = bindHost migrateConfig(ctx, restConfig, opts) backoff := wait.Backoff{ Duration: 100 * time.Millisecond, Factor: 2, Steps: 3, } // Try listen and serve over if there is an already exist error which comes from // creating the ca. Rancher will hit this error during HA startup as all servers // will race to create the ca secret. err = wait.ExponentialBackoff(backoff, func() (bool, error) { if err := server.ListenAndServe(ctx, httpsPort, httpPort, handler, opts); err != nil { if apierrors.IsAlreadyExists(err) { return false, nil } return false, err } return true, nil }) if err != nil { return errors.Wrap(err, "failed to ListenAndServe") } internalPort := 0 if httpsPort != 0 { internalPort = httpsPort + 1 } serverOptions := &server.ListenOpts{ Storage: opts.Storage, Secrets: opts.Secrets, CAName: "tls-rancher-internal-ca", CANamespace: namespace.System, CertNamespace: namespace.System, CertName: "tls-rancher-internal", } clusterIP, err := getClusterIP(core.Core().V1().Service()) if err != nil { return err } hostIPs, err := getHostIPs(apps.Apps().V1().Deployment(), core.Core().V1().Node()) if err != nil { return err } if clusterIP != "" { hostIPs = append(hostIPs, clusterIP) } if len(hostIPs) > 0 { serverOptions.TLSListenerConfig = dynamiclistener.Config{ SANs: hostIPs, } } internalAPICtx := context.WithValue(ctx, InternalAPI, true) err = wait.ExponentialBackoff(backoff, func() (bool, error) { if err := server.ListenAndServe(internalAPICtx, internalPort, 0, handler, serverOptions); err != nil { if apierrors.IsAlreadyExists(err) { return false, nil } return false, err } return true, nil }) if err != nil { return errors.Wrap(err, "failed to ListenAndServe for fleet") } if err := core.Start(ctx, 5); err != nil { return err } <-ctx.Done() return ctx.Err() } func migrateConfig(ctx context.Context, restConfig *rest.Config, opts *server.ListenOpts) { c, err := dynamic.NewForConfig(restConfig) if err != nil { return } config, err := c.Resource(schema.GroupVersionResource{ Group: "management.cattle.io", Version: "v3", Resource: "listenconfigs", }).Get(ctx, "cli-config", metav1.GetOptions{}) if err != nil { return } known := convert.ToStringSlice(config.Object["knownIps"]) for k := range convert.ToMapInterface(config.Object["generatedCerts"]) { if strings.HasPrefix(k, "local/") { continue } known = append(known, k) } for _, k := range known { k = strings.SplitN(k, ":", 2)[0] found := false for _, san := range opts.TLSListenerConfig.SANs { if san == k { found = true break } } if !found { opts.TLSListenerConfig.SANs = append(opts.TLSListenerConfig.SANs, k) } } } func SetupListener(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (*server.ListenOpts, error) { caForAgent, noCACerts, opts, err := readConfig(secrets, acmeDomains, noCACerts) if err != nil { return nil, err } if noCACerts { caForAgent = "" } else if caForAgent == "" { caCert, caKey, err := kubernetes.LoadOrGenCA(secrets, opts.CANamespace, opts.CAName) if err != nil { return nil, err } caForAgent = string(cert.EncodeCertPEM(caCert)) opts.CA = caCert opts.CAKey = caKey } caForAgent = strings.TrimSpace(caForAgent) if settings.CACerts.Get() != caForAgent { if err := settings.CACerts.Set(caForAgent); err != nil { return nil, err } } return opts, nil } func readConfig(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (string, bool, *server.ListenOpts, error) { var ( ca string err error ) tlsConfig, err := baseTLSConfig(settings.TLSMinVersion.Get(), settings.TLSCiphers.Get()) if err != nil { return "", noCACerts, nil, err } expiration, err := strconv.Atoi(settings.RotateCertsIfExpiringInDays.Get()) if err != nil { return "", noCACerts, nil, errors.Wrapf(err, "parsing %s", settings.RotateCertsIfExpiringInDays.Get()) } sans := []string{"localhost", "127.0.0.1", "rancher.cattle-system"} ip, err := net.ChooseHostInterface() if err == nil { sans = append(sans, ip.String()) } opts := &server.ListenOpts{ Secrets: secrets, CAName: "tls-rancher", CANamespace: "cattle-system", CertNamespace: "cattle-system", AcmeDomains: acmeDomains, TLSListenerConfig: dynamiclistener.Config{ TLSConfig: tlsConfig, ExpirationDaysCheck: expiration, SANs: sans, FilterCN: filterCN, CloseConnOnCertChange: true, }, } // ACME / Let's Encrypt // If --acme-domain is set, configure and return if len(acmeDomains) > 0 { return "", true, opts, nil } // Mounted certificates // If certificate file/key are set certFileExists := fileExists(rancherCertFile) keyFileExists := fileExists(rancherKeyFile) // If certificate file exists but not certificate key, or other way around, error out if (certFileExists && !keyFileExists) || (!certFileExists && keyFileExists) { return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set both certificate file and certificate key file (one is missing)") } caFileExists := fileExists(rancherCACertsFile) // If certificate file and certificate key file exists, load files into listenConfig if certFileExists && keyFileExists { cert, err := tls.LoadX509KeyPair(rancherCertFile, rancherKeyFile) if err != nil { return "", noCACerts, nil, err } opts.TLSListenerConfig.TLSConfig.Certificates = []tls.Certificate{cert} // Selfsigned needs cacerts, recognized CA needs --no-cacerts but can't be used together if (caFileExists && noCACerts) || (!caFileExists && !noCACerts) { return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time") } // Load cacerts if exists if caFileExists { ca, err = readPEM(rancherCACertsFile) if err != nil { return "", noCACerts, nil, err } } return ca, noCACerts, opts, nil } // External termination // We need to check if cacerts is passed or if --no-cacerts is used (when not providing certificate file and key) // If cacerts is passed if caFileExists { // We can't have --no-cacerts if noCACerts { return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time") } ca, err = readPEM(rancherCACertsFile) if err != nil { return "", noCACerts, nil, err } } // No certificates mounted or only --no-cacerts used return ca, noCACerts, opts, nil } func getClusterIP(services corev1controllers.ServiceController) (string, error) { service, err := services.Get(namespace.System, commonName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return "", nil } return "", err } if service.Spec.ClusterIP == "" { return "", fmt.Errorf("waiting on service %s/rancher to be assigned a ClusterIP", namespace.System) } return service.Spec.ClusterIP, nil } func getHostIPs(deployments appscontrollers.DeploymentController, nodes corev1controllers.NodeController) ([]string, error) { deployment, err := deployments.Get(namespace.System, commonName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return nil, nil } return nil, err } for _, container := range deployment.Spec.Template.Spec.Containers { if container.Name != commonName { continue } for _, port := range container.Ports { if port.HostIP != "" || port.HostPort != 0 { return collectNodeIPs(nodes) } } } return nil, nil } func collectNodeIPs(nodeController corev1controllers.NodeController) ([]string, error) { nodes, err := nodeController.List(metav1.ListOptions{}) if err != nil { return nil, err } nodeIPs := make([]string, 0, 2*len(nodes.Items)) for _, node := range nodes.Items { for _, ip := range node.Status.Addresses { if ip.Type == v1.NodeInternalIP || ip.Type == v1.NodeExternalIP { nodeIPs = append(nodeIPs, ip.Address) } } } return nodeIPs, nil } func filterCN(cns ...string) []string { serverURL := settings.ServerURL.Get() if serverURL == "" { return cns } u, err := url.Parse(serverURL) if err != nil { logrus.Errorf("invalid server-url, can not parse %s: %v", serverURL, err) return cns } host := u.Hostname() if host != "" { return []string{host} } return cns } func fileExists(path string) bool { if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { return false } } return true } func readPEM(path string) (string, error) { content, err := ioutil.ReadFile(path) if err != nil { return "", err } return string(content), nil }