268 lines
7.7 KiB
Go
268 lines
7.7 KiB
Go
package metrics
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/gorilla/mux"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/rancher/norman/httperror"
|
|
"github.com/rancher/rancher/pkg/auth/util"
|
|
"github.com/rancher/rancher/pkg/clustermanager"
|
|
"github.com/rancher/rancher/pkg/settings"
|
|
"github.com/rancher/rancher/pkg/types/config"
|
|
"github.com/rancher/wrangler/v3/pkg/ticker"
|
|
authV1 "k8s.io/api/authorization/v1"
|
|
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/client-go/kubernetes"
|
|
)
|
|
|
|
var (
|
|
prometheusMetrics = false
|
|
|
|
gcInterval = time.Duration(60 * time.Second)
|
|
|
|
clusterOwner = prometheus.NewGaugeVec(
|
|
prometheus.GaugeOpts{
|
|
Subsystem: "cluster_manager",
|
|
Name: "cluster_owner",
|
|
Help: "Set a cluster owner to determine which Rancher server is running a clusters controllers",
|
|
},
|
|
[]string{"cluster", "owner"},
|
|
)
|
|
)
|
|
|
|
type metricsHandler struct {
|
|
clusterManager *clustermanager.Manager
|
|
k8sClient kubernetes.Interface
|
|
next http.Handler
|
|
}
|
|
|
|
type ClusterCounts struct {
|
|
AppRevisions int `json:"appRevisions"`
|
|
CatalogTemplateVersions int `json:"catalogTemplateVersions"`
|
|
Projects int `json:"projects"`
|
|
ConfigMaps int `json:"configMaps"`
|
|
Secrets int `json:"secrets"`
|
|
Namespaces int `json:"namespaces"`
|
|
Nodes int `json:"nodes"`
|
|
RoleBindings int `json:"roleBindings"`
|
|
ClusterRoleBindings int `json:"clusterRoleBindings"`
|
|
}
|
|
|
|
func NewMetricsHandler(scaledContext *config.ScaledContext, clusterManager *clustermanager.Manager, handler http.Handler) http.Handler {
|
|
return &metricsHandler{
|
|
clusterManager: clusterManager,
|
|
k8sClient: scaledContext.K8sClient,
|
|
next: handler,
|
|
}
|
|
}
|
|
|
|
func (h *metricsHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
|
|
var reqGroup []string
|
|
if g, ok := req.Header["Impersonate-Group"]; ok {
|
|
reqGroup = g
|
|
}
|
|
|
|
review := authV1.SubjectAccessReview{
|
|
Spec: authV1.SubjectAccessReviewSpec{
|
|
User: req.Header.Get("Impersonate-User"),
|
|
Groups: reqGroup,
|
|
ResourceAttributes: &authV1.ResourceAttributes{
|
|
Verb: "get",
|
|
Resource: "ranchermetrics",
|
|
Group: "management.cattle.io",
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := h.k8sClient.AuthorizationV1().SubjectAccessReviews().Create(req.Context(), &review, metav1.CreateOptions{})
|
|
if err != nil {
|
|
util.ReturnHTTPError(rw, req, 500, err.Error())
|
|
return
|
|
}
|
|
|
|
if !result.Status.Allowed {
|
|
util.ReturnHTTPError(rw, req, 401, "Unauthorized")
|
|
return
|
|
}
|
|
|
|
if id := mux.Vars(req)["clusterID"]; id != "" {
|
|
h.getClusterObjectCount(id, rw, req)
|
|
return
|
|
}
|
|
|
|
h.next.ServeHTTP(rw, req)
|
|
}
|
|
|
|
// getClusterObjectCount uses the caches to get the number of items in a cluster and return a json blob with this
|
|
// information. The count is based off the cluster itself and where the object lives, not the count as you would
|
|
// see through the UI. For example projects only live in the management cluster so the count would only be displayed
|
|
// if getting 'local'. The count would show as 0 for a downstream cluster since the CR that backs the project only
|
|
// lives in management.
|
|
func (h *metricsHandler) getClusterObjectCount(clusterID string, rw http.ResponseWriter, req *http.Request) {
|
|
cluster, err := h.clusterManager.UserContext(clusterID)
|
|
if err != nil {
|
|
var (
|
|
normanError *httperror.APIError
|
|
k8sError *k8serrors.StatusError
|
|
code int
|
|
message string
|
|
)
|
|
|
|
if errors.As(err, &normanError) {
|
|
code = normanError.Code.Status
|
|
message = normanError.Message
|
|
} else if errors.As(err, &k8sError) {
|
|
code = int(k8sError.ErrStatus.Code)
|
|
message = k8sError.Error()
|
|
}
|
|
util.ReturnHTTPError(rw, req, code, message)
|
|
return
|
|
}
|
|
|
|
var cc ClusterCounts
|
|
labelSelector := labels.NewSelector()
|
|
|
|
ar, err := cluster.Project.AppRevisions(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.AppRevisions = len(ar)
|
|
|
|
// None of these resources are pushed downstream to a user cluster so we can only count them in the management cluster.
|
|
if clusterID == "local" {
|
|
ctv, err := cluster.Management.Management.CatalogTemplateVersions(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.CatalogTemplateVersions = len(ctv)
|
|
|
|
projects, err := cluster.Management.Management.Projects(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.Projects = len(projects)
|
|
}
|
|
|
|
cf, err := cluster.Core.ConfigMaps(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.ConfigMaps = len(cf)
|
|
|
|
s, err := cluster.Core.Secrets(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.Secrets = len(s)
|
|
|
|
ns, err := cluster.Core.Namespaces(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.Namespaces = len(ns)
|
|
|
|
nodes, err := cluster.Core.Nodes(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.Nodes = len(nodes)
|
|
|
|
rb, err := cluster.RBAC.RoleBindings(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.RoleBindings = len(rb)
|
|
|
|
crb, err := cluster.RBAC.ClusterRoleBindings(metav1.NamespaceAll).Controller().Lister().List(metav1.NamespaceAll, labelSelector)
|
|
if err != nil {
|
|
returnK8serror(err, rw, req)
|
|
return
|
|
}
|
|
cc.ClusterRoleBindings = len(crb)
|
|
|
|
js, err := json.Marshal(cc)
|
|
if err != nil {
|
|
util.ReturnHTTPError(rw, req, 500, err.Error())
|
|
return
|
|
}
|
|
|
|
rw.Header().Set("Content-Type", "application/json")
|
|
rw.Write(js)
|
|
}
|
|
|
|
// returnK8serror attempts to respond using the k8s error and its code/message
|
|
func returnK8serror(err error, rw http.ResponseWriter, req *http.Request) {
|
|
var k8sError *k8serrors.StatusError
|
|
if errors.As(err, &k8sError) {
|
|
util.ReturnHTTPError(rw, req, int(k8sError.ErrStatus.Code), k8sError.Error())
|
|
return
|
|
}
|
|
// Well, it wasn't a k8s error, give it back
|
|
util.ReturnHTTPError(rw, req, 500, err.Error())
|
|
}
|
|
|
|
func Register(ctx context.Context, scaledContext *config.ScaledContext) {
|
|
prometheusMetrics = true
|
|
|
|
// Cluster Owner
|
|
prometheus.MustRegister(clusterOwner)
|
|
|
|
// node and node core metrics
|
|
prometheus.MustRegister(numNodes)
|
|
prometheus.MustRegister(numCores)
|
|
|
|
gc := metricGarbageCollector{
|
|
clusterLister: scaledContext.Management.Clusters("").Controller().Lister(),
|
|
nodeLister: scaledContext.Management.Nodes("").Controller().Lister(),
|
|
endpointLister: scaledContext.Core.Endpoints(settings.Namespace.Get()).Controller().Lister(),
|
|
}
|
|
|
|
nm := &nodeMetrics{
|
|
nodeCache: scaledContext.Wrangler.Mgmt.Node().Cache(),
|
|
clusterCache: scaledContext.Wrangler.Mgmt.Cluster().Cache(),
|
|
}
|
|
|
|
go func(ctx context.Context) {
|
|
for range ticker.Context(ctx, gcInterval) {
|
|
gc.metricGarbageCollection()
|
|
}
|
|
}(ctx)
|
|
|
|
go nm.collect(ctx)
|
|
}
|
|
|
|
func SetClusterOwner(id, clusterID string) {
|
|
if prometheusMetrics {
|
|
clusterOwner.With(
|
|
prometheus.Labels{
|
|
"cluster": clusterID,
|
|
"owner": id,
|
|
}).Set(float64(1))
|
|
}
|
|
}
|
|
|
|
func UnsetClusterOwner(id, clusterID string) {
|
|
if prometheusMetrics {
|
|
clusterOwner.With(
|
|
prometheus.Labels{
|
|
"cluster": clusterID,
|
|
"owner": id,
|
|
}).Set(float64(0))
|
|
}
|
|
}
|