一文读懂 Kubernetes APIServer 原理

前言 整个Kubernetes技术体系由声明式API以及Controller构成，而kube-apiserver是Kubernetes的声明式api server，并为其它组件交互提供了桥梁。因此加深对kube-apiserver的理解就显得至关重要了。 整体组件功能 kube-apiserver作为整个Kubernetes集群操作etcd的唯一入口，负责Kubernetes各资源的认证&鉴权，校验以及CRUD等操作，提供RESTful APIs，供其它组件调用： kube-apiserver包含三种APIServer： aggregatorServer：负责处理 apiregistration.k8s.io 组下的APIService资源请求，同时将来自 域名交易 用户的请求拦截转发给aggregated server(AA) kubeAPIServer：负责对请求的一些通用处理，包括：认证、鉴权以及各个内建资源(pod, deployment，service and etc)的REST服务等 apiExtensionsServer：负责CustomResourceDefinition（CRD）apiResources以及apiVersions的注册，同时处理CRD以及相应CustomResource（CR）的REST请求(如果对应CR不能被处理的话则会返回404)，也是apiserver Delegation的最后一环 另外还包括bootstrap-controller，主要负责Kubernetes default apiserver service的创建以及管理。 接下来将对上述组件进行概览性总结。 bootstrap-controller apiserver bootstrap-controller创建&运行逻辑在k8s.io/kubernetes/pkg/master目录 bootstrap-controller主要用于创建以及维护内部kubernetes default apiserver service kubernetes default apiserver service spec.selector为空，这是default apiserver service与其它正常service的最大区别，表明了这个特殊的service对应的endpoints不由endpoints controller控制，而是直接受kube-apiserver bootstrap-controller管理(maintained by this code, not by the pod selector) bootstrap-controller的几个主要功能如下： 创建 default、kube-system 和 kube-public 以及 kube-node-lease 命名空间 创建&维护kubernetes default apiserver service以及对应的endpoint 提供基于Service ClusterIP的检查及修复功能(--service-cluster-ip-range指定范围) 提供基于Service NodePort的检查及修复功能(--service-node-port-range指定范围) // k8s.io/kubernetes/pkg/master/controller.go:142// Start begins the core controller loops that must exist for bootstrapping// a cluster.func (c *Controller) Start() {     if c.runner != nil {         return     }     // Reconcile during first run removing itself until server is ready.     endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https", c.ExtraEndpointPorts)     if err := c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts); err != nil {         klog.Errorf("Unable to remove old endpoints from kubernetes service: %v", err)     }     repairClusterIPs := servicecontroller.NewRepair(c.ServiceClusterIPInterval, c.ServiceClient, c.EventClient, &c.ServiceClusterIPRange, c.ServiceClusterIPRegistry, &c.SecondaryServiceClusterIPRange, c.SecondaryServiceClusterIPRegistry)     repairNodePorts := portallocatorcontroller.NewRepair(c.ServiceNodePortInterval, c.ServiceClient, c.EventClient, c.ServiceNodePortRange, c.ServiceNodePortRegistry)     // run all of the controllers once prior to returning from Start.     if err := repairClusterIPs.RunOnce(); err != nil {         // If we fail to repair cluster IPs apiserver is useless. We should restart and retry.         klog.Fatalf("Unable to perform initial IP allocation check: %v", err)     }     if err := repairNodePorts.RunOnce(); err != nil {         // If we fail to repair node ports apiserver is useless. We should restart and retry.         klog.Fatalf("Unable to perform initial service nodePort check: %v", err)     }     // 定期执行bootstrap controller主要的四个功能(reconciliation)       c.runner = async.NewRunner(c.RunKubernetesNamespaces, c.RunKubernetesService, repairClusterIPs.RunUntil, repairNodePorts.RunUntil)     c.runner.Start() KubeAPIServer主要提供对内建API Resources的操作请求，为Kubernetes中各API Resources注册路由信息，同时暴露RESTful API，使集群中以及集群外的服务都可以通过RESTful API操作Kubernetes中的资源 另外，kubeAPIServer是整个Kubernetes apiserver的核心，下面将要讲述的aggregatorServer以及apiExtensionsServer都是建立在kubeAPIServer基础上进行扩展的(补充了Kubernetes对用户自定义资源的能力支持) kubeAPIServer最核心的功能是为Kubernetes内置资源添加路由，如下： 调用 m.InstallLegacyAPI 将核心 API Resources添加到路由中，在apiserver中即是以 /api 开头的 resource； 调用 m.InstallAPIs 将扩展的 API Resources添加到路由中，在apiserver中即是以 /apis 开头的 resource； // k8s.io/kubernetes/pkg/master/master.go:332// New returns a new instance of Master from the given config.// Certain config fields will be set to a default value if unset.// Certain config fields must be specified, including://   KubeletClientConfigfunc (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Master, error) {     ...     // 安装 LegacyAPI(core API)     // install legacy rest storage     if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {         legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{             StorageFactory:              c.ExtraConfig.StorageFactory,             ProxyTransport:              c.ExtraConfig.ProxyTransport,             KubeletClientConfig:         c.ExtraConfig.KubeletClientConfig,             EventTTL:                    c.ExtraConfig.EventTTL,             ServiceIPRange:              c.ExtraConfig.ServiceIPRange,             SecondaryServiceIPRange:     c.ExtraConfig.SecondaryServiceIPRange,             ServiceNodePortRange:        c.ExtraConfig.ServiceNodePortRange,             LoopbackClientConfig:        c.GenericConfig.LoopbackClientConfig,             ServiceAccountIssuer:        c.ExtraConfig.ServiceAccountIssuer,             ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,             APIAudiences:                c.GenericConfig.Authentication.APIAudiences,         }         if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil {             return nil, err         }     }     ...     // 安装 APIs(named groups apis)     if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {         return nil, err     }     ...     return m, nil } 整个kubeAPIServer提供了三类API Resource接口： core group：主要在 /api/v1 下； named groups：其 path 为 /apis/$GROUP/$VERSION； 系统状态的一些 API：如/metrics 、/version 等； kubeAPIServer会为每种API资源创建对应的RESTStorage，RESTStorage的目的是将每种资源的访问路径及其后端存储的操作对应起来：通过构造的REST Storage实现的接口判断该资源可以执行哪些操作（如：create、update等），将其对应的操作存入到action中，每一个操作对应一个标准的REST method，如create对应REST method为POST，而update对应REST method为PUT。最终根据actions数组依次遍历，对每一个操作添加一个handler(handler对应REST Storage实现的相关接口)，并注册到route，最终对外提供RESTful API，如下： // m.GenericAPIServer.InstallLegacyAPIGroup --> s.installAPIResources --> apiGroupVersion.InstallREST --> installer.Install --> a.registerResourceHandlers// k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go:181func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {     ...     // 1、判断该 resource 实现了哪些 REST 操作接口，以此来判断其支持的 verbs 以便为其添加路由     // what verbs are supported by the storage, used to know what verbs we support per path     creater, isCreater := storage.(rest.Creater)     namedCreater, isNamedCreater := storage.(rest.NamedCreater)     lister, isLister := storage.(rest.Lister)     getter, isGetter := storage.(rest.Getter)     ...     // 2、为 resource 添加对应的 actions(+根据是否支持 namespace)     // Get the list of actions for the given scope.     switch {     case !namespaceScoped:         // Handle non-namespace scoped resources like nodes.         resourcePath := resource         resourceParams := params         itemPath := resourcePath + "/{name}"         nameParams := append(params, nameParam)         proxyParams := append(nameParams, pathParam)         ...         // Handler for standard REST verbs (GET, PUT, POST and DELETE).         // Add actions at the resource path: /api/apiVersion/resource         actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)         actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)         ...     }     ...     // 3、从 rest.Storage 到 restful.Route 映射     // 为每个操作添加对应的 handler     for _, action := range actions {         ...         switch action.Verb {         ...         case "POST": // Create a resource.             var handler restful.RouteFunction             // 4、初始化 handler             if isNamedCreater {                 handler = restfulCreateNamedResource(namedCreater, reqScope, admit)             } else {                 handler = restfulCreateResource(creater, reqScope, admit)             }             handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)             ...             // 5、route 与 handler 进行绑定                 route := ws.POST(action.Path).To(handler).                 Doc(doc).                 Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).                 Operation("create"+namespaced+kind+strings.Title(subresource)+operationSuffix).                 Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).                 Returns(http.StatusOK, "OK", producedObject).                 // TODO: in some cases, the API may return a v1.Status instead of the versioned object                 // but currently go-restful can't handle multiple different objects being returned.                 Returns(http.StatusCreated, "Created", producedObject).                 Returns(http.StatusAccepted, "Accepted", producedObject).                 Reads(defaultVersionedObject).                 Writes(producedObject)             if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {                 return nil, err             }             addParams(route, action.Params)             // 6、添加到路由中                 routes = append(routes, route)         case "DELETE": // Delete a resource.         ...         default:             return nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)         }         for _, route := range routes {             route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{                 Group:   reqScope.Kind.Group,                 Version: reqScope.Kind.Version,                 Kind:    reqScope.Kind.Kind,             })             route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb))             ws.Route(route)         }         // Note: update GetAuthorizerAttributes() when adding a custom handler .