k8s Api server 源码解读

最新推荐文章于 2025-05-21 23:32:05 发布

酷乐喵

最新推荐文章于 2025-05-21 23:32:05 发布

阅读量2.1k

点赞数 2

CC 4.0 BY-SA版权

分类专栏： k8s 文章标签： 1.10 k8s api-server

本文链接：https://blog.youkuaiyun.com/qq_32381527/article/details/86112091

本文深入解析了K8s API Server v1.10的源码，从如何初始化Go-restful框架到Resource Quota的创建流程。通过分析关键方法如ExecuteC、NewAPIServerCommand、CreateServerChain等，揭示了API Server启动过程。在理解Go-restful的Route、Webservice和Container概念后，文章详细跟踪了Resource Quota的创建、存储在etcd中的实现细节，涉及Creater接口、RESTStorage的New和Create方法，以及cache的使用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

背景

由于在工作中，需要引入k8s Resource Quota,所以这次借此契机研究Api Sever源码和resource quota的创建流程。

K8s源码版本: v 1.10

Golang版本: 1.10.5

etcd： 3.0+

ide： idea

对于1.10及以上的k8s要求golang版本必须在1.10及以上。

利用git下载K8s源码本地编译启动api-server 需要执行下面命令

cd ./go/src/k8s.io/kubernetes

make WHAT=cmd/kube-apiserver all

先启动etcd，然后就可以在本地直接启动api-server, 进入debug模式调试。如果遇见编译nil不兼容的问题，请将nil改为0值即可。

类比Java的spring体系，k8s实现restful架构是通过Go-restful第三方模块实现的。

Go-restful主要名词解释：

Route: 每个请求对应一个route，通过route指定具体的逻辑处理函数。

Webservice：多个route组成一个webservice。

Container: 多个webservice组成一个container。

直接上go-restful 官方demo, 比较直观:

package main
import (
   "github.com/emicklei/go-restful"
   "log"
   "net/http"
)


// This example shows the minimal code needed to get a restful.WebService working.

// GET http://localhost:8080/hello

//  Go restful github site: https://github.com/emicklei/go-restful

func main() {

   ws := new(restful.WebService)

   ws.Route(ws.GET("/hello").To(hello))

   ws.Route(ws.GET("/hello/world").To(world))

   restful.Add(ws)

   http.ListenAndServe(":8080", nil)

   log.Printf("start")

}

func hello(req *restful.Request, resp *restful.Response) {
   //resp.w
   resp.Write([]byte("hello world"))
}

func world(req *restful.Request, resp *restful.Response) {
   //resp.w
   resp.Write([]byte("this is go world."))

}
-----------------------------------------------------
package main

import (
"github.com/emicklei/go-restful"
"io"
"log"
"net/http"
)

func main() {
   ws := new(restful.WebService)
   ws.Route(ws.GET("/hello").To(hello))
   restful.Add(ws)
   go func() {
      log.Fatal(http.ListenAndServe(":8080", nil))
   }()

   container2 := restful.NewContainer()
   ws2 := new(restful.WebService)
   ws2.Route(ws2.GET("/hello").To(hello2))
   container2.Add(ws2)
   server := &http.Server{Addr: ":8081", Handler: container2}
   log.Fatal(server.ListenAndServe())
}

func hello(req *restful.Request, resp *restful.Response) {
   io.WriteString(resp, "default world")
}

func hello2(req *restful.Request, resp *restful.Response) {
   io.WriteString(resp, "second world")
}

所以现在的目标：

1，找到api server如和初始化go-restful框架？

2，找到对应的handler看k8s如和创建一个resoure quota？

3，找到k8s如何将create的resource quota对象存储在etcd中？

现在我们来解决问题1.

由k8s的 apiserver启动cmd/kube-apiserver/app中main函数：

package main

import (
   goflag "flag"
   "fmt"
   "math/rand"
   "os"
   "time"
   "github.com/spf13/pflag"
   utilflag "k8s.io/apiserver/pkg/util/flag"
   "k8s.io/apiserver/pkg/util/logs"
   "k8s.io/kubernetes/cmd/kube-apiserver/app"
   _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
   _ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
)

func main() {
   rand.Seed(time.Now().UTC().UnixNano())
   // new新的command。
   command := app.NewAPIServerCommand()
   // TODO: once we switch everything over to Cobra commands, we can go back to calling

   // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
   // normalize func and add the go flag set by hand.
   //pflog主要是命令行参数标准化 可忽略。
   pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
   pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
   // utilflag.InitFlags()
   logs.InitLogs()
   defer logs.FlushLogs()
   // 这里是入口
   if err := command.Execute(); err != nil {
      fmt.Fprintf(os.Stderr, "error: %v\n", err)
      os.Exit(1)
   }
}

command.Execute()调用的是ExcuteC()方法:

func (c *Command) ExecuteC() (cmd *Command, err error) {
   // Regardless of what command execute is called on, run on Root only
   if c.HasParent() {
      return c.Root().ExecuteC()
   }
   // windows hook
   if preExecHookFn != nil {
      preExecHookFn(c)
   }
   // initialize help as the last point possible to allow for user
   // overriding
   c.InitDefaultHelpCmd()
   var args []string
   // Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155
   // 解析命令行参数
   if c.args == nil && filepath.Base(os.Args[0]) != "cobra.test" {
      args = os.Args[1:]
   } else {
      args = c.args
   }
   var flags []string
   if c.TraverseChildren {
      cmd, flags, err = c.Traverse(args)
   } else {
      cmd, flags, err = c.Find(args)
   }
   if err != nil {
      // If found parse to a subcommand and then failed, talk about the subcommand
      if cmd != nil {
         c = cmd
      }
      if !c.SilenceErrors {
         c.Println("Error:", err.Error())
         c.Printf("Run '%v --help' for usage.\n", c.CommandPath())
      }
      return c, err
   }
   cmd.commandCalledAs.called = true
   if cmd.commandCalledAs.name == "" {
      cmd.commandCalledAs.name = cmd.Name()
   }
   // 这里才是真正的入口函数调用开始new command中的run方法入口，前面只是一些参数封装。
   err = cmd.execute(flags)
   ………
  cmd.execute(flags)方法：
   if c == nil {
      return fmt.Errorf("Called Execute() on a nil Command")
   }

   if len(c.Deprecated) > 0 {
      c.Printf("Command %q is deprecated, %s\n", c.Name(), c.Deprecated)
   }
   // initialize help and version flag at the last point possible to allow for user
   // overriding
   c.InitDefaultHelpFlag()
   c.InitDefaultVersionFlag()
   err = c.ParseFlags(a)
   if err != nil {
      return c.FlagErrorFunc()(c, err)
   }

   // If help is called, regardless of other flags, return we want help.
   // Also say we need help if the command isn't runnable.
   helpVal, err := c.Flags().GetBool("help")
   if err != nil {
      // should be impossible to get here as we always declare a help
      // flag in InitDefaultHelpFlag()
      c.Println("\"help\" flag declared as non-bool. Please correct your code")
      return err
   }

   if helpVal {
      return flag.ErrHelp
   }

   // for back-compat, only add version flag behavior if version is defined
   if c.Version != "" {
      versionVal, err := c.Flags().GetBool("version")
      if err != nil {
         c.Println("\"version\" flag declared as non-bool. Please correct your code")
         return err
      }
      if versionVal {
         err := tmpl(c.OutOrStdout(), c.VersionTemplate(), c)
         if err != nil {
           c.Println(err)
         }
         return err
      }
   }

   if !c.Runnable() {
      return flag.ErrHelp
   }
   c.preRun()
   argWoFlags := c.Flags().Args()
   if c.DisableFlagParsing {
      argWoFlags = a
   }

   if err := c.ValidateArgs(argWoFlags); err != nil {
      return err
   }

   for p := c; p != nil; p = p.Parent() {
      if p.PersistentPreRunE != nil {
         if err := p.PersistentPreRunE(c, argWoFlags); err != nil {
            return err
         }
         break
      } else if p.PersistentPreRun != nil {
         p.PersistentPreRun(c, argWoFlags)
         break
      }
   }
   if c.PreRunE != nil {
      if err := c.PreRunE(c, argWoFlags); err != nil {
         return err
      }

   } else if c.PreRun != nil {
     c.PreRun(c, argWoFlags)
   }
   if err := c.validateRequiredFlags(); err != nil {
      return err
   }
   if c.RunE != nil {
      if err := c.RunE(c, argWoFlags); err != nil {
         return err
      }
   } else {
      // 在此之前，主要就是参数的解析，版本验证等预处理过程，这里才是其new command时候传入的Run方法。
     c.Run(c, argWoFlags)
   }

…………

回到app. NewAPIServerCommand()方法:

func NewAPIServerCommand() *cobra.Command {

   // 设置默认值：注意，我们在命令行中传入的参数，在k8s中统一称之为option，经过sdk flag包处理后新建一个config对象并将option赋值给config中的属性字段
   /*
    *  MaxRequestsInFlight:         defaults.MaxRequestsInFlight, 400
    * MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,  200
    * RequestTimeout:              defaults.RequestTimeout,  60s
    * MinRequestTimeout:           defaults.MinRequestTimeout, 1800
    */

   s := options.NewServerRunOptions()
   cmd := &cobra.Command{
      Use: "kube-apiserver",
      Long: `The Kubernetes API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
      Run: func(cmd *cobra.Command, args []string) {
         verflag.PrintAndExitIfRequested()
         utilflag.PrintFlags(cmd.Flags())
         stopCh := server.SetupSignalHandler()
         // 核心入口方法
         if err := Run(s, stopCh); err != nil {
            fmt.Fprintf(os.Stderr, "%v\n", err)
            os.Exit(1)
         }
      },
   }

   // 读取命令行参数配置信息例如 admission-control,如果要开启resourcequota必须配置此参数.
   s.AddFlags(cmd.Flags())
   return cmd
}

进入上面标红的Run方法:

func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
   // To help debugging, immediately log version
   glog.Infof("Version: %+v", version.Get())
   server, err := CreateServerChain(runOptions, stopCh)
   if err != nil {
      return err
   }
   return server.PrepareRun().Run(stopCh)
}

进入CreateServerChain()方法:

// 这里才是apiserver自己的核心入口方法，之前的只是一系列封装和初始化。
func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) {
   // 创建代理隧道，这个没搞懂，猜测应该是master 和各个node通信会用到。 
   nodeTunneler, proxyTransport, err := CreateNodeDialer(runOptions)
   if err != nil {
      return nil, err
   }
   // 初始化api-server配置 例如集群默认的ip段配置等，初始化admission插件。
   // 第一个返回值kubeAPIServerConfig: 前面解释过命令行参数统称为option，但是真正使用的时候是config。
   // 第二个返回值sharedInformers:这里暂时不做介绍。只需要知道这个是一个缓存事件异步处理框架就可以。如kube-scheduler、kubelet等跟apiserver都是通过缓存间接通信的，SharedInformer一方面收集客户端和其它组件的请求，一方面负责通知该事件的关注者，还有它会把缓存中的数据同步到ETCD中后面在kube-controller-manager模块会详细介绍。

   // 第三个返回值versionedInformers: 同样是一个缓存器，但是以版本为区分，因为在k8s支持的api版本目前为/api/v1,因此这个暂时是硬代码，后面会看见。
   // insecureServingOptions：server的配置信息, 可忽略
   // serviceResolver：可忽略
   // pluginInitializer：这个比较重要，很多功能都是通过插件的方式提供，我们在启动api-server时候，通过--admission-       //control参数指定需要加载的插件，例如: NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,Priority.
   kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
   if err != nil {
      return nil, err
   }
   // TPRs are enabled and not yet beta, since this these are the successor, they fall under the same enablement rule
   // If additional API servers are added, they should be gated.
   // 加载自定义的一些api。可忽略
   apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, pluginInitializer, runOptions)
   if err != nil {
      return nil, err
   }
    // 加载自定义的一些api。可忽略
   apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.EmptyDelegate)
   if err != nil {
      return nil, err
   }
   // 高潮来了，通过名字也可以看出，apiserver的创建方法。
   kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, versionedInformers)
   if err != nil {
      return nil, err
   }
   // if we're starting up a hacked up version of this API server for a weird test case,
   // just start the API server as is because clients don't get built correctly when you do this
   if len(os.Getenv("KUBE_API_VERSIONS")) > 0 {
      if insecureServingOptions != nil {
         insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(kubeAPIServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
         if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, kubeAPIServerConfig.GenericConfig.RequestTimeout, stopCh); err != nil {
            return nil, err
         }
      }
      return kubeAPIServer.GenericAPIServer, nil
   }
   // otherwise go down the normal path of standing the aggregator up in front of the API server
   // this wires up openapi
   kubeAPIServer.GenericAPIServer.PrepareRun()
   // This will wire up openapi for extension api server
   apiExtensionsServer.GenericAPIServer.PrepareRun()
   // aggregator comes last in the chain
   aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport, pluginInitializer)
   if err != nil {
      return nil, err
   }
   aggregatorConfig.ExtraConfig.ProxyTransport = proxyTransport
   aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver
   aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
   if err != nil {
      // we don't need special handl