实现一个极简 K8s apiserver

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# APIService for CRD               ---> # APIService from scratch
apiVersion: apiregistration.k8s.io/v1 | apiVersion: apiregistration.k8s.io/v1
kind: APIService                      | kind: APIService
metadata:                             | metadata:
  name: v1.hello.zeng.dev             |   name: v1.hello.zeng.dev
spec:                                 | spec:
  group: hello.zeng.dev               |   group: hello.zeng.dev
  groupPriorityMinimum: 1000          |   groupPriorityMinimum: 100 
  version: v1                         |   version: v1
  versionPriority: 100                |   versionPriority: 10
                                      | + service:
                                      | +   name: apiserver
                                      | +   namespace: hello
                                      | + insecureSkipTLSVerify: true

1
2
3
4

Req /apis/hello.zeng.dev/v1/** ---> kube-apiserver 👉👉👉 hello.zeng.dev-apiserver ✅
                                        ❌
                                        ⬇️
                                    apiextensions-apiserver    

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

var apis = metav1.APIGroupList{
	TypeMeta: metav1.TypeMeta{
		Kind:       "APIGroupList",
		APIVersion: "v1",
	},
	Groups: []metav1.APIGroup{
		{
			TypeMeta: metav1.TypeMeta{
				Kind:       "APIGroup",
				APIVersion: "v1",
			},
			Name: "hello.zeng.dev",
			Versions: []metav1.GroupVersionForDiscovery{
				{
					GroupVersion: "hello.zeng.dev/v1",
					Version:      "v1",
				},
			},
			PreferredVersion: metav1.GroupVersionForDiscovery{GroupVersion: "hello.zeng.dev/v1", Version: "v1"},
		},
	},
}

var apidiscoveries = ...

//@Router /apis [get]
func APIs(w http.ResponseWriter, r *http.Request) {
	var gvk [3]string
	for _, acceptPart := range strings.Split(r.Header.Get("Accept"), ";") {
		if g_v_k := strings.Split(acceptPart, "="); len(g_v_k) == 2 {
			switch g_v_k[0] {
			case "g":
				gvk[0] = g_v_k[1]
			case "v":
				gvk[1] = g_v_k[1]
			case "as":
				gvk[2] = g_v_k[1]
			}
		}
	}

	if gvk[0] == "apidiscovery.k8s.io" && gvk[2] == "APIGroupDiscoveryList" {
		w.Header().Set("Content-Type", "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList")
		w.Write([]byte(apidiscoveries))
	} else {
		w.Header().Set("Content-Type", "application/json")
		renderJSON(w, apis)
	}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

const hellov1Resources = `{
	"kind": "APIResourceList",
	"apiVersion": "v1",
	"groupVersion": "hello.zeng.dev/v1",
	"resources": [
	  {
		"name": "foos",
		"singularName": "foo",
		"namespaced": true,
		"kind": "Foo",
		"verbs": [
		  "create",
		  "delete",
		  "get",
		  "list",
		  "update",
		  "patch"
		],
		"shortNames": [
		  "fo"
		],
		"categories": [
		  "all"
		]
	  }
	]}`

// @Router  /apis/hello.zeng.dev/v1 [get]
func APIGroupHelloV1Resources(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(hellov1Resources))
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

package main

import (
  "net/http"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type Foo struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec struct {
		// Msg says hello world!
		Msg string `json:"msg"`
		// Msg1 provides verbose information
		Msg1 string `json:"msg1"`
	} `json:"spec"`
}

// PostFoo swag doc
// @Summary      Create a Foo Object
// @Description  Create a Foo Object
// @Tags         foos
// @Consume      json
// @Produce      json
// @Param        namespace	path	string  true  "Namepsace"
// @Success      201  {object}  Foo
// @Router       /apis/hello.zeng.dev/v1/namespaces/{namespace}/foos [post]
func PostFoo(w http.ResponseWriter, r *http.Request) {
  ...
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

swagger: '2.0'
paths:
  "/apis/hello.zeng.dev/v1/namespaces/{namespace}/foos":
    post:
      description: Create a Foo Object
      produces:
      - application/json
      tags:
      - foos
      summary: Create a Foo Object
      parameters:
      - type: string
        description: Namepsace
        name: namespace
        in: path
        required: true
      responses:
        '201':
          description: Created
          schema:
            "$ref": "#/definitions/main.Foo"
definitions:
  main.Foo:
    type: object
    properties:
      apiVersion:
        description: "..."
        type: string
      kind:
        description: "..."
        type: string
      metadata:
        "$ref": "#/definitions/v1.ObjectMeta"
      spec:
        type: object
        properties:
          msg:
            description: Msg says hello world!
            type: string
          msg1:
            description: Msg1 provides some verbose information
            type: string
  v1.ObjectMeta: {...}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

import(
	"embed"
	"net/http"
	"strings"

	gnosticopenapiv2 "github.com/google/gnostic/openapiv2"
	"google.golang.org/protobuf/proto"
)

//go:embed docs/*
var embedFS embed.FS

//@Router	/openapi/v2	[get]
func OpenapiV2(w http.ResponseWriter, r *http.Request) {
	jsonbytes, _ := embedFS.ReadFile("docs/swagger.json")

	// 😭 kubectl (v1.26.2, v1.27.1 ...) api discovery module (which fetch /openapi/v2, /openapi/v3)
	//    only accept application/com.github.proto-openapi.spec.v2@v1.0+protobuf
	if !strings.Contains(r.Header.Get("Accept"), "application/json") && strings.Contains(r.Header.Get("Accept"), "protobuf") {
		w.Header().Set("Content-Type", "application/com.github.proto-openapi.spec.v2.v1.0+protobuf")
		if pbbytes, err := ToProtoBinary(jsonbytes); err != nil {
			w.Header().Set("Content-Type", "application/json")
			writeErrStatus(w, "", http.StatusInternalServerError, err.Error())
			return
		} else {
			w.Write(pbbytes)
			return
		}
	}

	// 😄 kube apiserver aggregation module accept application/json
	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonbytes)
}

func ToProtoBinary(json []byte) ([]byte, error) {
	document, err := gnosticopenapiv2.ParseDocument(json)
	if err != nil {
		return nil, err
	}
	return proto.Marshal(document)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

var x sync.RWMutex
var foos = map[string]Foo{}

func init() {
	foos["default/bar"] = Foo{
		TypeMeta:   metav1.TypeMeta{APIVersion: "hello.zeng.dev/v1", Kind: "Foo"},
		ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "bar", CreationTimestamp: metav1.Now()},
		Spec: struct {
			Msg  string "json:\"msg\""
			Msg1 string "json:\"msg1\""
		}{
			Msg:  "hello world",
			Msg1: "apiserver-from-scratch says '👋 hello world 👋'",
		},
	}
}

1
2
3
4
5
6
7

apiVersion: hello.zeng.dev/v1                               | apiVersion: hello.zeng.dev/v1
kind: Foo                                                   | kind: Foo
metadata:                                                   | metadata:                                          
  name: bar                                                 |   name: bar
spec:                                                       | spec:
  msg: hello world                                          |   msg: hi there
  msg1: "apiserver-from-scratch says '👋 hello world 👋'"    |   msg1: ''

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

// 省略部分依赖引入和错误处理...
import(
	jsonpatch "github.com/evanphx/json-patch"
	kruntime "k8s.io/apimachinery/pkg/runtime"
	kstrategicpatch "k8s.io/apimachinery/pkg/util/strategicpatch"
)

func PatchFoo(w http.ResponseWriter, r *http.Request, name string) {
	patchBytes, _ := io.ReadAll(r.Body)
	old := foos[nsname]
	var originalBytes, _ = json.Marshal(old)
	var patchedFoo []byte

	switch r.Header.Get("Content-Type") {
	case "application/merge-patch+json":
		patchedFoo, _ = jsonpatch.MergePatch(originalBytes, patchBytes)
	case "application/json-patch+json":
		patch, _ := jsonpatch.DecodePatch(patchBytes)
		patchedFoo, _ = patch.Apply(originalBytes)
	case "application/strategic-merge-patch+json":
		var patchMap map[string]interface{}
		_ = json.Unmarshal(patchBytes, &patchMap)
		var patchedObjMap, _ = kstrategicpatch.StrategicMergeMapPatch(originalObjMap, patchMap, schema)
		var theFoo Foo
		_ = kruntime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(patchedObjMap, &theFoo, false)
		patchedFoo, _ = json.Marshal(theFoo)	
	default:
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	}

	dec := json.NewDecoder(bytes.NewReader(patchedFoo))
	dec.DisallowUnknownFields()
	var f Foo
	dec.Decode(&f)
	foos[nsname] = f
	renderJSON(w, f) // serialize new Foo as JSON, fill response body
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

import(
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var fooCol = []metav1.TableColumnDefinition{
	{Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]},
	{Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]},
	{Name: "Message", Type: "string", Format: "message", Description: "foo message"},
	{Name: "Message1", Type: "string", Format: "message1", Description: "foo message plus", Priority: 1}, // kubectl -o wide
}

func tryConvert2Table(obj interface{}, acceptedContentType string) interface{} {
	if strings.Contains(acceptedContentType, "application/json") && strings.Contains(acceptedContentType, "as=Table") {
		switch typedObj := obj.(type) {
		case Foo:
			return metav1.Table{
				TypeMeta: metav1.TypeMeta{
					Kind:       "Table",
					APIVersion: "meta.k8s.io/v1",
				},
				ColumnDefinitions: fooCol,
				Rows:              foo2TableRow(&typedObj),
			}
		case FooList:
			return metav1.Table{
				TypeMeta: metav1.TypeMeta{
					Kind:       "Table",
					APIVersion: "meta.k8s.io/v1",
				},
				ColumnDefinitions: fooCol,
				Rows:              fooList2TableRows(&typedObj),
			}
		default:
			return obj
		}
	}
	return obj
}