统一管理服务集群gateway

reverse_proxy.go


package apiserver

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"k8s.io/klog/v2"

	//"k8s.io/apiserver/pkg/endpoints/filters"
	//"k8s.io/apiserver/pkg/server"
	"io/ioutil"
	"k8s.io/client-go/util/connrotation"
	"net"
	"net/http"
	"net/http/httputil"
	"time"
)

// ReverseProxy represents a real pair of http request and response
type ReverseProxy struct {
	backendProxy *httputil.ReverseProxy

	backendUrl  string
	backendPort int

	transport *Transport
}

type Transport struct {
	d         *connrotation.Dialer
	Transport *http.Transport
}

func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.Transport.RoundTrip(req)
}

func makeTransport(tlsClientConfig *tls.Config) *Transport {
	d := connrotation.NewDialer(
		(&net.Dialer{
			Timeout:   time.Duration(10) * time.Second,
			KeepAlive: 60 * time.Second,
		}).DialContext)

	return &Transport{
		d: d,
		// TODO enable http2 if using go1.15
		// the params are same with http.DefaultTransport
		Transport: &http.Transport{
			DialContext:           d.DialContext,
			Proxy:                 http.ProxyFromEnvironment,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig:       tlsClientConfig,
		},
	}
}

func getRootCertPool(caFile string) (*x509.CertPool, error) {
	caCrt, err := ioutil.ReadFile(caFile)
	if err != nil {
		klog.Errorf("read ca file %s err: %v", caFile, err)
		return nil, err
	}

	pool := x509.NewCertPool()
	ok := pool.AppendCertsFromPEM(caCrt)
	if !ok {
		klog.Errorf("append ca certs %s error.", caFile)
		return nil, fmt.Errorf("append ca certs %s error.\n", caFile)
	}

	return pool, nil
}

func NewReverseProxy(transport *Transport, backendUrl string, backendPort int) *ReverseProxy {
	p := &ReverseProxy{
		backendPort: backendPort,
		backendUrl:  backendUrl,
		transport:   transport,
	}

	reverseProxy := &httputil.ReverseProxy{
		Director:       p.makeDirector,
		Transport:      p.transport,
		ModifyResponse: p.modifyResponse,
		ErrorHandler:   p.handlerError,
	}

	reverseProxy.FlushInterval = -1

	p.backendProxy = reverseProxy
	return p
}

func (p *ReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	klog.Infof("New request: method->%s, url->%s", r.Method, r.URL.String())

	// handle http
	p.backendProxy.ServeHTTP(w, r)
}

func (p *ReverseProxy) makeDirector(req *http.Request) {
	req.URL.Scheme = "https"
	req.URL.Host = fmt.Sprintf("%s:%d", p.backendUrl, p.backendPort)
}

func (p *ReverseProxy) modifyResponse(resp *http.Response) error {
	if resp == nil || resp.Request == nil {
		klog.Infof("no response or request, skip cache response")
		return nil
	}

	return nil
}

func (p *ReverseProxy) handlerError(rw http.ResponseWriter, req *http.Request, err error) {
	klog.V(2).Infof("Request url=%s, error=%v", req.URL, err)

	klog.V(6).Infof("Ignore request %s", req.URL)
	rw.WriteHeader(http.StatusServiceUnavailable)
	_, err = rw.Write([]byte(err.Error()))
	if err != nil {
		klog.Errorf("Write error response err: %v", err)
	}
	return
}

func loadCert(cert string, key string) (*tls.Certificate, string, error) {
	tlsCert, err := tls.LoadX509KeyPair(cert, key)
	if err != nil {
		klog.Errorf("load cert and key error: %v", err)
		return nil, "", err
	}

	var leaf *x509.Certificate
	if tlsCert.Leaf == nil {
		l, err := x509.ParseCertificate(tlsCert.Certificate[0])
		if err != nil {
			klog.Errorf("parse cert %s,%s error: %v", cert, key, err)
			return nil, "", err
		}
		leaf = l
		tlsCert.Leaf = l
	} else {
		leaf = tlsCert.Leaf
	}
	commonName := leaf.Subject.CommonName

	if len(commonName) == 0 {
		klog.Errorf("cert common name nil")
		return nil, "", fmt.Errorf("cert common name nil")
	}

	return &tlsCert, commonName, nil
}

//
//func  buildHandlerChain(handler http.Handler) http.Handler {
//	cfg := &server.Config{
//		LegacyAPIGroupPrefixes: sets.NewString(server.DefaultLegacyAPIPrefix),
//	}
//	resolver := server.NewRequestInfoResolver(cfg)
//	handler = filters.WithRequestInfo(handler, resolver)
//	return handler
//}






proxy_handler.go


package apiserver

import (
	"crypto/tls"
	"k8s.io/klog/v2"
	"net/http"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/endpoints/filters"
	"k8s.io/apiserver/pkg/server"
)

// ProxyHandler is the real handler for each request
type ProxyHandler struct {

	// apiserverInfo is used to proxy to.
	apiserverUrl  string
	apiserverPort int
	reverseProxy  *ReverseProxy
}

// apiServerCA 用户集群对应的apiserver的ca证书
// apiServerCert apiserver对应得ca证书
func NewProxyHandler(apiServerCA, apiServerCert, apiServerKey, apiserverUrl string, apiserverPort int) (http.Handler, error) {
	h := &ProxyHandler{
		apiserverPort: apiserverPort,
		apiserverUrl:  apiserverUrl,
	}

	// 获取证书池
	apiServerCertPool, err := getRootCertPool(apiServerCA)
	if err != nil {
		return nil, err
	}
	tlsCert, commonName, err := loadCert(apiServerCert, apiServerKey)
	klog.Infof("--->load cert %s", commonName)

	// 在这里重现生成发给用户集群apiserver的签名请求
	f := func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
		klog.Infof("cert for %s is %+v", commonName, tlsCert.Leaf)
		return tlsCert, nil
	}
	tlsConfig := &tls.Config{
		MinVersion:           tls.VersionTLS12,
		RootCAs:              apiServerCertPool,
		GetClientCertificate: f,
	}
	transPort := makeTransport(tlsConfig)
	h.reverseProxy = NewReverseProxy(transPort, h.apiserverUrl, h.apiserverPort)
	return h.buildHandlerChain(h), nil
}

func (h *ProxyHandler) buildHandlerChain(handler http.Handler) http.Handler {
	cfg := &server.Config{
		LegacyAPIGroupPrefixes: sets.NewString(server.DefaultLegacyAPIPrefix),
	}
	resolver := server.NewRequestInfoResolver(cfg)
	handler = filters.WithRequestInfo(handler, resolver)
	return handler
}

func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	klog.Infof("request tls:%v", r.TLS)
	if r.TLS != nil {
		for _, cert := range r.TLS.PeerCertificates {
			klog.Infof("request cert:%v", cert)
		}
	}
	h.reverseProxy.ServeHTTP(w, r)
}





handle_test.go

package apiserver

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"k8s.io/klog/v2"
	"net/http"
	"testing"
)

func TestTLS(t *testing.T) {
	mux := http.NewServeMux()

	// ca
	ca := "./ca.pem"

	// server端公钥
	serverCrt := "./server.pem"
	// server端私钥
	serverKey := "./server-key.pem"

	// 用户集群apiserver的签名
	apiServerCA := "./apiserver-ca.pem"
	// 签名
	apiServerCert := "./jane.crt"
	// 用户集群apiserver的私钥
	apiServerKey := "./jane.key"
	// 用户集群apiserver地址
	apiServerUrl := "30.23.82.89"
	// 用户集群apiserver端口
	apiServerPort := 6443

	// 获取根证书池
	pool := x509.NewCertPool()

	caCrt, err := ioutil.ReadFile(ca)
	if err != nil {
		t.Fatal(err)
		klog.Errorf("read ca file %s err: %v", ca, err)
	}

	ok := pool.AppendCertsFromPEM(caCrt)
	if !ok {
		klog.Errorf("append ca certs %s error.", ca)
	}

	apiSrvCa, err := ioutil.ReadFile(apiServerCA)
	if err != nil {
		t.Fatal(err)
	}
	ok = pool.AppendCertsFromPEM(apiSrvCa)
	if !ok {
		klog.Errorf("append ca certs %s error.", ca)
	}

	// 增加apiserver请求的handler
	h, err := NewProxyHandler(apiServerCA, apiServerCert, apiServerKey, apiServerUrl, apiServerPort)
	if err != nil {
		t.Fatal(err)
	}

	mux.Handle("/", h)

	srv := &http.Server{
		Addr:    ":7689",
		Handler: mux,
		TLSConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
			// 设置client ca证书池
			ClientCAs:  pool,
			ClientAuth: tls.VerifyClientCertIfGiven,
		},
	}

	// 拉起对应的服务，启用TLS进行安全请求验证
	// serverCrt apiserver的公钥
	// serverKey apiserver的私钥
	err = srv.ListenAndServeTLS(serverCrt, serverKey)
	if err != nil {
		t.Fatal(err)
	}
}