Kubernetes生产实践系列之十九：ServiceMesh之使用Ingress Nginx暴露Kubernetes上的gRPC服务

关注

原创最新推荐文章于 2026-10-27 16:17:10 发布 1.8k 阅读 AI 写同款· GEO 优化 ›

Gateway API 与 Ingress：Kubernetes 网络的未来随着 Kubernetes 成为部署云原生应用程序的实际编排平台，网络和流量管理在管理对服务和基础设施的访问方面已成为关键挑战。Kubernetes 的核心 Ingress 资源解决了基本的第 7 层（L7）路由需求，但在灵活性、功能和标准化方面存在局限性。作为 Ingress 的后继者，Gateway API 项目引入了一套新的可移植网络管理资源，这是对传统 Ingress 功能的超越。阅读详情

一、前言

Ingress Nginx支持gRPC服务的暴露，低于0.21的版本使用如下的annotation支持gRPC：

nginx.ingress.kubernetes.io/grpc-backend: "true"

0.21版本以上使用如下annotation支持gRPC：

nginx.ingress.kubernetes.io/backend-protocol: "GRPC"

本文基于Ingress Ngixn版本0.30.0(quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0)进行部署和测试

转载自https://blog.youkuaiyun.com/cloudvtech

二、部署gRPC服务

2.1 服务端代码和镜像构建

server.go

package main

import (
	"context"
	"fmt"
	"os/signal"

	"log"
	"net"
	"os"

	"github.com/javiramos1/grpcapi"
	"google.golang.org/grpc"
	"google.golang.org/grpc/reflection"
)

type grpcServer struct{}

func (*grpcServer) GrpcService(ctx context.Context, req *grpcapi.GrpcRequest) (*grpcapi.GrpcResponse, error) {
	fmt.Printf("grpcServer %v\n", req)
	name, _ := os.Hostname()

	input := req.GetInput()
	result := "Got input " + input + " server host: " + name
	res := &grpcapi.GrpcResponse{
		Response: result,
	}
	return res, nil
}

func main() {
	fmt.Println("Starting Server...")

	log.SetFlags(log.LstdFlags | log.Lshortfile)

	hostname := os.Getenv("SVC_HOST_NAME")

	if len(hostname) <= 0 {
		hostname = "0.0.0.0"
	}

	port := os.Getenv("SVC_PORT")

	if len(port) <= 0 {
		port = "50051"
	}

	lis, err := net.Listen("tcp", hostname+":"+port)
	if err != nil {
		log.Fatalf("Failed to listen: %v", err)
	}

	opts := []grpc.ServerOption{}
	s := grpc.NewServer(opts...)
	grpcapi.RegisterGrpcServiceServer(s, &grpcServer{})

	// reflection service on gRPC server.
	reflection.Register(s)

	go func() {
		fmt.Println("Server running on ", (hostname + ":" + port))
		if err := s.Serve(lis); err != nil {
			log.Fatalf("failed to serve: %v", err)
		}
	}()

	// Wait for Control C to exit
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)

	// Block until a signal is received
	<-ch
	fmt.Println("Stopping the server")
	s.Stop()
	fmt.Println("Closing the listener")
	lis.Close()
	fmt.Println("Server Shutdown")

}

Dockerfile

FROM iron/go
WORKDIR /app

ADD grpc_server /app/

CMD [ "./grpc_server" ]

构建binary和镜像：

CGO_ENABLED=0 GOOS=linux go build -o grpc_server -ldflags "-s -w -X 'main.build=$(git rev-parse --short HEAD)' -X 'main.buildDate=$(date --rfc-3339=seconds)'" -a -installsuffix cgo server.go
docker build -t myregistry.com/grpc_server .
docker push myregistry.comgrpc_server

2.2 部署gRPC服务

apiVersion: v1
kind: Service
metadata:
  name: grpcserver
spec:
  ports:
  - port: 50051
    protocol: TCP
    targetPort: 50051
  selector:
    run: grpcserver
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    run: grpcserver
  name: grpcserver
spec:
  replicas: 2
  selector:
    matchLabels:
      run: grpcserver
  template:
    metadata:
      labels:
        run: grpcserver
    spec:
      containers:
      - image: myregistry.com/grpc_server:latest
        name: grpcserver
        ports:
        - containerPort: 50051

包含两个replica和一个cluster service：

转载自https://blog.youkuaiyun.com/cloudvtech

三、通过Ingress Nginx暴露gRPC服务

参考文档：

https://kubernetes.github.io/ingress-nginx/examples/grpc/

3.1 Ingress Nginx暴露gRPC服务的TLS要求

Ingress Nginx暴露gRPC服务的时候，暂时只支持TLS(HTTPS)的方式，而不能通过普通HTTP方式，所以我们要配置TLS secret

生成key

openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout sslforingress.key -out sslforingress.pem -subj "/CN=grpc.test.com"

生成secret

kubectl create secret tls grpcserver-secret  --cert sslforingress.pem --key sslforingress.key -n grpcserver

3.2 部署Ingress

暴露的地址是 grpc.test.com:443

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
  name: grpcserver
  namespace: grpcserver
spec:
  rules:
    - host: grpc.test.com
      http:
        paths:
          - backend:
              serviceName: grpcserver
              servicePort: 50051
  tls:
# This secret must exist beforehand
# The cert must also contain the subj-name grpc.test.com
# https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates
    - secretName: grpcserver-secret
      hosts:
        - grpc.test.com

nginx.conf生成的内容：

        ## start server grpc.test.com                                   
        server {                                                                                     
                server_name grpc.test.com ;                                                            
                                                                                                                           
                listen 80  ;                                                                                               
                listen [::]:80  ;                                                                                          
                listen 443  ssl http2 ;                                                                                    
                listen [::]:443  ssl http2 ;                                                                               
                                                                                                                           
                set $proxy_upstream_name "-";                                                                              
                                                                                                                           
                ssl_certificate_by_lua_block {                                                                             
                        certificate.call()                                                                                 
                }

                location / {                                                  
                                                                                                     
                        set $namespace      "grpcserver";             
                        set $ingress_name   "grpcserver";                                      
                        set $service_name   "grpcserver";                   
                        set $service_port   "50051";                                                 
                        set $location_path  "/";                                                

        ... ...
                        # Allow websocket connections                                                
                        grpc_set_header                        Upgrade           $http_upgrade;      
                                                                                                     
                        grpc_set_header                        Connection        $connection_upgrade;
                                                                                   
                        grpc_set_header X-Request-ID           $req_id;            
                        grpc_set_header X-Real-IP              $remote_addr;       
                                                                                   
                        grpc_set_header X-Forwarded-For        $remote_addr;       
                                                                                       
                        grpc_set_header X-Forwarded-Host       $best_http_host;        
                        grpc_set_header X-Forwarded-Port       $pass_port;             
                        grpc_set_header X-Forwarded-Proto      $pass_access_scheme;                  
                                                                                                     
                        grpc_set_header X-Scheme               $pass_access_scheme;                  
                                                                                                     
                        # Pass the original X-Forwarded-For                                          
                        grpc_set_header X-Original-Forwarded-For $http_x_forwarded_for;              
                                                                                                     
                        # mitigate HTTPoxy Vulnerability                                             
                        # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
                        grpc_set_header Proxy                  "";    
                                                                    
                        # Custom headers to proxied server           
        
        ... ...
                
                }                                            
                                                              
        }                                                            
        ## end server grpc.test.com

转载自https://blog.youkuaiyun.com/cloudvtech

四、通过Ingress Nginx访问gRPC服务

4.1 安装grpcurl

https://github.com/fullstorydev/grpcurl

go get github.com/fullstorydev/grpcurl
go install github.com/fullstorydev/grpcurl/cmd/grpcurl

运行之后在go的PATH里面会有grpcurl的binary，将grpcurl拷贝到/usr/local/bin

4.2 通过grpcurl访问Kubernetes的gRPC服务

查看服务暴露的gRPC方法：

grpcurl -insecure grpc.test.com:443 list
greet.GrpcService
grpc.reflection.v1alpha.ServerReflection

调用服务：

[root@k8s-node-01 ~]# grpcurl  -insecure grpc.test.com:443 greet.GrpcService/grpcService
{
  "response": "Got input  server host: grpcserver-5bfd56f94b-bc6fg"
}
[root@k8s-node-01 ~]# grpcurl  -insecure grpc.test.com:443 greet.GrpcService/grpcService
{
  "response": "Got input  server host: grpcserver-5bfd56f94b-w7frq"
}

可以看到服务在两个节点之间轮转

五、通过API访问Ingress Nginx暴露的gRPC服务

5.1 访问基于TLS的gRPC服务

建立连接的时候加上如下代码，可以忽略SSL证书的有效性：

config := &tls.Config{

InsecureSkipVerify: true,

}

cc, err := grpc.Dial(hostname+":"+port, grpc.WithTransportCredentials(credentials.NewTLS(config)))

5.2 通过短连接访问gRPC服务

client_short_connection.go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"github.com/javiramos1/grpcapi"
	"google.golang.org/grpc"
        "google.golang.org/grpc/credentials"
        "crypto/tls"
)

func main() {

	fmt.Println("Starting client...")

	hostname := os.Getenv("SVC_HOST_NAME")

	if len(hostname) <= 0 {
		hostname = "0.0.0.0"
	}

	port := os.Getenv("SVC_PORT")

	if len(port) <= 0 {
		port = "50051"
	}

	config := &tls.Config{
    		InsecureSkipVerify: true,
	}
	cc, err := grpc.Dial(hostname+":"+port, grpc.WithTransportCredentials(credentials.NewTLS(config)))


	if err != nil {
		log.Fatalf("could not connect: %v", err)
	}
	defer cc.Close()

	c := grpcapi.NewGrpcServiceClient(cc)
	fmt.Printf("Created client: %f", c)
		callService(c)

}

func callService(c grpcapi.GrpcServiceClient) {
	fmt.Println("callService...")
	req := &grpcapi.GrpcRequest{
		Input: "test",
	}
	res, err := c.GrpcService(context.Background(), req)
	if err != nil {
		log.Fatalf("error while calling gRPC: %v", err)
	}
	log.Printf("Response from Service: %v", res.Response)
}

编译：

CGO_ENABLED=0 GOOS=linux go build -o grpc_client_short_connection -ldflags "-s -w -X 'main.build=$(git rev-parse --short HEAD)' -X 'main.buildDate=$(date --rfc-3339=seconds)'" -a -installsuffix cgo client.go

运行：

export SVC_HOST_NAME=grpc.test.com
export SVC_PORT=443

while true; do ./grpc_client_short_connection; sleep 1;done

2020/05/12 13:21:39 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:40 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
2020/05/12 13:21:41 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
2020/05/12 13:21:42 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:43 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:44 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
2020/05/12 13:21:45 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:46 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:47 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
2020/05/12 13:21:48 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
2020/05/12 13:21:49 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
2020/05/12 13:21:50 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg

同时可以看到系统存在大量端口被短连接占用之后等待关闭：

netstat -anp | grep 443 -w
tcp        0      0 172.3.0.11:41984        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41948        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41910        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41974        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41934        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41970        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41928        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41912        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41940        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41942        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41976        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41994        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41980        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41988        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41998        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41916        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41978        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41924        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41922        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41996        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41986        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41962        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41906        172.2.2.11:443          TIME_WAIT   -                   
tcp        0      0 172.3.0.11:41960        172.2.2.11:443          TIME_WAIT   -

5.3 通过长连接访问gRPC服务

client_longlive_connection.go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
        "time"
	"github.com/javiramos1/grpcapi"
	"google.golang.org/grpc"
        "google.golang.org/grpc/credentials"
        "crypto/tls"
)

func main() {

	fmt.Println("Starting client...")

	hostname := os.Getenv("SVC_HOST_NAME")

	if len(hostname) <= 0 {
		hostname = "0.0.0.0"
	}

	port := os.Getenv("SVC_PORT")

	if len(port) <= 0 {
		port = "50051"
	}

	config := &tls.Config{
    		InsecureSkipVerify: true,
	}
	cc, err := grpc.Dial(hostname+":"+port, grpc.WithTransportCredentials(credentials.NewTLS(config)))


	if err != nil {
		log.Fatalf("could not connect: %v", err)
	}
	defer cc.Close()

	c := grpcapi.NewGrpcServiceClient(cc)
	fmt.Printf("Created client: %f", c)
        for i := 1; i <= 100; i++ {
		callService(c)
      		time.Sleep(2000 * time.Millisecond)
        }

}

func callService(c grpcapi.GrpcServiceClient) {
	fmt.Println("callService...")
	req := &grpcapi.GrpcRequest{
		Input: "test",
	}
	res, err := c.GrpcService(context.Background(), req)
	if err != nil {
		log.Fatalf("error while calling gRPC: %v", err)
	}
	log.Printf("Response from Service: %v", res.Response)
}

编译并运行：

CGO_ENABLED=0 GOOS=linux go build -o grpc_client_longlive_connection -ldflags "-s -w -X 'main.build=$(git rev-parse --short HEAD)' -X 'main.buildDate=$(date --rfc-3339=seconds)'" -a -installsuffix cgo client.go

export SVC_HOST_NAME=grpc.test.com
export SVC_PORT=443

./grpc_client_longlive 
2020/05/12 13:40:06 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
callService...
2020/05/12 13:40:08 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
callService...
2020/05/12 13:40:10 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
callService...
2020/05/12 13:40:12 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
callService...
2020/05/12 13:40:14 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
callService...
2020/05/12 13:40:16 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
callService...
2020/05/12 13:40:18 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
callService...
2020/05/12 13:40:20 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq
callService...
2020/05/12 13:40:22 Response from Service: Got input test server host: grpcserver-5bfd56f94b-bc6fg
callService...
2020/05/12 13:40:24 Response from Service: Got input test server host: grpcserver-5bfd56f94b-w7frq

整个运行过程中只有一个到443端口的长连接：