K8S部署API网关Kong

 随着全面微服务化的落地,在网关层上对运维提出的新的要求,经过了几轮测试与验证,最终选型微服务网关kong来替换nginx。本文将简要介绍如何将Kong网关部署在K8S环境中。

关于Kong的简要介绍请参考前文。

1、下载相关进行并上传harbor私服

docker pull kong:2.1
docker pull kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0
docker tag kong:2.1 harbor.59iedu.com/fjhb/kong:2.1
docker tag kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0 harbor.59iedu.com/fjhb/kong-ingress-controller:1.0
docker push harbor.59iedu.com/fjhb/kong:2.1
docker push harbor.59iedu.com/fjhb/kong-ingress-controller:1.0

2、创建pg数据库

本例中PG数据库放在k8s集群外部

create database kong_cce;

3、创建namespace

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: kong
EOF

4、创建crds

cat <<EOF | kubectl apply -f -
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: kongclusterplugins.configuration.konghq.com
spec:
  additionalPrinterColumns:
  - JSONPath: .plugin
    description: Name of the plugin
    name: Plugin-Type
    type: string
  - JSONPath: .metadata.creationTimestamp
    description: Age
    name: Age
    type: date
  - JSONPath: .disabled
    description: Indicates if the plugin is disabled
    name: Disabled
    priority: 1
    type: boolean
  - JSONPath: .config
    description: Configuration of the plugin
    name: Config
    priority: 1
    type: string
  group: configuration.konghq.com
  names:
    kind: KongClusterPlugin
    plural: kongclusterplugins
    shortNames:
    - kcp
  scope: Cluster
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        config:
          type: object
        configFrom:
          properties:
            secretKeyRef:
              properties:
                key:
                  type: string
                name:
                  type: string
                namespace:
                  type: string
              required:
              - name
              - namespace
              - key
              type: object
          type: object
        disabled:
          type: boolean
        plugin:
          type: string
        protocols:
          items:
            enum:
            - http
            - https
            - grpc
            - grpcs
            - tcp
            - tls
            type: string
          type: array
        run_on:
          enum:
          - first
          - second
          - all
          type: string
      required:
      - plugin
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: kongconsumers.configuration.konghq.com
spec:
  additionalPrinterColumns:
  - JSONPath: .username
    description: Username of a Kong Consumer
    name: Username
    type: string
  - JSONPath: .metadata.creationTimestamp
    description: Age
    name: Age
    type: date
  group: configuration.konghq.com
  names:
    kind: KongConsumer
    plural: kongconsumers
    shortNames:
    - kc
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        credentials:
          items:
            type: string
          type: array
        custom_id:
          type: string
        username:
          type: string
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: kongingresses.configuration.konghq.com
spec:
  group: configuration.konghq.com
  names:
    kind: KongIngress
    plural: kongingresses
    shortNames:
    - ki
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        proxy:
          properties:
            connect_timeout:
              minimum: 0
              type: integer
            path:
              pattern: ^/.*$
              type: string
            protocol:
              enum:
              - http
              - https
              - grpc
              - grpcs
              - tcp
              - tls
              type: string
            read_timeout:
              minimum: 0
              type: integer
            retries:
              minimum: 0
              type: integer
            write_timeout:
              minimum: 0
              type: integer
          type: object
        route:
          properties:
            headers:
              additionalProperties:
                items:
                  type: string
                type: array
              type: object
            https_redirect_status_code:
              type: integer
            methods:
              items:
                type: string
              type: array
            path_handling:
              enum:
              - v0
              - v1
              type: string
            preserve_host:
              type: boolean
            protocols:
              items:
                enum:
                - http
                - https
                - grpc
                - grpcs
                - tcp
                - tls
                type: string
              type: array
            regex_priority:
              type: integer
            strip_path:
              type: boolean
        upstream:
          properties:
            algorithm:
              enum:
              - round-robin
              - consistent-hashing
              - least-connections
              type: string
            hash_fallback:
              type: string
            hash_fallback_header:
              type: string
            hash_on:
              type: string
            hash_on_cookie:
              type: string
            hash_on_cookie_path:
              type: string
            hash_on_header:
              type: string
            healthchecks:
              properties:
                active:
                  properties:
                    concurrency:
                      minimum: 1
                      type: integer
                    healthy:
                      properties:
                        http_statuses:
                          items:
                            type: integer
                          type: array
                        interval:
                          minimum: 0
                          type: integer
                        successes:
                          minimum: 0
                          type: integer
                      type: object
                    http_path:
                      pattern: ^/.*$
                      type: string
                    timeout:
                      minimum: 0
                      type: integer
                    unhealthy:
                      properties:
                        http_failures:
                          minimum: 0
                          type: integer
                        http_statuses:
                          items:
                            type: integer
                          type: array
                        interval:
                          minimum: 0
                          type: integer
                        tcp_failures:
                          minimum: 0
                          type: integer
                        timeout:
                          minimum: 0
                          type: integer
                      type: object
                  type: object
                passive:
                  properties:
                    healthy:
                      properties:
                        http_statuses:
                          items:
                            type: integer
                          type: array
                        interval:
                          minimum: 0
                          type: integer
                        successes:
                          minimum: 0
                          type: integer
                      type: object
                    unhealthy:
                      properties:
                        http_failures:
                          minimum: 0
                          type: integer
                        http_statuses:
                          items:
                            type: integer
                          type: array
                        interval:
                          minimum: 0
                          type: integer
                        tcp_failures:
                          minimum: 0
                          type: integer
                        timeout:
                          minimum: 0
                          type: integer
                      type: object
                  type: object
                threshold:
                  type: integer
              type: object
            host_header:
              type: string
            slots:
              minimum: 10
              type: integer
          type: object
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: kongplugins.configuration.konghq.com
spec:
  additionalPrinterColumns:
  - JSONPath: .plugin
    description: Name of the plugin
    name: Plugin-Type
    type: string
  - JSONPath: .metadata.creationTimestamp
    description: Age
    name: Age
    type: date
  - JSONPath: .disabled
    description: Indicates if the plugin is disabled
    name: Disabled
    priority: 1
    type: boolean
  - JSONPath: .config
    description: Configuration of the plugin
    name: Config
    priority: 1
    type: string
  group: configuration.konghq.com
  names:
    kind: KongPlugin
    plural: kongplugins
    shortNames:
    - kp
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        config:
          type: object
        configFrom:
          properties:
            secretKeyRef:
              properties:
                key:
                  type: string
                name:
                  type: string
              required:
              - name
              - key
              type: object
          type: object
        disabled:
          type: boolean
        plugin:
          type: string
        protocols:
          items:
            enum:
            - http
            - https
            - grpc
            - grpcs
            - tcp
            - tls
            type: string
          type: array
        run_on:
          enum:
          - first
          - second
          - all
          type: string
      required:
      - plugin
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: tcpingresses.configuration.konghq.com
spec:
  additionalPrinterColumns:
  - JSONPath: .status.loadBalancer.ingress[*].ip
    description: Address of the load balancer
    name: Address
    type: string
  - JSONPath: .metadata.creationTimestamp
    description: Age
    name: Age
    type: date
  group: configuration.konghq.com
  names:
    kind: TCPIngress
    plural: tcpingresses
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          type: string
        kind:
          type: string
        metadata:
          type: object
        spec:
          properties:
            rules:
              items:
                properties:
                  backend:
                    properties:
                      serviceName:
                        type: string
                      servicePort:
                        format: int32
                        type: integer
                    type: object
                  host:
                    type: string
                  port:
                    format: int32
                    type: integer
                type: object
              type: array
            tls:
              items:
                properties:
                  hosts:
                    items:
                      type: string
                    type: array
                  secretName:
                    type: string
                type: object
              type: array
          type: object
        status:
          type: object
  version: v1beta1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
EOF

5、创建pgsql endpoint

主要用途是为了在K8S中直接使用svc来访问集群外部的pgsql

cat <<EOF | kubectl apply -f - 
apiVersion: v1
kind: Endpoints
metadata:
  name: postgres
  namespace: kong
subsets:
  - addresses:
      - ip: 192.168.1.14
    ports:
      - port: 5432
---
apiVersion: v1
kind: Service
metadata:
  name: postgres
  namespace: kong
spec:
  ports:
    - port: 5432
      protocol: TCP
      targetPort: 5432
EOF

 6、创建rbac

cat <<EOF | kubectl apply -f - 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kong-serviceaccount
  namespace: kong
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: kong-ingress-clusterrole
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - nodes
  - pods
  - secrets
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  - extensions
  - networking.internal.knative.dev
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
  - patch
- apiGroups:
  - networking.k8s.io
  - extensions
  - networking.internal.knative.dev
  resources:
  - ingresses/status
  verbs:
  - update
- apiGroups:
  - configuration.konghq.com
  resources:
  - tcpingresses/status
  verbs:
  - update
- apiGroups:
  - configuration.konghq.com
  resources:
  - kongplugins
  - kongclusterplugins
  - kongcredentials
  - kongconsumers
  - kongingresses
  - tcpingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - create
  - get
  - update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: kong-ingress-clusterrole-nisa-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kong-ingress-clusterrole
subjects:
- kind: ServiceAccount
  name: kong-serviceaccount
  namespace: kong
EOF

7、创建job

主要用来执行kong migrate up指令初始化数据库

cat <<EOF | kubectl apply -f - 
apiVersion: batch/v1
kind: Job
metadata:
  name: kong-migrations
  namespace: kong
spec:
  template:
    metadata:
      name: kong-migrations
    spec:
      containers:
      - command:
        - /bin/sh
        - -c
        - kong migrations bootstrap
        env:
        - name: KONG_DATABASE
          value: postgres
        - name: KONG_PG_HOST
          value: postgres
        - name: KONG_PG_USER
          value: postgres
        - name: KONG_PG_PASSWORD
          value: "123456"
        - name: KONG_PG_DATABASE
          value: kong_cce
        - name: KONG_PG_PORT
          value: "5432"
        image: harbor.59iedu.com/fjhb/kong:2.1
        name: kong-migrations
      initContainers:
      - command:
        - /bin/sh
        - -c
        - until nc -zv $KONG_PG_HOST $KONG_PG_PORT -w1; do echo 'waiting for db'; sleep 1; done
        env:
        - name: KONG_DATABASE
          value: postgres
        - name: KONG_PG_HOST
          value: postgres
        - name: KONG_PG_USER
          value: postgres
        - name: KONG_PG_PASSWORD
          value: "123456"
        - name: KONG_PG_DATABASE
          value: kong_cce
        - name: KONG_PG_PORT
          value: "5432"
        image: busybox
        name: wait-for-postgres
      restartPolicy: OnFailure
	  imagePullSecrets:
        - name: harborsecret
EOF

8、创建工作负载

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
    service.beta.kubernetes.io/aws-load-balancer-type: nlb
  name: kong-proxy
  namespace: kong
spec:
  ports:
  - name: proxy
    port: 80
    protocol: TCP
    targetPort: 8000
  - name: proxy-ssl
    port: 443
    protocol: TCP
    targetPort: 8443
  selector:
    app: ingress-kong
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: ingress-kong
  name: ingress-kong
  namespace: kong
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ingress-kong
  template:
    metadata:
      annotations:
        kuma.io/gateway: enabled
        prometheus.io/port: "8100"
        prometheus.io/scrape: "true"
        traffic.sidecar.istio.io/includeInboundPorts: ""
      labels:
        app: ingress-kong
    spec:
      containers:
      - env:
        - name: KONG_DATABASE
          value: postgres
        - name: KONG_PG_HOST
          value: postgres
        - name: KONG_PG_USER
          value: postgres
        - name: KONG_PG_PASSWORD
          value: "123456"
        - name: KONG_PG_DATABASE
          value: kong_cce
        - name: KONG_PG_PORT
          value: "5432"
        - name: KONG_PROXY_LISTEN
          value: 0.0.0.0:8000, 0.0.0.0:8443 ssl http2
        - name: KONG_PORT_MAPS
          value: 80:8000, 443:8443
        - name: KONG_ADMIN_LISTEN
          value: 0.0.0.0:8444 
        - name: KONG_STATUS_LISTEN
          value: 0.0.0.0:8100
        - name: KONG_NGINX_WORKER_PROCESSES
          value: "2"
        - name: KONG_ADMIN_ACCESS_LOG
          value: /dev/stdout
        - name: KONG_ADMIN_ERROR_LOG
          value: /dev/stderr
        - name: KONG_PROXY_ERROR_LOG
          value: /dev/stderr
        image: harbor.59iedu.com/fjhb/kong:2.1
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - -c
              - kong quit
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /status
            port: 8100
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        name: proxy
        ports:
        - containerPort: 8000
          name: proxy
          protocol: TCP
        - containerPort: 8443
          name: proxy-ssl
          protocol: TCP
        - containerPort: 8100
          name: metrics
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /status
            port: 8100
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
      - env:
        - name: CONTROLLER_KONG_ADMIN_URL
          value: http://0.0.0.0:8444
        - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY
          value: "true"
        - name: CONTROLLER_PUBLISH_SERVICE
          value: kong/kong-proxy
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        image: harbor.59iedu.com/fjhb/kong-ingress-controller:1.0
        imagePullPolicy: Always
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        name: ingress-controller
        ports:
        - containerPort: 8080
          name: webhook
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
      initContainers:
      - command:
        - /bin/sh
        - -c
        - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi; sleep 2; done;
        env:
        - name: KONG_DATABASE
          value: postgres
        - name: KONG_PG_HOST
          value: postgres
        - name: KONG_PG_USER
          value: postgres
        - name: KONG_PG_PASSWORD
          value: "123456"
        - name: KONG_PG_DATABASE
          value: kong_cce
        - name: KONG_PG_PORT
          value: "5432"
        image: kong:2.1
        name: wait-for-migrations
      serviceAccountName: kong-serviceaccount
      imagePullSecrets:
        - name: harborsecret
EOF


上一篇:03-K8s部署安装配置nginx-ingress和配置外网访问


下一篇:容器云平台No.4~kubernetes 服务暴露之Ingress