随着全面微服务化的落地，在网关层上对运维提出的新的要求，经过了几轮测试与验证，最终选型微服务网关kong来替换nginx。本文将简要介绍如何将Kong网关部署在K8S环境中。
1、下载相关进行并上传harbor私服
docker pull kong:2.1 docker pull kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0 docker tag kong:2.1 harbor.59iedu.com/fjhb/kong:2.1 docker tag kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0 harbor.59iedu.com/fjhb/kong-ingress-controller:1.0 docker push harbor.59iedu.com/fjhb/kong:2.1 docker push harbor.59iedu.com/fjhb/kong-ingress-controller:1.0
2、创建pg数据库
本例中PG数据库放在k8s集群外部
create database kong_cce;
3、创建namespace
cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Namespace metadata: name: kong EOF
4、创建crds
cat <<EOF | kubectl apply -f -
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: kongclusterplugins.configuration.konghq.com
spec:
additionalPrinterColumns:
- JSONPath: .plugin
description: Name of the plugin
name: Plugin-Type
type: string
- JSONPath: .metadata.creationTimestamp
description: Age
name: Age
type: date
- JSONPath: .disabled
description: Indicates if the plugin is disabled
name: Disabled
priority: 1
type: boolean
- JSONPath: .config
description: Configuration of the plugin
name: Config
priority: 1
type: string
group: configuration.konghq.com
names:
kind: KongClusterPlugin
plural: kongclusterplugins
shortNames:
- kcp
scope: Cluster
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
config:
type: object
configFrom:
properties:
secretKeyRef:
properties:
key:
type: string
name:
type: string
namespace:
type: string
required:
- name
- namespace
- key
type: object
type: object
disabled:
type: boolean
plugin:
type: string
protocols:
items:
enum:
- http
- https
- grpc
- grpcs
- tcp
- tls
type: string
type: array
run_on:
enum:
- first
- second
- all
type: string
required:
- plugin
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: kongconsumers.configuration.konghq.com
spec:
additionalPrinterColumns:
- JSONPath: .username
description: Username of a Kong Consumer
name: Username
type: string
- JSONPath: .metadata.creationTimestamp
description: Age
name: Age
type: date
group: configuration.konghq.com
names:
kind: KongConsumer
plural: kongconsumers
shortNames:
- kc
scope: Namespaced
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
credentials:
items:
type: string
type: array
custom_id:
type: string
username:
type: string
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: kongingresses.configuration.konghq.com
spec:
group: configuration.konghq.com
names:
kind: KongIngress
plural: kongingresses
shortNames:
- ki
scope: Namespaced
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
proxy:
properties:
connect_timeout:
minimum: 0
type: integer
path:
pattern: ^/.*$
type: string
protocol:
enum:
- http
- https
- grpc
- grpcs
- tcp
- tls
type: string
read_timeout:
minimum: 0
type: integer
retries:
minimum: 0
type: integer
write_timeout:
minimum: 0
type: integer
type: object
route:
properties:
headers:
additionalProperties:
items:
type: string
type: array
type: object
https_redirect_status_code:
type: integer
methods:
items:
type: string
type: array
path_handling:
enum:
- v0
- v1
type: string
preserve_host:
type: boolean
protocols:
items:
enum:
- http
- https
- grpc
- grpcs
- tcp
- tls
type: string
type: array
regex_priority:
type: integer
strip_path:
type: boolean
upstream:
properties:
algorithm:
enum:
- round-robin
- consistent-hashing
- least-connections
type: string
hash_fallback:
type: string
hash_fallback_header:
type: string
hash_on:
type: string
hash_on_cookie:
type: string
hash_on_cookie_path:
type: string
hash_on_header:
type: string
healthchecks:
properties:
active:
properties:
concurrency:
minimum: 1
type: integer
healthy:
properties:
http_statuses:
items:
type: integer
type: array
interval:
minimum: 0
type: integer
successes:
minimum: 0
type: integer
type: object
http_path:
pattern: ^/.*$
type: string
timeout:
minimum: 0
type: integer
unhealthy:
properties:
http_failures:
minimum: 0
type: integer
http_statuses:
items:
type: integer
type: array
interval:
minimum: 0
type: integer
tcp_failures:
minimum: 0
type: integer
timeout:
minimum: 0
type: integer
type: object
type: object
passive:
properties:
healthy:
properties:
http_statuses:
items:
type: integer
type: array
interval:
minimum: 0
type: integer
successes:
minimum: 0
type: integer
type: object
unhealthy:
properties:
http_failures:
minimum: 0
type: integer
http_statuses:
items:
type: integer
type: array
interval:
minimum: 0
type: integer
tcp_failures:
minimum: 0
type: integer
timeout:
minimum: 0
type: integer
type: object
type: object
threshold:
type: integer
type: object
host_header:
type: string
slots:
minimum: 10
type: integer
type: object
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: kongplugins.configuration.konghq.com
spec:
additionalPrinterColumns:
- JSONPath: .plugin
description: Name of the plugin
name: Plugin-Type
type: string
- JSONPath: .metadata.creationTimestamp
description: Age
name: Age
type: date
- JSONPath: .disabled
description: Indicates if the plugin is disabled
name: Disabled
priority: 1
type: boolean
- JSONPath: .config
description: Configuration of the plugin
name: Config
priority: 1
type: string
group: configuration.konghq.com
names:
kind: KongPlugin
plural: kongplugins
shortNames:
- kp
scope: Namespaced
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
config:
type: object
configFrom:
properties:
secretKeyRef:
properties:
key:
type: string
name:
type: string
required:
- name
- key
type: object
type: object
disabled:
type: boolean
plugin:
type: string
protocols:
items:
enum:
- http
- https
- grpc
- grpcs
- tcp
- tls
type: string
type: array
run_on:
enum:
- first
- second
- all
type: string
required:
- plugin
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: tcpingresses.configuration.konghq.com
spec:
additionalPrinterColumns:
- JSONPath: .status.loadBalancer.ingress[*].ip
description: Address of the load balancer
name: Address
type: string
- JSONPath: .metadata.creationTimestamp
description: Age
name: Age
type: date
group: configuration.konghq.com
names:
kind: TCPIngress
plural: tcpingresses
scope: Namespaced
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
apiVersion:
type: string
kind:
type: string
metadata:
type: object
spec:
properties:
rules:
items:
properties:
backend:
properties:
serviceName:
type: string
servicePort:
format: int32
type: integer
type: object
host:
type: string
port:
format: int32
type: integer
type: object
type: array
tls:
items:
properties:
hosts:
items:
type: string
type: array
secretName:
type: string
type: object
type: array
type: object
status:
type: object
version: v1beta1
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
EOF
5、创建pgsql endpoint
主要用途是为了在K8S中直接使用svc来访问集群外部的pgsql
cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Endpoints metadata: name: postgres namespace: kong subsets: - addresses: - ip: 192.168.1.14 ports: - port: 5432 --- apiVersion: v1 kind: Service metadata: name: postgres namespace: kong spec: ports: - port: 5432 protocol: TCP targetPort: 5432 EOF
6、创建rbac
cat <<EOF | kubectl apply -f - apiVersion: v1 kind: ServiceAccount metadata: name: kong-serviceaccount namespace: kong --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: kong-ingress-clusterrole rules: - apiGroups: - "" resources: - endpoints - nodes - pods - secrets verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get - apiGroups: - "" resources: - services verbs: - get - list - watch - apiGroups: - networking.k8s.io - extensions - networking.internal.knative.dev resources: - ingresses verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - networking.k8s.io - extensions - networking.internal.knative.dev resources: - ingresses/status verbs: - update - apiGroups: - configuration.konghq.com resources: - tcpingresses/status verbs: - update - apiGroups: - configuration.konghq.com resources: - kongplugins - kongclusterplugins - kongcredentials - kongconsumers - kongingresses - tcpingresses verbs: - get - list - watch - apiGroups: - "" resources: - configmaps verbs: - create - get - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: kong-ingress-clusterrole-nisa-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: kong-ingress-clusterrole subjects: - kind: ServiceAccount name: kong-serviceaccount namespace: kong EOF
7、创建job
主要用来执行kong migrate up指令初始化数据库
cat <<EOF | kubectl apply -f - apiVersion: batch/v1 kind: Job metadata: name: kong-migrations namespace: kong spec: template: metadata: name: kong-migrations spec: containers: - command: - /bin/sh - -c - kong migrations bootstrap env: - name: KONG_DATABASE value: postgres - name: KONG_PG_HOST value: postgres - name: KONG_PG_USER value: postgres - name: KONG_PG_PASSWORD value: "123456" - name: KONG_PG_DATABASE value: kong_cce - name: KONG_PG_PORT value: "5432" image: harbor.59iedu.com/fjhb/kong:2.1 name: kong-migrations initContainers: - command: - /bin/sh - -c - until nc -zv $KONG_PG_HOST $KONG_PG_PORT -w1; do echo 'waiting for db'; sleep 1; done env: - name: KONG_DATABASE value: postgres - name: KONG_PG_HOST value: postgres - name: KONG_PG_USER value: postgres - name: KONG_PG_PASSWORD value: "123456" - name: KONG_PG_DATABASE value: kong_cce - name: KONG_PG_PORT value: "5432" image: busybox name: wait-for-postgres restartPolicy: OnFailure imagePullSecrets: - name: harborsecret EOF
8、创建工作负载
cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Service metadata: annotations: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp service.beta.kubernetes.io/aws-load-balancer-type: nlb name: kong-proxy namespace: kong spec: ports: - name: proxy port: 80 protocol: TCP targetPort: 8000 - name: proxy-ssl port: 443 protocol: TCP targetPort: 8443 selector: app: ingress-kong --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: ingress-kong name: ingress-kong namespace: kong spec: replicas: 1 selector: matchLabels: app: ingress-kong template: metadata: annotations: kuma.io/gateway: enabled prometheus.io/port: "8100" prometheus.io/scrape: "true" traffic.sidecar.istio.io/includeInboundPorts: "" labels: app: ingress-kong spec: containers: - env: - name: KONG_DATABASE value: postgres - name: KONG_PG_HOST value: postgres - name: KONG_PG_USER value: postgres - name: KONG_PG_PASSWORD value: "123456" - name: KONG_PG_DATABASE value: kong_cce - name: KONG_PG_PORT value: "5432" - name: KONG_PROXY_LISTEN value: 0.0.0.0:8000, 0.0.0.0:8443 ssl http2 - name: KONG_PORT_MAPS value: 80:8000, 443:8443 - name: KONG_ADMIN_LISTEN value: 0.0.0.0:8444 - name: KONG_STATUS_LISTEN value: 0.0.0.0:8100 - name: KONG_NGINX_WORKER_PROCESSES value: "2" - name: KONG_ADMIN_ACCESS_LOG value: /dev/stdout - name: KONG_ADMIN_ERROR_LOG value: /dev/stderr - name: KONG_PROXY_ERROR_LOG value: /dev/stderr image: harbor.59iedu.com/fjhb/kong:2.1 lifecycle: preStop: exec: command: - /bin/sh - -c - kong quit livenessProbe: failureThreshold: 3 httpGet: path: /status port: 8100 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 name: proxy ports: - containerPort: 8000 name: proxy protocol: TCP - containerPort: 8443 name: proxy-ssl protocol: TCP - containerPort: 8100 name: metrics protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /status port: 8100 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 - env: - name: CONTROLLER_KONG_ADMIN_URL value: http://0.0.0.0:8444 - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY value: "true" - name: CONTROLLER_PUBLISH_SERVICE value: kong/kong-proxy - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: harbor.59iedu.com/fjhb/kong-ingress-controller:1.0 imagePullPolicy: Always livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 name: ingress-controller ports: - containerPort: 8080 name: webhook protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 initContainers: - command: - /bin/sh - -c - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi; sleep 2; done; env: - name: KONG_DATABASE value: postgres - name: KONG_PG_HOST value: postgres - name: KONG_PG_USER value: postgres - name: KONG_PG_PASSWORD value: "123456" - name: KONG_PG_DATABASE value: kong_cce - name: KONG_PG_PORT value: "5432" image: kong:2.1 name: wait-for-migrations serviceAccountName: kong-serviceaccount imagePullSecrets: - name: harborsecret EOF