?随着全面微服务化的落地,在网关层上对运维提出的新的要求,经过了几轮测试与验证,最终选型微服务网关kong来替换nginx。本文将简要介绍如何将Kong网关部署在K8S环境中。

关于Kong的简要介绍请参考前文。

1、下载相关进行并上传harbor私服

docker?pull?kong:2.1
docker?pull?kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0
docker?tag?kong:2.1?harbor.59iedu.com/fjhb/kong:2.1
docker?tag?kong-docker-kubernetes-ingress-controller.bintray.io/kong-ingress-controller:1.0?harbor.59iedu.com/fjhb/kong-ingress-controller:1.0
docker?push?harbor.59iedu.com/fjhb/kong:2.1
docker?push?harbor.59iedu.com/fjhb/kong-ingress-controller:1.0

2、创建pg数据库

本例中PG数据库放在k8s集群外部

create?database?kong_cce;

3、创建namespace

cat?<<EOF?|?kubectl?apply?-f?-
apiVersion:?v1
kind:?Namespace
metadata:
??name:?kong
EOF

4、创建crds

cat?<<EOF?|?kubectl?apply?-f?-
apiVersion:?apiextensions.k8s.io/v1beta1
kind:?CustomResourceDefinition
metadata:
??name:?kongclusterplugins.configuration.konghq.com
spec:
??additionalPrinterColumns:
??-?JSONPath:?.plugin
????description:?Name?of?the?plugin
????name:?Plugin-Type
????type:?string
??-?JSONPath:?.metadata.creationTimestamp
????description:?Age
????name:?Age
????type:?date
??-?JSONPath:?.disabled
????description:?Indicates?if?the?plugin?is?disabled
????name:?Disabled
????priority:?1
????type:?boolean
??-?JSONPath:?.config
????description:?Configuration?of?the?plugin
????name:?Config
????priority:?1
????type:?string
??group:?configuration.konghq.com
??names:
????kind:?KongClusterPlugin
????plural:?kongclusterplugins
????shortNames:
????-?kcp
??scope:?Cluster
??subresources:
????status:?{}
??validation:
????openAPIV3Schema:
??????properties:
????????config:
??????????type:?object
????????configFrom:
??????????properties:
????????????secretKeyRef:
??????????????properties:
????????????????key:
??????????????????type:?string
????????????????name:
??????????????????type:?string
????????????????namespace:
??????????????????type:?string
??????????????required:
??????????????-?name
??????????????-?namespace
??????????????-?key
??????????????type:?object
??????????type:?object
????????disabled:
??????????type:?boolean
????????plugin:
??????????type:?string
????????protocols:
??????????items:
????????????enum:
????????????-?http
????????????-?https
????????????-?grpc
????????????-?grpcs
????????????-?tcp
????????????-?tls
????????????type:?string
??????????type:?array
????????run_on:
??????????enum:
??????????-?first
??????????-?second
??????????-?all
??????????type:?string
??????required:
??????-?plugin
??version:?v1
---
apiVersion:?apiextensions.k8s.io/v1beta1
kind:?CustomResourceDefinition
metadata:
??name:?kongconsumers.configuration.konghq.com
spec:
??additionalPrinterColumns:
??-?JSONPath:?.username
????description:?Username?of?a?Kong?Consumer
????name:?Username
????type:?string
??-?JSONPath:?.metadata.creationTimestamp
????description:?Age
????name:?Age
????type:?date
??group:?configuration.konghq.com
??names:
????kind:?KongConsumer
????plural:?kongconsumers
????shortNames:
????-?kc
??scope:?Namespaced
??subresources:
????status:?{}
??validation:
????openAPIV3Schema:
??????properties:
????????credentials:
??????????items:
????????????type:?string
??????????type:?array
????????custom_id:
??????????type:?string
????????username:
??????????type:?string
??version:?v1
---
apiVersion:?apiextensions.k8s.io/v1beta1
kind:?CustomResourceDefinition
metadata:
??name:?kongingresses.configuration.konghq.com
spec:
??group:?configuration.konghq.com
??names:
????kind:?KongIngress
????plural:?kongingresses
????shortNames:
????-?ki
??scope:?Namespaced
??subresources:
????status:?{}
??validation:
????openAPIV3Schema:
??????properties:
????????proxy:
??????????properties:
????????????connect_timeout:
??????????????minimum:?0
??????????????type:?integer
????????????path:
??????????????pattern:?^/.*$
??????????????type:?string
????????????protocol:
??????????????enum:
??????????????-?http
??????????????-?https
??????????????-?grpc
??????????????-?grpcs
??????????????-?tcp
??????????????-?tls
??????????????type:?string
????????????read_timeout:
??????????????minimum:?0
??????????????type:?integer
????????????retries:
??????????????minimum:?0
??????????????type:?integer
????????????write_timeout:
??????????????minimum:?0
??????????????type:?integer
??????????type:?object
????????route:
??????????properties:
????????????headers:
??????????????additionalProperties:
????????????????items:
??????????????????type:?string
????????????????type:?array
??????????????type:?object
????????????https_redirect_status_code:
??????????????type:?integer
????????????methods:
??????????????items:
????????????????type:?string
??????????????type:?array
????????????path_handling:
??????????????enum:
??????????????-?v0
??????????????-?v1
??????????????type:?string
????????????preserve_host:
??????????????type:?boolean
????????????protocols:
??????????????items:
????????????????enum:
????????????????-?http
????????????????-?https
????????????????-?grpc
????????????????-?grpcs
????????????????-?tcp
????????????????-?tls
????????????????type:?string
??????????????type:?array
????????????regex_priority:
??????????????type:?integer
????????????strip_path:
??????????????type:?boolean
????????upstream:
??????????properties:
????????????algorithm:
??????????????enum:
??????????????-?round-robin
??????????????-?consistent-hashing
??????????????-?least-connections
??????????????type:?string
????????????hash_fallback:
??????????????type:?string
????????????hash_fallback_header:
??????????????type:?string
????????????hash_on:
??????????????type:?string
????????????hash_on_cookie:
??????????????type:?string
????????????hash_on_cookie_path:
??????????????type:?string
????????????hash_on_header:
??????????????type:?string
????????????healthchecks:
??????????????properties:
????????????????active:
??????????????????properties:
????????????????????concurrency:
??????????????????????minimum:?1
??????????????????????type:?integer
????????????????????healthy:
??????????????????????properties:
????????????????????????http_statuses:
??????????????????????????items:
????????????????????????????type:?integer
??????????????????????????type:?array
????????????????????????interval:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????successes:
??????????????????????????minimum:?0
??????????????????????????type:?integer
??????????????????????type:?object
????????????????????http_path:
??????????????????????pattern:?^/.*$
??????????????????????type:?string
????????????????????timeout:
??????????????????????minimum:?0
??????????????????????type:?integer
????????????????????unhealthy:
??????????????????????properties:
????????????????????????http_failures:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????http_statuses:
??????????????????????????items:
????????????????????????????type:?integer
??????????????????????????type:?array
????????????????????????interval:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????tcp_failures:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????timeout:
??????????????????????????minimum:?0
??????????????????????????type:?integer
??????????????????????type:?object
??????????????????type:?object
????????????????passive:
??????????????????properties:
????????????????????healthy:
??????????????????????properties:
????????????????????????http_statuses:
??????????????????????????items:
????????????????????????????type:?integer
??????????????????????????type:?array
????????????????????????interval:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????successes:
??????????????????????????minimum:?0
??????????????????????????type:?integer
??????????????????????type:?object
????????????????????unhealthy:
??????????????????????properties:
????????????????????????http_failures:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????http_statuses:
??????????????????????????items:
????????????????????????????type:?integer
??????????????????????????type:?array
????????????????????????interval:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????tcp_failures:
??????????????????????????minimum:?0
??????????????????????????type:?integer
????????????????????????timeout:
??????????????????????????minimum:?0
??????????????????????????type:?integer
??????????????????????type:?object
??????????????????type:?object
????????????????threshold:
??????????????????type:?integer
??????????????type:?object
????????????host_header:
??????????????type:?string
????????????slots:
??????????????minimum:?10
??????????????type:?integer
??????????type:?object
??version:?v1
---
apiVersion:?apiextensions.k8s.io/v1beta1
kind:?CustomResourceDefinition
metadata:
??name:?kongplugins.configuration.konghq.com
spec:
??additionalPrinterColumns:
??-?JSONPath:?.plugin
????description:?Name?of?the?plugin
????name:?Plugin-Type
????type:?string
??-?JSONPath:?.metadata.creationTimestamp
????description:?Age
????name:?Age
????type:?date
??-?JSONPath:?.disabled
????description:?Indicates?if?the?plugin?is?disabled
????name:?Disabled
????priority:?1
????type:?boolean
??-?JSONPath:?.config
????description:?Configuration?of?the?plugin
????name:?Config
????priority:?1
????type:?string
??group:?configuration.konghq.com
??names:
????kind:?KongPlugin
????plural:?kongplugins
????shortNames:
????-?kp
??scope:?Namespaced
??subresources:
????status:?{}
??validation:
????openAPIV3Schema:
??????properties:
????????config:
??????????type:?object
????????configFrom:
??????????properties:
????????????secretKeyRef:
??????????????properties:
????????????????key:
??????????????????type:?string
????????????????name:
??????????????????type:?string
??????????????required:
??????????????-?name
??????????????-?key
??????????????type:?object
??????????type:?object
????????disabled:
??????????type:?boolean
????????plugin:
??????????type:?string
????????protocols:
??????????items:
????????????enum:
????????????-?http
????????????-?https
????????????-?grpc
????????????-?grpcs
????????????-?tcp
????????????-?tls
????????????type:?string
??????????type:?array
????????run_on:
??????????enum:
??????????-?first
??????????-?second
??????????-?all
??????????type:?string
??????required:
??????-?plugin
??version:?v1
---
apiVersion:?apiextensions.k8s.io/v1beta1
kind:?CustomResourceDefinition
metadata:
??name:?tcpingresses.configuration.konghq.com
spec:
??additionalPrinterColumns:
??-?JSONPath:?.status.loadBalancer.ingress[*].ip
????description:?Address?of?the?load?balancer
????name:?Address
????type:?string
??-?JSONPath:?.metadata.creationTimestamp
????description:?Age
????name:?Age
????type:?date
??group:?configuration.konghq.com
??names:
????kind:?TCPIngress
????plural:?tcpingresses
??scope:?Namespaced
??subresources:
????status:?{}
??validation:
????openAPIV3Schema:
??????properties:
????????apiVersion:
??????????type:?string
????????kind:
??????????type:?string
????????metadata:
??????????type:?object
????????spec:
??????????properties:
????????????rules:
??????????????items:
????????????????properties:
??????????????????backend:
????????????????????properties:
??????????????????????serviceName:
????????????????????????type:?string
??????????????????????servicePort:
????????????????????????format:?int32
????????????????????????type:?integer
????????????????????type:?object
??????????????????host:
????????????????????type:?string
??????????????????port:
????????????????????format:?int32
????????????????????type:?integer
????????????????type:?object
??????????????type:?array
????????????tls:
??????????????items:
????????????????properties:
??????????????????hosts:
????????????????????items:
??????????????????????type:?string
????????????????????type:?array
??????????????????secretName:
????????????????????type:?string
????????????????type:?object
??????????????type:?array
??????????type:?object
????????status:
??????????type:?object
??version:?v1beta1
status:
??acceptedNames:
????kind:?""
????plural:?""
??conditions:?[]
??storedVersions:?[]
EOF

5、创建pgsql endpoint

主要用途是为了在K8S中直接使用svc来访问集群外部的pgsql

cat?<<EOF?|?kubectl?apply?-f?-?
apiVersion:?v1
kind:?Endpoints
metadata:
??name:?postgres
??namespace:?kong
subsets:
??-?addresses:
??????-?ip:?192.168.1.14
????ports:
??????-?port:?5432
---
apiVersion:?v1
kind:?Service
metadata:
??name:?postgres
??namespace:?kong
spec:
??ports:
????-?port:?5432
??????protocol:?TCP
??????targetPort:?5432
EOF

?6、创建rbac

cat?<<EOF?|?kubectl?apply?-f?-?
apiVersion:?v1
kind:?ServiceAccount
metadata:
??name:?kong-serviceaccount
??namespace:?kong
---
apiVersion:?rbac.authorization.k8s.io/v1beta1
kind:?ClusterRole
metadata:
??name:?kong-ingress-clusterrole
rules:
-?apiGroups:
??-?""
??resources:
??-?endpoints
??-?nodes
??-?pods
??-?secrets
??verbs:
??-?list
??-?watch
-?apiGroups:
??-?""
??resources:
??-?nodes
??verbs:
??-?get
-?apiGroups:
??-?""
??resources:
??-?services
??verbs:
??-?get
??-?list
??-?watch
-?apiGroups:
??-?networking.k8s.io
??-?extensions
??-?networking.internal.knative.dev
??resources:
??-?ingresses
??verbs:
??-?get
??-?list
??-?watch
-?apiGroups:
??-?""
??resources:
??-?events
??verbs:
??-?create
??-?patch
-?apiGroups:
??-?networking.k8s.io
??-?extensions
??-?networking.internal.knative.dev
??resources:
??-?ingresses/status
??verbs:
??-?update
-?apiGroups:
??-?configuration.konghq.com
??resources:
??-?tcpingresses/status
??verbs:
??-?update
-?apiGroups:
??-?configuration.konghq.com
??resources:
??-?kongplugins
??-?kongclusterplugins
??-?kongcredentials
??-?kongconsumers
??-?kongingresses
??-?tcpingresses
??verbs:
??-?get
??-?list
??-?watch
-?apiGroups:
??-?""
??resources:
??-?configmaps
??verbs:
??-?create
??-?get
??-?update
---
apiVersion:?rbac.authorization.k8s.io/v1beta1
kind:?ClusterRoleBinding
metadata:
??name:?kong-ingress-clusterrole-nisa-binding
roleRef:
??apiGroup:?rbac.authorization.k8s.io
??kind:?ClusterRole
??name:?kong-ingress-clusterrole
subjects:
-?kind:?ServiceAccount
??name:?kong-serviceaccount
??namespace:?kong
EOF

7、创建job

主要用来执行kong migrate up指令初始化数据库

cat?<<EOF?|?kubectl?apply?-f?-?
apiVersion:?batch/v1
kind:?Job
metadata:
??name:?kong-migrations
??namespace:?kong
spec:
??template:
????metadata:
??????name:?kong-migrations
????spec:
??????containers:
??????-?command:
????????-?/bin/sh
????????-?-c
????????-?kong?migrations?bootstrap
????????env:
????????-?name:?KONG_DATABASE
??????????value:?postgres
????????-?name:?KONG_PG_HOST
??????????value:?postgres
????????-?name:?KONG_PG_USER
??????????value:?postgres
????????-?name:?KONG_PG_PASSWORD
??????????value:?"123456"
????????-?name:?KONG_PG_DATABASE
??????????value:?kong_cce
????????-?name:?KONG_PG_PORT
??????????value:?"5432"
????????image:?harbor.59iedu.com/fjhb/kong:2.1
????????name:?kong-migrations
??????initContainers:
??????-?command:
????????-?/bin/sh
????????-?-c
????????-?until?nc?-zv?$KONG_PG_HOST?$KONG_PG_PORT?-w1;?do?echo?'waiting?for?db';?sleep?1;?done
????????env:
????????-?name:?KONG_DATABASE
??????????value:?postgres
????????-?name:?KONG_PG_HOST
??????????value:?postgres
????????-?name:?KONG_PG_USER
??????????value:?postgres
????????-?name:?KONG_PG_PASSWORD
??????????value:?"123456"
????????-?name:?KONG_PG_DATABASE
??????????value:?kong_cce
????????-?name:?KONG_PG_PORT
??????????value:?"5432"
????????image:?busybox
????????name:?wait-for-postgres
??????restartPolicy:?OnFailure
	??imagePullSecrets:
????????-?name:?harborsecret
EOF

8、创建工作负载

cat?<<EOF?|?kubectl?apply?-f?-
apiVersion:?v1
kind:?Service
metadata:
??annotations:
????service.beta.kubernetes.io/aws-load-balancer-backend-protocol:?tcp
????service.beta.kubernetes.io/aws-load-balancer-type:?nlb
??name:?kong-proxy
??namespace:?kong
spec:
??ports:
??-?name:?proxy
????port:?80
????protocol:?TCP
????targetPort:?8000
??-?name:?proxy-ssl
????port:?443
????protocol:?TCP
????targetPort:?8443
??selector:
????app:?ingress-kong
---
apiVersion:?apps/v1
kind:?Deployment
metadata:
??labels:
????app:?ingress-kong
??name:?ingress-kong
??namespace:?kong
spec:
??replicas:?1
??selector:
????matchLabels:
??????app:?ingress-kong
??template:
????metadata:
??????annotations:
????????kuma.io/gateway:?enabled
????????prometheus.io/port:?"8100"
????????prometheus.io/scrape:?"true"
????????traffic.sidecar.istio.io/includeInboundPorts:?""
??????labels:
????????app:?ingress-kong
????spec:
??????containers:
??????-?env:
????????-?name:?KONG_DATABASE
??????????value:?postgres
????????-?name:?KONG_PG_HOST
??????????value:?postgres
????????-?name:?KONG_PG_USER
??????????value:?postgres
????????-?name:?KONG_PG_PASSWORD
??????????value:?"123456"
????????-?name:?KONG_PG_DATABASE
??????????value:?kong_cce
????????-?name:?KONG_PG_PORT
??????????value:?"5432"
????????-?name:?KONG_PROXY_LISTEN
??????????value:?0.0.0.0:8000,?0.0.0.0:8443?ssl?http2
????????-?name:?KONG_PORT_MAPS
??????????value:?80:8000,?443:8443
????????-?name:?KONG_ADMIN_LISTEN
??????????value:?0.0.0.0:8444?
????????-?name:?KONG_STATUS_LISTEN
??????????value:?0.0.0.0:8100
????????-?name:?KONG_NGINX_WORKER_PROCESSES
??????????value:?"2"
????????-?name:?KONG_ADMIN_ACCESS_LOG
??????????value:?/dev/stdout
????????-?name:?KONG_ADMIN_ERROR_LOG
??????????value:?/dev/stderr
????????-?name:?KONG_PROXY_ERROR_LOG
??????????value:?/dev/stderr
????????image:?harbor.59iedu.com/fjhb/kong:2.1
????????lifecycle:
??????????preStop:
????????????exec:
??????????????command:
??????????????-?/bin/sh
??????????????-?-c
??????????????-?kong?quit
????????livenessProbe:
??????????failureThreshold:?3
??????????httpGet:
????????????path:?/status
????????????port:?8100
????????????scheme:?HTTP
??????????initialDelaySeconds:?5
??????????periodSeconds:?10
??????????successThreshold:?1
??????????timeoutSeconds:?1
????????name:?proxy
????????ports:
????????-?containerPort:?8000
??????????name:?proxy
??????????protocol:?TCP
????????-?containerPort:?8443
??????????name:?proxy-ssl
??????????protocol:?TCP
????????-?containerPort:?8100
??????????name:?metrics
??????????protocol:?TCP
????????readinessProbe:
??????????failureThreshold:?3
??????????httpGet:
????????????path:?/status
????????????port:?8100
????????????scheme:?HTTP
??????????initialDelaySeconds:?5
??????????periodSeconds:?10
??????????successThreshold:?1
??????????timeoutSeconds:?1
??????-?env:
????????-?name:?CONTROLLER_KONG_ADMIN_URL
??????????value:?http://sb138.99sbgw.com:8444
????????-?name:?CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY
??????????value:?"true"
????????-?name:?CONTROLLER_PUBLISH_SERVICE
??????????value:?kong/kong-proxy
????????-?name:?POD_NAME
??????????valueFrom:
????????????fieldRef:
??????????????apiVersion:?v1
??????????????fieldPath:?metadata.name
????????-?name:?POD_NAMESPACE
??????????valueFrom:
????????????fieldRef:
??????????????apiVersion:?v1
??????????????fieldPath:?metadata.namespace
????????image:?harbor.59iedu.com/fjhb/kong-ingress-controller:1.0
????????imagePullPolicy:?Always
????????livenessProbe:
??????????failureThreshold:?3
??????????httpGet:
????????????path:?/healthz
????????????port:?10254
????????????scheme:?HTTP
??????????initialDelaySeconds:?5
??????????periodSeconds:?10
??????????successThreshold:?1
??????????timeoutSeconds:?1
????????name:?ingress-controller
????????ports:
????????-?containerPort:?8080
??????????name:?webhook
??????????protocol:?TCP
????????readinessProbe:
??????????failureThreshold:?3
??????????httpGet:
????????????path:?/healthz
????????????port:?10254
????????????scheme:?HTTP
??????????initialDelaySeconds:?5
??????????periodSeconds:?10
??????????successThreshold:?1
??????????timeoutSeconds:?1
??????initContainers:
??????-?command:
????????-?/bin/sh
????????-?-c
????????-?while?true;?do?kong?migrations?list;?if?[[?0?-eq?$??]];?then?exit?0;?fi;?sleep?2;?done;
????????env:
????????-?name:?KONG_DATABASE
??????????value:?postgres
????????-?name:?KONG_PG_HOST
??????????value:?postgres
????????-?name:?KONG_PG_USER
??????????value:?postgres
????????-?name:?KONG_PG_PASSWORD
??????????value:?"123456"
????????-?name:?KONG_PG_DATABASE
??????????value:?kong_cce
????????-?name:?KONG_PG_PORT
??????????value:?"5432"
????????image:?kong:2.1
????????name:?wait-for-migrations
??????serviceAccountName:?kong-serviceaccount
??????imagePullSecrets:
????????-?name:?harborsecret
EOF