mTLS Everywhere: How to Configure TLS for APISIX

helm repo add jetstack https://charts.jetstack.io  #1

helm install \
  cert-manager jetstack/cert-manager \
  --namespace cert-manager \                       #2
  --create-namespace \                             #2
  --version v1.11.0 \
  --set installCRDs=true \
  --set prometheus.enabled=false                   #3

kubectl get pods -n cert-manager

cert-manager-cainjector-7f694c4c58-fc9bk  1/1  Running  2  (2d1h ago)  7d
cert-manager-cc4b776cf-8p2t8              1/1  Running  1  (2d1h ago)  7d
cert-manager-webhook-7cd8c769bb-494tl     1/1  Running  1  (2d1h ago)  7d

apiVersion: cert-manager.io/v1
kind: ClusterIssuer                           #1
metadata:
  name: selfsigned-issuer
spec:
  selfSigned: {}
---
apiVersion: v1
kind: Namespace
metadata:
  name: tls                                   #2
---
apiVersion: cert-manager.io/v1
kind: Certificate                             #3
metadata:
  name: selfsigned-ca
  namespace: tls
spec:
  isCA: true
  commonName: selfsigned-ca
  secretName: root-secret
  issuerRef:
    name: selfsigned-issuer
    kind: ClusterIssuer
    group: cert-manager.io
---
apiVersion: cert-manager.io/v1
kind: Issuer                                  #4
metadata:
  name: ca-issuer
  namespace: tls
spec:
  ca:
    secretName: root-secret

kubectl get certificate -n tls

NAME            READY   SECRET        AGE
selfsigned-ca   True    root-secret   7s

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: etcd-server                         #1
  namespace: tls
spec:
  secretName: etcd-secret                   #2
  isCA: false
  usages:
    - client auth                           #3
    - server auth                           #3
  dnsNames:
    - etcd                                  #4
  issuerRef:
    name: ca-issuer                         #5
    kind: Issuer
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: apisix-client                       #6
  namespace: tls
spec:
  secretName: apisix-client-secret
  isCA: false
  usages:
    - client auth
  emailAddresses:
    - apisix@apache.org                     #7
  issuerRef:
    name: ca-issuer                         #5
    kind: Issuer

kubectl get certificates -n tls

NAME              READY   SECRET                 AGE
selfsigned-ca     True    root-secret            8m59s    //1
apisix-client     True    apisix-client-secret   8m22s    //2
etcd-server       True    etcd-secret            8m54s    //2

kubectl get secrets -n tls

NAME                   TYPE                DATA   AGE
apisix-client-secret   kubernetes.io/tls   3      35m
etcd-secret            kubernetes.io/tls   3      35m
root-secret            kubernetes.io/tls   3      35m

kubectl describe apisix-client-secret -n tls

Name:         apisix-client-secret
Namespace:    tls
Labels:       controller.cert-manager.io/fao=true
Annotations:  cert-manager.io/alt-names:
              cert-manager.io/certificate-name: apisix-client
              cert-manager.io/common-name:
              cert-manager.io/ip-sans:
              cert-manager.io/issuer-group:
              cert-manager.io/issuer-kind: Issuer
              cert-manager.io/issuer-name: ca-issuer
              cert-manager.io/uri-sans:

Type:  kubernetes.io/tls

Data
====
ca.crt:   1099 bytes
tls.crt:  1115 bytes
tls.key:  1679 bytes

 kubectl get secret etcd-secret -n tls -o jsonpath='{ .data.tls\.crt }' | base64

-----BEGIN CERTIFICATE-----
MIIDBjCCAe6gAwIBAgIQM3JUR8+R0vuUndjGK/aOgzANBgkqhkiG9w0BAQsFADAY
MRYwFAYDVQQDEw1zZWxmc2lnbmVkLWNhMB4XDTIzMDMxNjEwMTYyN1oXDTIzMDYx
NDEwMTYyN1owADCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMQpMj/0
giDVOjOosSRRKUwTzl1Wo2R9YYAeteOW3fuMiAd+XaBGmRO/+GWZQN1tyRQ3pITM
ezBgogYAUUNcuqN/UAsgH/JM58niMjZdjRKn4+it94Nj1e24jFL4ts2snCn7FfKJ
3zRtY9tyS7Agw3tCwtXV68Xpmf3CsfhPmn3rGdWHXyYctzAZhqYfEswN3hxpJZxR
YVeb55WgDoPo5npZo3+yYiMtoOimIprcmZ2Ye8Wai9S4QKDafUWlvU5GQ65VVLzH
PEdOMwbWcwiLqwUv889TiKiC5cyAD6wJOuPRF0KKxxFnG+lHlg9J2S1i5sC3pqoc
i0pEQ+atOOyLMMECAwEAAaNkMGIwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUF
BwMBMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAU2ZaAdEficKUWPFRjdsKSEX/l
gbMwEgYDVR0RAQH/BAgwBoIEZXRjZDANBgkqhkiG9w0BAQsFAAOCAQEABcNvYTm8
ZJe3jUq6f872dpNVulb2UvloTpWxQ8jwXgcrhekSKU6pZ4p9IPwfauHLjceMFJLp
t2eDi5fSQ1upeqXOofeyKSYjjyA/aVf1zMI8ReCCQtQuAVYyJWBlNLc3XMMecbcp
JLGtd/OAZnKDeYYkUX7cJ2wN6Wl/wGLM2lxsqDhEHEZwvGL0DmsdHw7hzSjdVmxs
0Qgkh4jVbNUKdBok5U9Ivr3P1xDPaD/FqGFyM0ssVOCHxtPxhOUA/m3DSr6klfEF
McOfudZE958bChOrJgVrUnY3inR0J335bGQ1luEp5tYwPgyD9dG4MQEDD3oLwp+l
+NtTUqz8WVlMxQ==
-----END CERTIFICATE-----

apiVersion: v1
kind: Pod
metadata:
  name: etcd
  namespace: tls
  labels:
    role: config
spec:
  containers:
    - name: etcd
      image: bitnami/etcd:3.5.7
      ports:
        - containerPort: 2379
      env:
        - name: ETCD_TRUSTED_CA_FILE        #1
          value: /etc/ssl/private/ca.crt
        - name: ETCD_CERT_FILE              #2
          value: /etc/ssl/private/tls.crt
        - name: ETCD_KEY_FILE               #3
          value: /etc/ssl/private/tls.key
        - name: ETCD_ROOT_PASSWORD
          value: whatever
        - name: ETCD_CLIENT_CERT_AUTH       #4
          value: "true"
        - name: ETCD_LISTEN_CLIENT_URLS
          value: https://0.0.0.0:2379
      volumeMounts:
        - name: ssl
          mountPath: /etc/ssl/private       #5
  volumes:
    - name: ssl
      secret:
        secretName: etcd-secret             #5

apiVersion: v1
kind: ConfigMap                                            #1
metadata:
  name: apisix-config
  namespace: tls
data:
  config.yaml: >-
    apisix:
      ssl:
        ssl_trusted_certificate: /etc/ssl/certs/ca.crt     #2
    deployment:
      etcd:
        host:
          - https://etcd:2379
        tls:
          cert: /etc/ssl/certs/tls.crt                     #2
          key: /etc/ssl/certs/tls.key                      #2
      admin:
        allow_admin:
          - 0.0.0.0/0
        https_admin: true                                  #3
        admin_api_mtls:
          admin_ssl_cert: /etc/ssl/private/tls.crt         #3
          admin_ssl_cert_key: /etc/ssl/private/tls.key     #3
          admin_ssl_ca_cert: /etc/ssl/private/ca.crt       #3
---
apiVersion: v1
kind: Pod
metadata:
  name: apisix
  namespace: tls
  labels:
    role: gateway
spec:
  containers:
    - name: apisix
      image: apache/apisix:3.2.0-debian
      ports:
        - containerPort: 9443                              #4
        - containerPort: 9180                              #5
      volumeMounts:
        - name: config                                      #1
          mountPath: /usr/local/apisix/conf/config.yaml
          subPath: config.yaml
        - name: ssl                                        #6
          mountPath: /etc/ssl/private
        - name: etcd-client                                #7
          mountPath: /etc/ssl/certs
  volumes:
    - name: config
      configMap:
        name: apisix-config
    - name: ssl                                            #6,8
      secret:
        secretName: apisix-server-secret
    - name: etcd-client                                    #7,8
      secret:
        secretName: apisix-client-secret

NAME     READY   STATUS    RESTARTS   AGE
apisix   1/1     Running   0          179m
etcd     1/1     Running   0          179m

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: curl-client
  namespace: tls
spec:
  secretName: curl-secret
  isCA: false
  usages:
    - client auth
  emailAddresses:
    - curl@localhost.dev
  issuerRef:
    name: ca-issuer
    kind: Issuer

curl --resolve 'admin:32180:127.0.0.1' https://admin:32180/apisix/admin/routes \                     #1
     --cert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.crt }' | base64 -d) \  #2
     --key =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.key }' | base64 -d) \   #2
     --cacert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.ca\.crt }' | base64 -d) \ #2
     -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1'

{"total":0,"list":[]}

apiVersion: v1
kind: ConfigMap                                           #1
metadata:
  name: nginx-config
  namespace: tls
data:
  nginx.conf: >-
    events {
      worker_connections 1024;
    }
    http {
      server {
        listen              443 ssl;
        server_name         upstream;
        ssl_certificate     /etc/ssl/private/tls.crt;     #2
        ssl_certificate_key /etc/ssl/private/tls.key;     #2

        root /www/data;
        location / {
            index index.json;
        }
      }
    }
---
apiVersion: v1
kind: Pod
metadata:
  name: upstream
  namespace: tls
  labels:
    role: upstream
spec:
  containers:
    - name: upstream
      image: nginx:1.23-alpine
      ports:
        - containerPort: 443
      volumeMounts:
        - name: config
          mountPath: /etc/nginx/nginx.conf                #1
          subPath: nginx.conf
        - name: content
          mountPath: /www/data/index.json                 #3
          subPath: index.json
        - name: ssl                                       #2
          mountPath: /etc/ssl/private
  volumes:
    - name: config
      configMap:
        name: nginx-config
    - name: ssl                                           #2
      secret:
        secretName: upstream-secret
    - name: content                                       #3
      configMap:
        name: nginx-content

curl --resolve 'admin:32180:127.0.0.1' https://admin:32180/apisix/admin/routes/1 \
     --cert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.crt }' | base64 -d) \     #1
     --key =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.key }' | base64 -d) \      #1
     --cacert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.ca\.crt }' | base64 -d) \    #1
     -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -i -d "{
        \"uri\": \"/\",
        \"upstream\": {
          \"scheme\": \"https\",                                                                        #2
          \"nodes\": {
            \"upstream:443\": 1
          },
          \"tls\": {
            \"client_cert\": \"$(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.crt }' | base64 -d)\", #3
            \"client_key\": \"$(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.key }' | base64 -d)\"   #3
          }
        }
     }"

curl --resolve 'upstream:32443:127.0.0.1' https://upstream:32443/ \
     --cert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.crt }' | base64 -d) \
     --key =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.tls\.key }' | base64 -d) \
     --cacert =(kubectl get secret curl-secret -n tls -o jsonpath='{ .data.ca\.crt }' | base64 -d)

{ "hello": "world" }