كيفية استخدام cert-manager و HashiCorp Vault لإدارة الشهادات؟

➜  ~ kubectl create secret tls moelove-tls --cert=./cert.pem --key=./cert-key.pem
secret/moelove-tls created
➜  ~ kubectl get secret moelove-tls -oyaml
apiVersion: v1
data:
  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNFekNDQWJtZ0F3SUJBZ0lVVHhCTC9aQkdpOEJCOUFVN2JRWi9jK3c2L1Rzd0NnWUlLb1pJemowRUF3SXcKVFRFTE1Ba0dBMVVFQmhNQ1EwNHhFREFPQmdOVkJBY1RCMEpsYVdwcGJtY3hGVEFUQmdOVkJBb1RERTF2WlV4dgpkbVVnU1U1R1R6RVZNQk1HQTFVRUF4TU1iVzlsYkc5MlpTNXBibVp2TUI0WERUSXlNVEF4T1RBM01UY3dNRm9YCkRUSXpNVEF4T1RBM01UY3dNRm93VFRFTE1Ba0dBMVVFQmhNQ1EwNHhFREFPQmdOVkJBY1RCMEpsYVdwcGJtY3gKRlRBVEJnTlZCQW9UREUxdlpVeHZkbVVnU1U1R1R6RVZNQk1HQTFVRUF4TU1iVzlsYkc5MlpTNXBibVp2TUZrdwpFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRVVTcEFjNGE1UXQwQ0NVa2hGSGY3WnZvR1FReVVPUUxSClJhZG0rSUUrV1ZkOThyWkc5NFpob08ybDZSWkY2MnVPN3FpZ2VsaUJwY0FGQ3FzWU9HNnVLcU4zTUhVd0RnWUQKVlIwUEFRSC9CQVFEQWdXZ01CMEdBMVVkSlFRV01CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFNQmdOVgpIUk1CQWY4RUFqQUFNQjBHQTFVZERnUVdCQlFnS01icnBUb3k4NVcvRy9hMGZtYzlDMUJRbURBWEJnTlZIUkVFCkVEQU9nZ3h0YjJWc2IzWmxMbWx1Wm04d0NnWUlLb1pJemowRUF3SURTQUF3UlFJZ1EzTzhJZ0N2MlRkNUhhV00KcE1LWmRCLzNXdEMreERlSVdPbER6L2hCdzE0Q0lRRExQNG0weFpmSkJvRGc5cERocThGdHN5VDdVZVhVdlZGQQpsS0tReFZNOXFBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
  tls.key: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUsyZjZHQlNZQ0R4eVoycnB2bVZ1YW5MNDhxeW9SK1NiWmxiQzNqSUZybzhvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFVVNwQWM0YTVRdDBDQ1VraEZIZjdadm9HUVF5VU9RTFJSYWRtK0lFK1dWZDk4clpHOTRaaApvTzJsNlJaRjYydU83cWlnZWxpQnBjQUZDcXNZT0c2dUtnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
kind: Secret
metadata:
  creationTimestamp: "2022-10-19T07:24:26Z"
  name: moelove-tls
  namespace: default
  resourceVersion: "2103326"
  uid: 14f86514-a1d1-4d99-b000-9ed8b5189d56
type: kubernetes.io/tls

➜  ~ kubectl create ingress moelove-ing --rule="moelove.info/=moelove:8080,tls=moelove-tls"
ingress.networking.k8s.io/moelove-ing created
➜  ~ kubectl get ing moelove-ing -oyaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  creationTimestamp: "2022-10-19T07:32:43Z"
  generation: 1
  name: moelove-ing
  namespace: default
  resourceVersion: "2104268"
  uid: b90f09f7-8036-4b9f-9744-a247141ea8da
spec:
  rules:
  - host: moelove.info
    http:
      paths:
      - backend:
          service:
            name: moelove
            port:
              number: 8080
        path: /
        pathType: Exact
  tls:
  - hosts:
    - moelove.info
    secretName: moelove-tls
status:
  loadBalancer: {}

➜  ~ kubectl get secrets moelove-tls -o jsonpath='{ .data.tls\.crt }' |base64 -d
-----BEGIN CERTIFICATE-----
MIICEzCCAbmgAwIBAgIUTxBL/ZBGi8BB9AU7bQZ/c+w6/TswCgYIKoZIzj0EAwIw
TTELMAkGA1UEBhMCQ04xEDAOBgNVBAcTB0JlaWppbmcxFTATBgNVBAoTDE1vZUxv
dmUgSU5GTzEVMBMGA1UEAxMMbW9lbG92ZS5pbmZvMB4XDTIyMTAxOTA3MTcwMFoX
DTIzMTAxOTA3MTcwMFowTTELMAkGA1UEBhMCQ04xEDAOBgNVBAcTB0JlaWppbmcx
FTATBgNVBAoTDE1vZUxvdmUgSU5GTzEVMBMGA1UEAxMMbW9lbG92ZS5pbmZvMFkw
EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEUSpAc4a5Qt0CCUkhFHf7ZvoGQQyUOQLR
Radm+IE+WVd98rZG94ZhoO2l6RZF62uO7qigeliBpcAFCqsYOG6uKqN3MHUwDgYD
VR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNV
HRMBAf8EAjAAMB0GA1UdDgQWBBQgKMbrpToy85W/G/a0fmc9C1BQmDAXBgNVHREE
EDAOggxtb2Vsb3ZlLmluZm8wCgYIKoZIzj0EAwIDSAAwRQIgQ3O8IgCv2Td5HaWM
pMKZdB/3WtC+xDeIWOlDz/hBw14CIQDLP4m0xZfJBoDg9pDhq8FtsyT7UeXUvVFA
lKKQxVM9qA==
-----END CERTIFICATE-----

tao@moelove:~$ helm repo add apisix https://charts.apiseven.com
tao@moelove:~$ helm repo add bitnami https://charts.bitnami.com/bitnami
tao@moelove:~$ helm repo update
tao@moelove:~$ helm install apisix apisix/apisix --set gateway.tls.enabled=true --set gateway.type=NodePort   --set ingress-controller.enabled=true   --set ingress-controller.config.apisix.serviceNamespace=apisix   --namespace apisix   --create-namespace   --set ingress-controller.config.apisix.serviceName=apisix-admin --set ingress-controller.config.ingressPublishService="apisix/apisix-gateway"
NAME: apisix
LAST DEPLOYED: Wed Oct 19 21:33:37 2022
NAMESPACE: apisix
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
1. Get the application URL by running these commands:
  export NODE_PORT=$(kubectl get --namespace apisix -o jsonpath="{.spec.ports[0].nodePort}" services apisix-gateway)
  export NODE_IP=$(kubectl get nodes --namespace apisix -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT

tao@moelove:~$ kubectl -n apisix get pods  
NAME                                         READY   STATUS    RESTARTS   AGE
apisix-777c9fdd67-rf8zs                      1/1     Running   0          6m48s
apisix-etcd-0                                1/1     Running   0          6m48s
apisix-etcd-1                                1/1     Running   0          6m48s
apisix-etcd-2                                1/1     Running   0          6m48s
apisix-ingress-controller-568544b554-k7nd4   1/1     Running   0          6m48s

tao@moelove:~$ helm repo add hashicorp https://helm.releases.hashicorp.com
tao@moelove:~$ helm install vault hashicorp/vault --set "injector.enabled=false" --set "server.dev.enabled=true"
NAME: vault
LAST DEPLOYED: Wed Oct 19 21:53:50 2022
NAMESPACE: default
STATUS: deployed
REVISION: 1
NOTES:
Thank you for installing HashiCorp Vault!

Now that you have deployed Vault, you should look over the docs on using
Vault with Kubernetes available here:

https://www.vaultproject.io/docs/


Your release is named "vault". To learn more about the release, try the following:

  $ helm status vault
  $ helm get manifest vault

tao@moelove:~$ kubectl get pods  
NAME      READY   STATUS    RESTARTS   AGE
vault-0   1/1     Running   0          29s
tao@moelove:~$ kubectl get svc
NAME             TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)             AGE
kubernetes       ClusterIP   10.96.0.1      <none>        443/TCP             84m
vault            ClusterIP   10.96.190.88   <none>        8200/TCP,8201/TCP   4m14s
vault-internal   ClusterIP   None           <none>        8200/TCP,8201/TCP   4m14s

tao@moelove:~$ kubectl  exec -it vault-0 -- sh
/ $ vault secrets enable pki
Success! Enabled the pki secrets engine at: pki/
/ $ vault write pki/root/generate/internal common_name=moelove.info ttl=8760h
Key              Value
---              -----
certificate      -----BEGIN CERTIFICATE-----
MIIDODCCAiCgAwIBAgIUds5uMJV9rOkwFEt6Xof5T2SVFccwDQYJKoZIhvcNAQEL
...
VM4DRVgDkqY9JdHU
-----END CERTIFICATE-----
expiration       1668983612
issuer_id        8df13015-7c70-df9a-7bb7-9b3b4afe7f82
issuer_name      n/a
issuing_ca       -----BEGIN CERTIFICATE-----
MIIDODCCAiCgAwIBAgIUds5uMJV9rOkwFEt6Xof5T2SVFccwDQYJKoZIhvcNAQEL
...
VM4DRVgDkqY9JdHU
-----END CERTIFICATE-----
key_id           c9fcfcb0-3548-a9a7-e706-30510592c797
key_name         n/a
serial_number    76:ce:6e:30:95:7d:ac:e9:30:14:4b:7a:5e:87:f9:4f:64:95:15:c7
/ $
/ $ vault write pki/config/urls issuing_certificates="http://vault.default:8200/v1/pki/ca" crl_distribution_points="http://vault.default:8200/v1/pki/crl"
Success! Data written to: pki/config/urls
/ $ vault write pki/roles/moelove-dot-info allowed_domains=moelove.info allow_subdomains=true max_ttl=72h
Success! Data written to: pki/roles/moelove-dot-info
/ $
/ $ vault policy write pki - <<EOF
> path "pki*"                        { capabilities = ["read", "list"] }
> path "pki/sign/moelove-dot-info"    { capabilities = ["create", "update"] }
> path "pki/issue/moelove-dot-info"   { capabilities = ["create"] }
> EOF
Success! Uploaded policy: pki

/ $ vault auth enable kubernetes
Success! Enabled kubernetes auth method at: kubernetes/
/ $ vault write auth/kubernetes/config kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"
Success! Data written to: auth/kubernetes/config
/ $ vault write auth/kubernetes/role/issuer  bound_service_account_names=issuer bound_service_account_namespaces=default policies=pki ttl=20m
Success! Data written to: auth/kubernetes/role/issuer

tao@moelove:~$ helm repo add jetstack https://charts.jetstack.io
tao@moelove:~$ helm repo update jetstack
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "jetstack" chart repository
Update Complete. ⎈Happy Helming!⎈
tao@moelove:~$ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.0/cert-manager.crds.yaml
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created
tao@moelove:~$ helm install \
>   cert-manager jetstack/cert-manager \
>   --namespace cert-manager \
>   --create-namespace \
>   --version v1.10.0

xNAME: cert-manager
LAST DEPLOYED: Wed Oct 19 22:51:06 2022
NAMESPACE: cert-manager
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
cert-manager v1.10.0 has been deployed successfully!

In order to begin issuing certificates, you will need to set up a ClusterIssuer
or Issuer resource (for example, by creating a 'letsencrypt-staging' issuer).

More information on the different types of issuers and how to configure them
can be found in our documentation:

https://cert-manager.io/docs/configuration/

For information on how to configure cert-manager to automatically provision
Certificates for Ingress resources, take a look at the `ingress-shim`
documentation:

https://cert-manager.io/docs/usage/ingress/

tao@moelove:~$ kubectl -n cert-manager get pods
NAME                                       READY   STATUS    RESTARTS   AGE
cert-manager-69b456d85c-znpq4              1/1     Running   0          117s
cert-manager-cainjector-5f44d58c4b-wcd27   1/1     Running   0          117s
cert-manager-webhook-566bd88f7b-7rptf      1/1     Running   0          117s

tao@moelove:~$ kubectl create serviceaccount issuer
serviceaccount/issuer created
tao@moelove:~$ kubectl get secret
NAME                          TYPE                 DATA   AGE
sh.helm.release.v1.vault.v1   helm.sh/release.v1   1      36m
tao@moelove:~$ vim issuer-secret.yaml
tao@moelove:~$ cat issuer-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: issuer-token-moelove
  annotations:
    kubernetes.io/service-account.name: issuer
type: kubernetes.io/service-account-token
tao@moelove:~$ kubectl apply -f issuer-secret.yaml
secret/issuer-token-moelove created
tao@moelove:~$ kubectl get sa,secret
NAME                     SECRETS   AGE
serviceaccount/default   0         118m
serviceaccount/issuer    0         2m11s
serviceaccount/vault     0         38m

NAME                                 TYPE                                  DATA   AGE
secret/issuer-token-moelove          kubernetes.io/service-account-token   3      35s
secret/sh.helm.release.v1.vault.v1   helm.sh/release.v1                    1      38m

tao@moelove:~$ cat vault-issuer.yaml
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: vault-issuer
  namespace: default
spec:
  vault:
    server: http://vault.default
    path: pki/sign/moelove-dot-info
    auth:
      kubernetes:
        mountPath: /v1/auth/kubernetes
        role: moelove-dot-info
        secretRef:
          name: issuer-token-moelove
          key: token
tao@moelove:~$ kubectl apply -f vault-issuer.yaml
issuer.cert-manager.io/vault-issuer created

tao@moelove:~$ cat moelove-dot-info-cert.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: moelove-info
  namespace: default
spec:
  secretName: moelove-info-tls
  issuerRef:
    name: vault-issuer
  commonName: www.moelove.info
  dnsNames:
  - www.moelove.info
tao@moelove:~$ kubectl apply -f moelove-dot-info-cert.yaml
certificate.cert-manager.io/moelove-info created

kubectl run httpbin --image kennethreitz/httpbin
kubectl expose pod httpbin --port=80

# تعريف كائنات ApisixTls
apiVersion: apisix.apache.org/v2
kind: ApisixTls
metadata:
  name: moelove
spec:
  hosts:
  - moelove.info
  secret:
    name: moelove-info-tls

---
# تعريف المسار للوصول إلى الخلفية
apiVersion: apisix.apache.org/v2
kind: ApisixRoute
metadata:
  name: moelove
spec:
  http:
  - name: httpbin
    match:
      paths:
      - /*
      hosts:
      - moelove.info
    backends:
    - serviceName: httpbin
      servicePort: 80

$ ~ kubectl port-forward -n ingress-apisix svc/apisix-gateway 8443:443 &
$ ~ curl -sk https://moelove.info:8443/ip --resolve 'moelove.info:8443:127.0.0.1'
{
  "origin": "172.17.18.1"
}