diff --git a/Makefile b/Makefile index 5b0e5e3..fef5521 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,3 @@ - - markdown-lint: # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md # https://github.com/markdownlint/markdownlint/blob/master/lib/mdl/rules.rb diff --git a/api-gateway/2-expose/README.md b/api-gateway/2-expose/README.md new file mode 100644 index 0000000..43b045f --- /dev/null +++ b/api-gateway/2-expose/README.md @@ -0,0 +1,983 @@ +# Expose + +As the entry point on your cluster, Traefik Hub API Gateway can be used to expose resources. + +We'll see in this tutorial: + +1. How to expose an external API +2. How to expose an internal website +3. How to configure HTTPS with CA certificates +4. How to configure HTTPS with an automated CA + +:information_source: To follow this tutorial, you must install Traefik Hub following the [getting started](../1-getting-started/README.md) instructions. + +## Expose an external API + +Traefik Hub is able to handle various API protocols and use cases. + +We will expose a GraphQL API to demonstrate that Traefik Hub can manage and secure not only REST APIs but also other types of APIs, such as GraphQL. + +We will use the well known [countries](https://github.com/trevorblades/countries) from Trevor Blades. + +First, let's see if we can query it: + +```sh +curl -s 'https://countries.trevorblades.com/' -X POST \ + -H 'content-type: application/json' \ + --data '{ "query": "{ continents { code name } }" }' | jq +``` + +```yaml +{ + "data": { + "continents": [ + { + "code": "AF", + "name": "Africa" + }, + { + "code": "AN", + "name": "Antarctica" + }, + { + "code": "AS", + "name": "Asia" + }, + { + "code": "EU", + "name": "Europe" + }, + { + "code": "NA", + "name": "North America" + }, + { + "code": "OC", + "name": "Oceania" + }, + { + "code": "SA", + "name": "South America" + } + ] + } +} +``` + +In Kubernetes, we can declare this endpoint as an ExternalName `Service`: + +```yaml :manifests/graphql-service.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: graphql + namespace: apps +spec: + type: ExternalName + externalName: countries.trevorblades.com + ports: + - port: 443 +``` + +Like any other `Service`, it can be exposed on our cluster using an `IngressRoute`: + +```yaml :manifests/graphql-ingressroute.yaml +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-graphql + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) && Path(`/graphql`) + kind: Rule + services: + - name: graphql + port: 443 +``` + +Let's apply it: + +```sh +kubectl apply -f src/manifests/apps-namespace.yaml +kubectl apply -f api-gateway/2-expose/manifests/graphql-service.yaml +kubectl apply -f api-gateway/2-expose/manifests/graphql-ingressroute.yaml +``` + +And test it: + +```sh +curl -s 'http://expose.apigateway.docker.localhost/graphql' -X POST \ + -H 'content-type: application/json' \ + --data '{ "query": "{ continents { code name } }" }' +``` + +```sh +404 page not found +``` + +It does not work :sob:. Why? +Let's check the Traefik Hub logs: + +```sh +kubectl logs -n traefik -l app.kubernetes.io/name=traefik +``` + +```sh +[...] +2024-10-09T08:02:53Z ERR error="externalName services not allowed: apps/graphql" ingress=expose-apigateway-graphql namespace=apps providerName=kubernetescrd +2024-10-09T08:03:42Z ERR error="externalName services not allowed: apps/graphql" ingress=expose-apigateway-graphql namespace=apps providerName=kubernetescrd +``` + +For security reasons, since 2021, Traefik has disabled this feature by default. See PR [#8261](https://github.com/traefik/traefik/pull/8261). + +We can enable it using helm: + +```sh +helm upgrade traefik -n traefik --wait \ + --version v33.0.0 \ + --reuse-values \ + --set providers.kubernetesCRD.allowExternalNameServices=true \ + traefik/traefik +``` + +Let's test it again: + +```sh +curl -s 'http://expose.apigateway.docker.localhost/graphql' -X POST \ + -H 'content-type: application/json' \ + --data '{ "query": "{ continents { code name } }" }' +``` + +```less +Requested host does not match any Subject Alternative Names (SANs) on TLS certificate [da140de2fae284c9221f5584e2c672f658a41dfb9b627354f6dedad6804fd9d4] in use with this connection. + +Visit https://docs.fastly.com/en/guides/common-400-errors#error-421-misdirected-request for more information. +``` + +There is another error about the Host. :sob: It can be somehow expected; +our `expose.apigateway.docker.localhost` domain is not known by this external service. +Luckily, we can configure the IngressRoute to use upstream domain for the requested host. + +Let's update the `IngressRoute`: + +```diff :../../hack/diff.sh -r -a "manifests/graphql-ingressroute.yaml manifests/graphql-ingressroute-complete.yaml" +--- manifests/graphql-ingressroute.yaml ++++ manifests/graphql-ingressroute-complete.yaml +@@ -13,3 +13,4 @@ + services: + - name: graphql + port: 443 ++ passHostHeader: false +``` + +Let's update the `IngressRoute` and :crossed_fingers: for this test: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/graphql-ingressroute-complete.yaml +sleep 1 +curl -s 'http://expose.apigateway.docker.localhost/graphql' -X POST \ + -H 'content-type: application/json' \ + --data '{ "query": "{ continents { code name } }" }' | jq +``` + +```yaml +{ + "data": { + "continents": [ + { + "code": "AF", + "name": "Africa" + }, + { + "code": "AN", + "name": "Antarctica" + }, + { + "code": "AS", + "name": "Asia" + }, + { + "code": "EU", + "name": "Europe" + }, + { + "code": "NA", + "name": "North America" + }, + { + "code": "OC", + "name": "Oceania" + }, + { + "code": "SA", + "name": "South America" + } + ] + } +} +``` + +It works! :tada: We received the expected answer from the ExternalName `Service`. + +Now, let's see how to expose a website. + +## Expose a website + +In this section, we will use the [wordsmith](https://github.com/dockersamples/wordsmith) sample project from dockersamples. + +This project comes with three containers (web, api, db) and expose a web page with dynamic and static content. + +Let's deploy it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-db.yaml +kubectl apply -f api-gateway/2-expose/manifests/webapp-api.yaml +kubectl apply -f api-gateway/2-expose/manifests/webapp-front.yaml +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute.yaml +kubectl wait -n apps --for=condition=ready pod --selector="app in (db,api,web)" +``` + +We can now access it on http://expose.apigateway.docker.localhost/ + +![Sample webapp](images/webapp.png) + +### Compress static text content + +This website is downloading some javascript file. When trying to download them, we can check if it's using compression easily with curl: + +```sh +curl -si -H 'Accept-Encoding: gzip, deflate, bz, zstd' http://expose.apigateway.docker.localhost/app.js | head -n 4 +``` + +```less +HTTP/1.1 200 OK +Accept-Ranges: bytes +Content-Length: 1183 +Content-Type: text/javascript; charset=utf-8 +[...] +``` + +There is no `Content-Encoding` header, which means there is no compression used. + +We can configure it to provide faster download with the Compress `Middleware`. + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute.yaml manifests/webapp-ingressroute-compress.yaml" +--- manifests/webapp-ingressroute.yaml ++++ manifests/webapp-ingressroute-compress.yaml +@@ -1,5 +1,23 @@ + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: compress ++ namespace: apps ++spec: ++ compress: ++ includedContentTypes: ++ - application/json ++ - application/xml ++ - text/css ++ - text/csv ++ - text/html ++ - text/javascript ++ - text/plain ++ - text/xml ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: + name: expose-apigateway-webapp +@@ -10,6 +28,8 @@ + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule ++ middlewares: ++ - name: compress + services: + - name: web + port: 80 +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-compress.yaml +``` + +And test it: + +```sh +curl -si -H 'Accept-Encoding: gzip, deflate, bz, zstd' http://expose.apigateway.docker.localhost/app.js | head -n 4 +``` + +```less +HTTP/1.1 200 OK +Accept-Ranges: bytes +Content-Encoding: zstd +Content-Type: text/javascript; charset=utf-8 +``` + +We can see here the `Content-Encoding` header, letting the recipient know how to decode the body. + +### Protect with Security Headers and CORS + +Let's see how to use [CORS](https://en.wikipedia.org/wiki/Cross-origin_resource_sharing) to indicate to the browser if an API call from a different origin is allowed. + +First, let's check if it's possible to call it from a different origin: + +```sh +curl -I -H "Origin: http://test2.com" http://expose.apigateway.docker.localhost/ +``` + +```less +HTTP/1.1 200 OK +Accept-Ranges: bytes +Content-Length: 1415 +Content-Type: text/html; charset=utf-8 +``` + +We can see there are no security or CORS headers. Now, let's protect our sample webapp with Security and CORS headers: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-compress.yaml manifests/webapp-ingressroute-cors.yaml" +--- manifests/webapp-ingressroute-compress.yaml ++++ manifests/webapp-ingressroute-cors.yaml +@@ -18,6 +18,27 @@ + + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: security-headers ++ namespace: apps ++spec: ++ headers: ++ frameDeny: true ++ browserXssFilter: true ++ contentTypeNosniff: true ++ # HSTS ++ stsIncludeSubdomains: true ++ stsPreload: true ++ stsSeconds: 31536000 ++ # CORS ++ accessControlAllowMethods: [ "GET", "OPTIONS" ] ++ accessControlAllowHeaders: [ "*" ] ++ accessControlAllowOriginList: [ "http://test.com" ] ++ accessControlMaxAge: 100 ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: + name: expose-apigateway-webapp +@@ -30,6 +51,7 @@ + kind: Rule + middlewares: + - name: compress ++ - name: security-headers + services: + - name: web + port: 80 +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-cors.yaml +``` + +And test it: + +```sh +curl -I -H "Origin: http://test.com" http://expose.apigateway.docker.localhost/ +``` + +```less +HTTP/1.1 200 OK +Accept-Ranges: bytes +Access-Control-Allow-Origin: http://test.com +Content-Length: 1415 +Content-Type: text/html; charset=utf-8 +Date: Wed, 09 Oct 2024 14:44:41 GMT +Last-Modified: Sun, 17 Sep 2023 22:27:38 GMT +X-Content-Type-Options: nosniff +X-Frame-Options: DENY +X-Xss-Protection: 1; mode=block +``` + +If we change the `Origin` in _curl_, we can see that `Access-Control-Allow-Origin` is absent, as expected. + +### Error page + +When a user tries to access a page which does not exist, the user will get the default 404 page. + +For instance, when we navigate to http://expose.apigateway.docker.localhost/doesnotexist, +we should see the following: + +![404 without error Middleware](images/404-without-error-mdw.png) + +On a company website, we'll want to display a custom page with the company logo and more human friendly content. + +There is an `Error` middleware in Traefik Hub, so let's try it! + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-cors.yaml manifests/webapp-ingressroute-error-page.yaml" +--- manifests/webapp-ingressroute-cors.yaml ++++ manifests/webapp-ingressroute-error-page.yaml +@@ -39,6 +39,22 @@ + + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: error-page ++ namespace: apps ++spec: ++ errors: ++ status: ++ - "404" ++ - "500-599" ++ query: '/{status}.html' ++ service: ++ name: error-page ++ port: "http" ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: + name: expose-apigateway-webapp +@@ -52,6 +68,7 @@ + middlewares: + - name: compress + - name: security-headers ++ - name: error-page + services: + - name: web + port: 80 +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/error-page.yaml +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-error-page.yaml +kubectl wait -n apps --for=condition=ready pod --selector="app=error-page" +``` + +And test it on http://expose.apigateway.docker.localhost/doesnotexist + +![404 with error Middleware](images/404-with-error-mdw.png) + +:information_source: We have used https://github.com/tarampampam/error-pages in this tutorial. There are many other sources that can be used. + +:information_source: One can also serve local error page files with the [waeb](https://plugins.traefik.io/plugins/646b2816a8eb44f98ba6a325/waeb) Yaegi plugin. + +### HTTP Caching + +After compression, we can further improve the performance of our sample web application using HTTP Caching. + +Traefik Hub API Gateway comes with an _HTTP Cache_ `Middleware` that follows the [RFC 7234](https://tools.ietf.org/html/rfc7234). + +Let's add one to our `IngressRoute`: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-error-page.yaml manifests/webapp-ingressroute-cache.yaml" +--- manifests/webapp-ingressroute-error-page.yaml ++++ manifests/webapp-ingressroute-cache.yaml +@@ -55,6 +55,19 @@ + + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: cache ++ namespace: apps ++spec: ++ plugin: ++ httpCache: ++ store: ++ memory: ++ limit: "50Mi" ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: + name: expose-apigateway-webapp +@@ -69,6 +82,7 @@ + - name: compress + - name: security-headers + - name: error-page ++ - name: cache + services: + - name: web + port: 80 +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-cache.yaml +``` + +And test it: + +```sh +curl -sI http://expose.apigateway.docker.localhost/ | grep --color "X-Cache-Status" +curl -sI http://expose.apigateway.docker.localhost/ | grep --color "X-Cache-Status" +``` + +```less +X-Cache-Status: MISS +X-Cache-Status: HIT +``` + +:information_source: The memory limit is set per middleware and is not global to the gateway. If you have multiple middlewares with different memory limits, they will add-up. + +:information_source: If you share the same middleware across multiple routers they will use the same limit. + +### Configure HTTPS + +To configure HTTPS for our sample web application, we can simply use the websecure entrypoint: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-cache.yaml manifests/webapp-ingressroute-https.yaml" +--- manifests/webapp-ingressroute-cache.yaml ++++ manifests/webapp-ingressroute-https.yaml +@@ -75,6 +75,7 @@ + spec: + entryPoints: + - web ++ - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule +``` + +We can apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-https.yaml +``` + +And test it: + +```sh +curl -sI https://expose.apigateway.docker.localhost/ -v +``` + +```less +* Trying 127.0.0.1:443... +* Connected to expose.apigateway.docker.localhost (127.0.0.1) port 443 (#0) +* ALPN: offers h2,http/1.1 +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +* CAfile: /etc/ssl/certs/ca-certificates.crt +* CApath: /etc/ssl/certs +* TLSv1.3 (IN), TLS handshake, Server hello (2): +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +* TLSv1.3 (IN), TLS handshake, Certificate (11): +* TLSv1.3 (OUT), TLS alert, unknown CA (560): +* SSL certificate problem: self-signed certificate +* Closing connection 0 +``` + +We can see here that we have an issue: it's a "self-signed" certificate. Let's dig and see where this certificate comes from: + +```sh +curl -k -sI https://expose.apigateway.docker.localhost/ -v 2>&1 | grep -i "Server certificate" -A 6 +``` + +```less +* Server certificate: +* subject: CN=TRAEFIK DEFAULT CERT +* start date: Oct 10 08:23:38 20XX GMT +* expire date: Oct 10 08:23:38 20XX GMT +* issuer: CN=TRAEFIK DEFAULT CERT +* SSL certificate verify result: self-signed certificate (18), continuing anyway. +``` + +So we can see here that this certificate is the `TRAEFIK DEFAULT CERT`. + +The HTTP access is still working: + +```sh +curl -sI http://expose.apigateway.docker.localhost/ | head -n 1 +``` + +```less +HTTP/1.1 200 OK +``` + +We are doing this tutorial on localhost. It's a private domain, reserved for local use. +Let's Encrypt documentation [explains this](https://letsencrypt.org/docs/certificates-for-localhost/): +> Let’s Encrypt can’t provide certificates for “localhost” because nobody uniquely owns it, +> and it’s not rooted in a top level domain like “.com” or “.net”. + +So how do we test the HTTPS middleware we added? + +First, let's try the _manual_ way, as if we bought a Certificate pair from a SSL provider, without any automation. + +#### TLS with CA certificates + +We will use [minica](https://github.com/jsha/minica) to generate a Certificate Authority (CA). We want to use this (self-signed) CA to generate certificate for the domain of our webapp: _expose.apigateway.docker.localhost_. + +We can either install it or create it ourselves: + +```sh +## Complete way, needs go +# go install github.com/jsha/minica@latest +# minica -domains expose.apigateway.docker.localhost +# kubectl create secret tls expose -n apps --cert expose.apigateway.docker.localhost/cert.pem --key expose.apigateway.docker.localhost/key.pem + +# Simpler way +kubectl apply -f src/minica/expose.apigateway.docker.localhost/expose.yaml +``` + +In the IngressRoute, it's just a matter of referencing this secret: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-https.yaml manifests/webapp-ingressroute-https-manual.yaml" +--- manifests/webapp-ingressroute-https.yaml ++++ manifests/webapp-ingressroute-https-manual.yaml +@@ -87,3 +87,5 @@ + services: + - name: web + port: 80 ++ tls: ++ secretName: expose +``` + +We can apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-https-manual.yaml +``` + +With curl, we can test it with this CA: + +```sh +curl --cacert src/minica/minica.pem -sI https://expose.apigateway.docker.localhost/ -v 2>&1 | grep -i "Server certificate" -A 6 +``` + +We can see that it has the expected fields: + +```less +* Server certificate: +* subject: CN=expose.apigateway.docker.localhost +* start date: Oct 14 08:50:48 20XX GMT +* expire date: Nov 13 09:50:48 20XX GMT +* subjectAltName: host "expose.apigateway.docker.localhost" matched cert's "expose.apigateway.docker.localhost" +* issuer: CN=minica root ca 2e84d8 +* SSL certificate verify ok. +``` + +When we test HTTP access, without TLS: + +```sh +curl -sI http://expose.apigateway.docker.localhost/ | head -n 1 +``` + +```less +HTTP/1.1 404 Not Found +``` + +It's not working :x: :sob: ! Let's check why. + +First, we can take a look at the [dashboard](http://dashboard.docker.localhost/dashboard/#/http/routers?search=webapp): + +![TLS on web entrypoint](images/tls-on-web.png) + +There is TLS on the _web_ entrypoint! We can confirm it with a TLS request on this port: + +```sh +curl --cacert src/minica/minica.pem -sI https://expose.apigateway.docker.localhost:80/ | head -n 1 +``` + +```less +HTTP/2 200 +``` + +Let's take a look at the whole IngressRoute again: + +```yaml :manifests/webapp-ingressroute-https-manual.yaml -s 75 -e 91 +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + tls: + secretName: expose +``` + +This IngressRoute is declared for both _web_ and _websecure_ entrypoints. +It means the `tls:` section is applied on both entrypoints. + +Let's check if we can fix it by splitting `IngressRoute` like this: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-https-manual.yaml manifests/webapp-ingressroute-https-split.yaml" +--- manifests/webapp-ingressroute-https-manual.yaml ++++ manifests/webapp-ingressroute-https-split.yaml +@@ -75,6 +75,26 @@ + spec: + entryPoints: + - web ++ routes: ++ - match: Host(`expose.apigateway.docker.localhost`) ++ kind: Rule ++ middlewares: ++ - name: compress ++ - name: security-headers ++ - name: error-page ++ - name: cache ++ services: ++ - name: web ++ port: 80 ++ ++--- ++apiVersion: traefik.io/v1alpha1 ++kind: IngressRoute ++metadata: ++ name: expose-apigateway-webapp-tls ++ namespace: apps ++spec: ++ entryPoints: + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-https-split.yaml +``` + +This time, the dashboard shows what we wanted: + +![TLS split](images/tls-split.png) + +Let's confirm it: + +```sh +# 200 OK on http +curl -sI http://expose.apigateway.docker.localhost/ | head -n 1 +# 200 OK on https +curl --cacert src/minica/minica.pem -sI https://expose.apigateway.docker.localhost/ | head -n 1 +``` + +We can now see how it works with automation, using [ACME](https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment). + +#### TLS with automation + +On localhost, we cannot use an automated system like Let's encrypt. Fortunately, Let's encrypt has developed a software named [pebble](https://github.com/letsencrypt/pebble). It provides a simplified ACME testing front end. + +We can use pebble to test certificate generation on localhost with minica: + +```sh +## With minica +# minica -domains pebble.pebble.svc +# kubectl create namespace pebble +# kubectl create secret tls pebble -n pebble --cert pebble.pebble.svc/cert.pem --key pebble.pebble.svc/key.pem + +## With kubectl +kubectl apply -f src/minica/pebble.pebble.svc/pebble.yaml +``` + +With this certificate, we can now deploy pebble: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/pebble.yaml +``` + +Pebble will receive request from Traefik on HTTPS, so we need to add minica to Traefik. + +Let's create a secret on both namespaces with the CA certificate: + +```sh +kubectl apply -f src/minica/minica.yaml +``` + +We'll need to hack the DNS of our local kubernetes cluster for pebble to resolve the DNS of our webapp on the Traefik service: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/coredns-config.yaml --server-side=true +kubectl rollout restart -n kube-system deployment/coredns +``` + +And configure Traefik Hub API Gateway to use it, with Kubernetes storage and Pebble CA: + +```sh +helm upgrade traefik -n traefik --wait \ + --version v33.0.0 \ + --reuse-values \ + --set certificatesResolvers.pebble.distributedAcme.caServer=https://pebble.pebble.svc:14000/dir \ + --set certificatesResolvers.pebble.distributedAcme.email=test@example.com \ + --set certificatesResolvers.pebble.distributedAcme.storage.kubernetes=true \ + --set certificatesResolvers.pebble.distributedAcme.tlsChallenge=true \ + --set "volumes[0].name=minica" \ + --set "volumes[0].mountPath=/minica" \ + --set "volumes[0].type=secret" \ + --set "env[0].name=LEGO_CA_CERTIFICATES" \ + --set "env[0].value=/minica/minica.pem" \ + traefik/traefik +``` + +Now that it's configured, we can configure this certificate resolver on our `IngressRoute`: + +```diff :../../hack/diff.sh -r -a "manifests/webapp-ingressroute-https-split.yaml manifests/webapp-ingressroute-https-auto.yaml" +--- manifests/webapp-ingressroute-https-split.yaml ++++ manifests/webapp-ingressroute-https-auto.yaml +@@ -108,4 +108,4 @@ + - name: web + port: 80 + tls: +- secretName: expose ++ certResolver: pebble +``` + +Let's apply it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/webapp-ingressroute-https-auto.yaml +``` + +The logs show the creation of certificates, and Traefik Hub manages two new Kubernetes `Secret`: one for the resolver account and one for the certs. + +```sh +# Check logs +kubectl logs -n traefik -l app.kubernetes.io/name=traefik -f +# Check secret +kubectl get secret -n traefik -l app.kubernetes.io/managed-by=traefik-hub +``` + +#### Expose HTTPS management port of pebble + +It looks good, but we haven't yet tested the whole TLS chain with curl. +We'll need to retrieve the CA of Root and Intermediate certificates before calling it. +They can be obtained from the management port of _pebble_ using only _https_. + +With Traefik Hub, we can configure the connection with the backend using `ServersTransport`. +We'll need to configure: + +1. The `scheme: https` +2. `serverName` should match the service FQDN in this case +3. `Secret` with the certificate of our CA + +```yaml :manifests/pebble-ingressroute.yaml +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: stripprefix + namespace: pebble +spec: + stripPrefix: + prefixes: + - /pebble + +--- +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: pebble + namespace: pebble +spec: + serverName: pebble.pebble.svc + rootCAsSecrets: + - minica + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-pebble-strip + namespace: pebble +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) && PathPrefix(`/pebble`) + kind: Rule + middlewares: + - name: stripprefix + services: + - name: pebble + port: mgt + scheme: https + serversTransport: pebble +``` + +Let's deploy it: + +```sh +kubectl apply -f api-gateway/2-expose/manifests/pebble-ingressroute.yaml +``` + +Once it's deployed, the certificate chain can be retrieved, and we can test our webapp entirely: + +```sh +# Get ROOT CA certificate from pebble +curl -s http://expose.apigateway.docker.localhost/pebble/roots/0 > chain.pem +# Get Intermediate CA certificate from pebble +curl -s http://expose.apigateway.docker.localhost/pebble/intermediates/0 >> chain.pem +# Check with complete TLS chain +curl --cacert chain.pem -sI https://expose.apigateway.docker.localhost/ | head -n 1 +``` + +### Check TLS quality + +In 2021, TLS v1.0 and TLS v1.1 were deprecated (see [RFC 8996](https://datatracker.ietf.org/doc/html/rfc8996)). + +First, let's check if we can still use TLS v1.0 and v1.1 on our https route: + +```sh +# Test TLS v1.0 +curl --cacert chain.pem -sI https://expose.apigateway.docker.localhost/ --tlsv1.0 --tls-max 1.0 -w "%{errormsg}\n" | head -n 1 +# Test TLS v1.1 +curl --cacert chain.pem -sI https://expose.apigateway.docker.localhost/ --tlsv1.1 --tls-max 1.1 -w "%{errormsg}\n" | head -n 1 +# Test TLS v1.2 +curl --cacert chain.pem -sI https://expose.apigateway.docker.localhost/ --tlsv1.2 --tls-max 1.2 | head -n 1 +# Test TLS v1.3 +curl --cacert chain.pem -sI https://expose.apigateway.docker.localhost/ --tlsv1.3 | head -n 1 +``` + +With Traefik Hub, we can see TLS v1.0 and v1.1 are disabled. It's secured by default. + +If you want to go further, you can use the famous [testss.sh](https://github.com/drwetter/testssl.sh) to check our setup. It can run on our deployment with this command: + +```sh +docker run -v ./chain.pem:/home/testssl/chain.pem --network host --rm -ti drwetter/testssl.sh --add-ca /home/testssl/chain.pem --ip 127.0.0.1 https://expose.apigateway.docker.localhost/ +``` + +```less +[...] + Rating (experimental) + + Rating specs (not complete) SSL Labs's 'SSL Server Rating Guide' (version 2009q from 2020-01-30) + Specification documentation https://github.com/ssllabs/research/wiki/SSL-Server-Rating-Guide + Protocol Support (weighted) 100 (30) + Key Exchange (weighted) 100 (30) + Cipher Strength (weighted) 90 (36) + Final Score 96 + Overall Grade A+ +``` + +That's it! we have concluded this tutorial on how to expose resources with Traefik Hub. We: + +* Exposed an endpoint outside of our cluster +* Configured a webside inside the cluster with Compression, CORS, Error Page and HTTP Caching +* Configured HTTPS with CA Certificates +* Configured HTTPS with ACME +* Checked TLS quality diff --git a/api-gateway/2-expose/images/404-with-error-mdw.png b/api-gateway/2-expose/images/404-with-error-mdw.png new file mode 100644 index 0000000..82c42a6 Binary files /dev/null and b/api-gateway/2-expose/images/404-with-error-mdw.png differ diff --git a/api-gateway/2-expose/images/404-without-error-mdw.png b/api-gateway/2-expose/images/404-without-error-mdw.png new file mode 100644 index 0000000..4c88c93 Binary files /dev/null and b/api-gateway/2-expose/images/404-without-error-mdw.png differ diff --git a/api-gateway/2-expose/images/tls-on-web.png b/api-gateway/2-expose/images/tls-on-web.png new file mode 100644 index 0000000..f69849d Binary files /dev/null and b/api-gateway/2-expose/images/tls-on-web.png differ diff --git a/api-gateway/2-expose/images/tls-split.png b/api-gateway/2-expose/images/tls-split.png new file mode 100644 index 0000000..5ead463 Binary files /dev/null and b/api-gateway/2-expose/images/tls-split.png differ diff --git a/api-gateway/2-expose/images/webapp.png b/api-gateway/2-expose/images/webapp.png new file mode 100644 index 0000000..2675bad Binary files /dev/null and b/api-gateway/2-expose/images/webapp.png differ diff --git a/api-gateway/2-expose/manifests/coredns-config.yaml b/api-gateway/2-expose/manifests/coredns-config.yaml new file mode 100644 index 0000000..ce3bf12 --- /dev/null +++ b/api-gateway/2-expose/manifests/coredns-config.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + expose.apigateway.server: | + expose.apigateway.docker.localhost { + log + rewrite name expose.apigateway.docker.localhost traefik.traefik.svc.cluster.local + forward . 127.0.0.1 + } diff --git a/api-gateway/2-expose/manifests/error-page.yaml b/api-gateway/2-expose/manifests/error-page.yaml new file mode 100644 index 0000000..5c475c8 --- /dev/null +++ b/api-gateway/2-expose/manifests/error-page.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: error-page + namespace: apps +spec: + replicas: 1 + selector: + matchLabels: + app: error-page + template: + metadata: + labels: + app: error-page + spec: + containers: + - name: error-page + image: ghcr.io/tarampampam/error-pages + ports: + - name: http + containerPort: 8080 + +--- +apiVersion: v1 +kind: Service +metadata: + name: error-page + namespace: apps +spec: + selector: + app: error-page + ports: + - name: http + port: 8080 + targetPort: http diff --git a/api-gateway/2-expose/manifests/graphql-ingressroute-complete.yaml b/api-gateway/2-expose/manifests/graphql-ingressroute-complete.yaml new file mode 100644 index 0000000..51a3949 --- /dev/null +++ b/api-gateway/2-expose/manifests/graphql-ingressroute-complete.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-graphql + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) && Path(`/graphql`) + kind: Rule + services: + - name: graphql + port: 443 + passHostHeader: false diff --git a/api-gateway/2-expose/manifests/graphql-ingressroute.yaml b/api-gateway/2-expose/manifests/graphql-ingressroute.yaml new file mode 100644 index 0000000..8b92195 --- /dev/null +++ b/api-gateway/2-expose/manifests/graphql-ingressroute.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-graphql + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) && Path(`/graphql`) + kind: Rule + services: + - name: graphql + port: 443 diff --git a/api-gateway/2-expose/manifests/graphql-service.yaml b/api-gateway/2-expose/manifests/graphql-service.yaml new file mode 100644 index 0000000..f5b264b --- /dev/null +++ b/api-gateway/2-expose/manifests/graphql-service.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: graphql + namespace: apps +spec: + type: ExternalName + externalName: countries.trevorblades.com + ports: + - port: 443 diff --git a/api-gateway/2-expose/manifests/pebble-ingressroute.yaml b/api-gateway/2-expose/manifests/pebble-ingressroute.yaml new file mode 100644 index 0000000..f49cb0d --- /dev/null +++ b/api-gateway/2-expose/manifests/pebble-ingressroute.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: stripprefix + namespace: pebble +spec: + stripPrefix: + prefixes: + - /pebble + +--- +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: pebble + namespace: pebble +spec: + serverName: pebble.pebble.svc + rootCAsSecrets: + - minica + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-pebble-strip + namespace: pebble +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) && PathPrefix(`/pebble`) + kind: Rule + middlewares: + - name: stripprefix + services: + - name: pebble + port: mgt + scheme: https + serversTransport: pebble diff --git a/api-gateway/2-expose/manifests/pebble.yaml b/api-gateway/2-expose/manifests/pebble.yaml new file mode 100644 index 0000000..3d6f3b2 --- /dev/null +++ b/api-gateway/2-expose/manifests/pebble.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config + namespace: pebble + +data: + config.json: | + { + "pebble": { + "listenAddress": "0.0.0.0:14000", + "managementListenAddress": "0.0.0.0:15000", + "certificate": "/certs/tls.crt", + "privateKey": "/certs/tls.key", + "httpPort": 80, + "tlsPort": 443, + "ocspResponderURL": "", + "externalAccountBindingRequired": false + } + } + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pebble + namespace: pebble +spec: + selector: + matchLabels: + app: pebble + template: + metadata: + labels: + app: pebble + spec: + containers: + - name: pebble + image: letsencrypt/pebble + env: + - name: PEBBLE_VA_NOSLEEP + value: "1" + command: + - pebble + - -config + - /config/config.json + ports: + - name: dir + containerPort: 14000 + - name: mgt + containerPort: 15000 + volumeMounts: + - mountPath: /config + name: config + - mountPath: /certs + name: certs + + volumes: + - name: certs + secret: + secretName: pebble + - name: config + configMap: + name: config + +--- +apiVersion: v1 +kind: Service +metadata: + name: pebble + namespace: pebble +spec: + type: ClusterIP + selector: + app: pebble + ports: + - port: 14000 + targetPort: dir + name: dir + - port: 15000 + targetPort: mgt + name: mgt diff --git a/api-gateway/2-expose/manifests/traefik-pebble-ca.yaml b/api-gateway/2-expose/manifests/traefik-pebble-ca.yaml new file mode 100644 index 0000000..1aad2e1 --- /dev/null +++ b/api-gateway/2-expose/manifests/traefik-pebble-ca.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: pebble-ca + namespace: traefik + +data: + ca.crt: | + LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURTekNDQWpPZ0F3SUJBZ0lJVmhyT3Jrby9pQ2t3RFFZSktvWklodmNOQVFFTEJRQXdJREVlTUJ3R0ExVUUKQXhNVmJXbHVhV05oSUhKdmIzUWdZMkVnTlRZeFlXTmxNQ0FYRFRJeE1Ea3lNakUzTlRVMU9Wb1lEekl4TWpFdwpPVEl5TVRjMU5UVTVXakFnTVI0d0hBWURWUVFERXhWdGFXNXBZMkVnY205dmRDQmpZU0ExTmpGaFkyVXdnZ0VpCk1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRDFCdEFHVHZ0eUtFL1NmczBHQ0lNUThkWUsKRzM4QUoyMXFibFhBSHZRNjNUeFJpNElNendubFRFeSsybnRza2ovdTNXSEJuRWZpWDV2cklydFNyd2JTSkhZSApvcXZjV0R4RjRTVUkzSlZLZndTSzM3UHlCaXRsZnRmS3VzaWozYzRxWmRaU1ZzdHEwWHp5elBKZTlzMDZsdkliCjRHWXRpOCt0MHBjV1JDck40V01Sc0NWdUhQSEVUNE5BbG9abVlhcGxSeVZKMkc1VXYvQ2I3d0lpN2NqUjRRNUEKd1ZzMTUvNFZHdTVDWlkxSjRKUkZJQllYMCtaQzVuR3dHMlZlZEJZY2dxZXN5ckxHMFdmM3FFMGo4cFFYOVRjcQptM0grTUZ4OEJ4blZuMHRSVVdNdm5BV3pEcXNkZG9aZ295K05tR2lFdTdCdW12djVGdG1zV3N1MHlNN05BZ01CCkFBR2pnWVl3Z1lNd0RnWURWUjBQQVFIL0JBUURBZ0tFTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3IKQmdFRkJRY0RBakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VBTUIwR0ExVWREZ1FXQkJRN0JCRmgwRFFOVnJ5Uwp5UEJQcUU1ZUtFRzdYREFmQmdOVkhTTUVHREFXZ0JRN0JCRmgwRFFOVnJ5U3lQQlBxRTVlS0VHN1hEQU5CZ2txCmhraUc5dzBCQVFzRkFBT0NBUUVBZGs3cnRMbi9TUGxLTHBvdElHY1lFdUxDbVZ5clpxTHJTRG05ejMxNHBDUGEKSld3T0FqbEN2MUpubjMyNExRa05yVlVrWE1SMjlndU1nWVJ4TTJIWU9LbkJxZHl2S3BiQy9mQktCdWdpbVR1aQpERHVBNnpIaUVRZHNBWGVOUVFVTk1KTmp5RmJsVFRDZG14Rmpoa2lpalBZQXRiZ1VSNE84SCtsQ0phcHRqT1JDCmpXQ0hNL2J4d0lwY3NlT0hvaXNPZWpkaG13dElvWEFWa2ozRGFVem93Z2NqbVNEczVKSGFReWJsWXd1WW9KczMKSk9WT2l3ZUVubXd6UVd4b2RCNllSQjQ1dUlqR2N6OFpsdTR0anMvQ2w0a2tnL0xQdDNFVUtHd2ZJRDZ5b0tRSApKQzVlSFR4YWRvaHh4VkhjL20xR2tBMmQvTlpSdnB5di9uamgrdGkzbGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== diff --git a/api-gateway/2-expose/manifests/webapp-api.yaml b/api-gateway/2-expose/manifests/webapp-api.yaml new file mode 100644 index 0000000..a37d0e0 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-api.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: apps + labels: + app: api +spec: + ports: + - port: 8080 + targetPort: 8080 + name: api + selector: + app: api + clusterIP: None +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: apps + labels: + app: api +spec: + replicas: 1 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + containers: + - name: api + image: dockersamples/wordsmith-api + ports: + - containerPort: 8080 + name: api diff --git a/api-gateway/2-expose/manifests/webapp-db.yaml b/api-gateway/2-expose/manifests/webapp-db.yaml new file mode 100644 index 0000000..20cf716 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-db.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: db + namespace: apps + labels: + app: db +spec: + ports: + - port: 5432 + targetPort: 5432 + name: db + selector: + app: db + clusterIP: None +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: db + namespace: apps + labels: + app: db +spec: + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - name: db + image: postgres:10.0-alpine + ports: + - containerPort: 5432 + name: db + volumeMounts: + - name: db-schema + mountPath: /docker-entrypoint-initdb.d + volumes: + - name: db-schema + configMap: + name: db-schema +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: db-schema + namespace: apps +data: + words.sql: | + CREATE TABLE nouns (word TEXT NOT NULL); + CREATE TABLE verbs (word TEXT NOT NULL); + CREATE TABLE adjectives (word TEXT NOT NULL); + + INSERT INTO nouns(word) VALUES + ('cloud'), + ('elephant'), + ('gø language'), + ('laptøp'), + ('cøntainer'), + ('micrø-service'), + ('turtle'), + ('whale'), + ('gøpher'), + ('møby døck'), + ('server'), + ('bicycle'), + ('viking'), + ('mermaid'), + ('fjørd'), + ('legø'), + ('flødebolle'), + ('smørrebrød'); + + INSERT INTO verbs(word) VALUES + ('will drink'), + ('smashes'), + ('smøkes'), + ('eats'), + ('walks tøwards'), + ('løves'), + ('helps'), + ('pushes'), + ('debugs'), + ('invites'), + ('hides'), + ('will ship'); + + INSERT INTO adjectives(word) VALUES + ('the exquisite'), + ('a pink'), + ('the røtten'), + ('a red'), + ('the serverless'), + ('a brøken'), + ('a shiny'), + ('the pretty'), + ('the impressive'), + ('an awesøme'), + ('the famøus'), + ('a gigantic'), + ('the gløriøus'), + ('the nørdic'), + ('the welcøming'), + ('the deliciøus'); diff --git a/api-gateway/2-expose/manifests/webapp-front.yaml b/api-gateway/2-expose/manifests/webapp-front.yaml new file mode 100644 index 0000000..6a79a20 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-front.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: web + namespace: apps + labels: + app: web +spec: + ports: + - port: 80 + targetPort: 80 + name: web + selector: + app: web +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: apps + labels: + app: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: dockersamples/wordsmith-web + ports: + - containerPort: 80 + name: web diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-cache.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-cache.yaml new file mode 100644 index 0000000..16c9dc7 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-cache.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cache + namespace: apps +spec: + plugin: + httpCache: + store: + memory: + limit: "50Mi" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-compress.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-compress.yaml new file mode 100644 index 0000000..efb6806 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-compress.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-cors.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-cors.yaml new file mode 100644 index 0000000..7bdd07e --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-cors.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-error-page.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-error-page.yaml new file mode 100644 index 0000000..b1528d5 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-error-page.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-https-auto.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-https-auto.yaml new file mode 100644 index 0000000..57dd0f7 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-https-auto.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cache + namespace: apps +spec: + plugin: + httpCache: + store: + memory: + limit: "50Mi" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp-tls + namespace: apps +spec: + entryPoints: + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + tls: + certResolver: pebble diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-https-manual.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-https-manual.yaml new file mode 100644 index 0000000..8ab149f --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-https-manual.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cache + namespace: apps +spec: + plugin: + httpCache: + store: + memory: + limit: "50Mi" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + tls: + secretName: expose diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-https-split.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-https-split.yaml new file mode 100644 index 0000000..d62b511 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-https-split.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cache + namespace: apps +spec: + plugin: + httpCache: + store: + memory: + limit: "50Mi" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp-tls + namespace: apps +spec: + entryPoints: + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 + tls: + secretName: expose diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute-https.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute-https.yaml new file mode 100644 index 0000000..fc3d05d --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute-https.yaml @@ -0,0 +1,89 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: compress + namespace: apps +spec: + compress: + includedContentTypes: + - application/json + - application/xml + - text/css + - text/csv + - text/html + - text/javascript + - text/plain + - text/xml + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: apps +spec: + headers: + frameDeny: true + browserXssFilter: true + contentTypeNosniff: true + # HSTS + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + # CORS + accessControlAllowMethods: [ "GET", "OPTIONS" ] + accessControlAllowHeaders: [ "*" ] + accessControlAllowOriginList: [ "http://test.com" ] + accessControlMaxAge: 100 + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: error-page + namespace: apps +spec: + errors: + status: + - "404" + - "500-599" + query: '/{status}.html' + service: + name: error-page + port: "http" + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cache + namespace: apps +spec: + plugin: + httpCache: + store: + memory: + limit: "50Mi" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + middlewares: + - name: compress + - name: security-headers + - name: error-page + - name: cache + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/manifests/webapp-ingressroute.yaml b/api-gateway/2-expose/manifests/webapp-ingressroute.yaml new file mode 100644 index 0000000..4f9c658 --- /dev/null +++ b/api-gateway/2-expose/manifests/webapp-ingressroute.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: expose-apigateway-webapp + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`expose.apigateway.docker.localhost`) + kind: Rule + services: + - name: web + port: 80 diff --git a/api-gateway/2-expose/pebble.crt b/api-gateway/2-expose/pebble.crt new file mode 100644 index 0000000..6c94378 --- /dev/null +++ b/api-gateway/2-expose/pebble.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPDCCAiSgAwIBAgIIahIq/0poFgIwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVbWluaWNhIHJvb3QgY2EgNTYxYWNlMB4XDTIxMDkyMjE3NTYwMFoXDTIzMTAy +MjE3NTYwMFowHDEaMBgGA1UEAxMRcGViYmxlLnBlYmJsZS5zdmMwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDJV6LhzUV4AigvjlBi87PNHnvKlWFTq/PB +DW+MZL4YMmzM7o4JbkCAIqxzFXvg8IpQMkdTa0PMAwbm/bAdud1WH0CvzP821V1v +jNLXUIarV+li6gOMgEo+M2NKU8i6l74KM8+tJ/G3LDV56FaxylRbJOkHzap20rKj +jLLv23YVC9pZDAocWpTvZ91wT9g2atl9qFuyM2/rBTtpOdbIFBUPwsROR+k5rExn +YXeIWVu//vu96dCR1Qw9SLJEuw8xwMB8nzkFNaXG1mdKA4FYOSLWtA+tFIy5A9uW +BigOepT7ukP60hWRmPS/Lt+U6O++vdddY24irbpq341d/Wbk0E/NAgMBAAGjfjB8 +MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw +DAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQ7BBFh0DQNVrySyPBPqE5eKEG7XDAc +BgNVHREEFTATghFwZWJibGUucGViYmxlLnN2YzANBgkqhkiG9w0BAQsFAAOCAQEA +a92qYKGgx0iPJlUobYJBlmGVRausAjjIfnO+ty4bTN1iDBBH5r8ptxTiq98Fw9/b +wVZ2KZBW2Ff+S838cZvzeRfCtBSBmXv6+9h3OBQGAVAxLpz1W5Rd426IHV4s52uV +PzN9kiFOjqpMsRBBPmKZXVESCsF6btx5G8zgoKoXtm0nN6vkxLaj5ZxdX6lcXvp/ +B14cyX5v8XwJ6uB5BNf9kH3udg4h0/oKnn+OMxTVi4vgWDTSuDkHHvtww+YEYMaf +LLSwh8lSdPu0/+P3lgd4/FUtskPQq9jt65uweUs4Sms5ou7K1cMFb5/7gcOwhffc +2vSQ26nDLB+Ib0ScXFdvqw== +-----END CERTIFICATE----- diff --git a/api-gateway/3-secure-applications/README.md b/api-gateway/3-secure-applications/README.md new file mode 100644 index 0000000..4455911 --- /dev/null +++ b/api-gateway/3-secure-applications/README.md @@ -0,0 +1,8 @@ +# Secure applications + +There are many ways and many flows to secure applications. + +This tutorial is not exhaustive. It covers some common use cases to secure applications: + +1. [Machine-to-Machine (M2M) with OAuth2 Client Credentials](./m2m.md) +2. [Web Application with OpenID Connect (OIDC)](./oidc.md) diff --git a/api-gateway/3-secure-applications/images/oauth2-decode.png b/api-gateway/3-secure-applications/images/oauth2-decode.png new file mode 100644 index 0000000..6d2cf83 Binary files /dev/null and b/api-gateway/3-secure-applications/images/oauth2-decode.png differ diff --git a/api-gateway/3-secure-applications/m2m.md b/api-gateway/3-secure-applications/m2m.md new file mode 100644 index 0000000..615c949 --- /dev/null +++ b/api-gateway/3-secure-applications/m2m.md @@ -0,0 +1,289 @@ +## Machine to Machine (M2M) authentication with OAuth2 Client Credentials + +The OAuth2 Client Credentials (defined in [OAuth 2.0 RFC 6749, section 4.4](https://tools.ietf.org/html/rfc6749#section-4.4)) protocol provides a way to secure delegated access between applications with a JWT access token. + +With Traefik Hub, the authentication can be done between applications on the app level or on the gateway level. + +:information_source: To follow this tutorial, you'll need to install Traefik Hub following [getting started](../1-getting-started/README.md) instructions. + +Now that Traefik Hub is deployed, let's see how it works on the app level, using OAuth 2 Client Credentials flow: + +```mermaid +--- +title: OAuth2 Client Credentials flow +--- +sequenceDiagram + actor app as M2M Application + participant apigw as Traefik Hub
API Gateway + participant idp as Authorization
Server + participant api as Application + autonumber + + app->>apigw: Authenticate with Application Credentials + apigw->>idp: Validate Application Credentials. Request access token + idp->>apigw: Issue access token + apigw->>api: Send request with access token in HTTP Header +``` + +In this tutorial, we will use [Ory Hydra](https://www.ory.sh/hydra/), an OAuth 2 and OpenID Connect server. We will also use a simple [login/consent app](https://github.com/jlevesy/hail-hydra) that always says yes. + +We can deploy it: + +```shell +kubectl apply -f src/manifests/apps-namespace.yaml +kubectl apply -f src/manifests/hydra.yaml +kubectl wait -n hydra --for=condition=ready pod --selector=app=hydra --timeout=90s +kubectl wait -n hydra --for=condition=ready pod --selector=app=consent --timeout=90s +kubectl wait -n hydra --for=condition=complete job/create-hydra-clients --timeout=90s +``` + +TraefikLabs has open-sourced a simple _whoami_ application displaying technical information about the request. + +First, let's deploy and expose it: + +```shell +kubectl apply -f src/manifests/whoami-app.yaml +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml +sleep 5 +``` + +```shell +deployment.apps/whoami created +service/whoami created +ingressroute.traefik.io/secure-applications-apigateway-no-auth created +``` + +It should be accessible with curl on http://secure-applications.apigateway.docker.localhost/no-auth + +```shell +curl http://secure-applications.apigateway.docker.localhost/no-auth +``` + +```shell +Hostname: whoami-6f57d5d6b5-bgmfl +IP: 127.0.0.1 +IP: ::1 +IP: 10.42.0.10 +IP: fe80::c8f6:84ff:fe66:3158 +RemoteAddr: 10.42.0.6:38110 +GET /no-auth HTTP/1.1 +Host: secure-applications.apigateway.docker.localhost +User-Agent: curl/8.5.0 +Accept: */* +Accept-Encoding: gzip +X-Forwarded-For: 10.42.0.1 +X-Forwarded-Host: secure-applications.apigateway.docker.localhost +X-Forwarded-Port: 80 +X-Forwarded-Proto: http +X-Forwarded-Server: traefik-hub-6f5bbd6568-rp882 +X-Real-Ip: 10.42.0.1 +``` + +To secure it with OAuth2, we can use the OAuth2 Client Credentials middleware: + +```diff :../../hack/diff.sh -r -a "manifests/whoami-app-ingressroute.yaml manifests/whoami-app-oauth2-client-creds.yaml" +--- manifests/whoami-app-ingressroute.yaml ++++ manifests/whoami-app-oauth2-client-creds.yaml +@@ -1,15 +1,32 @@ + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: oauth2-client-creds ++ namespace: apps ++spec: ++ plugin: ++ oAuthClientCredentials: ++ url: http://hydra.hydra.svc:4444/oauth2/token ++ audience: https://traefik.io ++ usernameClaim: sub ++ forwardHeaders: ++ Sub: sub ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: +- name: secure-applications-apigateway-no-auth ++ name: secure-applications-apigateway-oauth2-client-credentials + namespace: apps + spec: + entryPoints: + - web + routes: +- - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/no-auth`) ++ - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/oauth2-client-credentials`) + kind: Rule + services: + - name: whoami + port: 80 ++ middlewares: ++ - name: oauth2-client-creds +``` + +We can deploy the secured `IngressRoute`: + +```shell +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml +``` + +```shell +middleware.traefik.io/oauth2-client-creds created +ingressroute.traefik.io/secure-applications-apigateway-oauth2-client-credentials created +``` + +Once it's ready, we can create a Hydra OAuth2 client with _client_credentials_ grant type. +This step is automated for you in this tutorial but here is how it's created: + +```shell :../../src/manifests/hydra.yaml -s 359 -e 366 -i s1 +hydra create oauth2-client \ + --endpoint http://hydra.hydra.svc:4445 \ + --name oauth-client \ + --secret traefiklabs \ + --grant-type client_credentials \ + --audience https://traefik.io \ + --token-endpoint-auth-method client_secret_post \ + --format json > /data/oauth-client.json +``` + +If needed, it can be run manually with `kubectl exec -it -n hydra deploy/hydra -- hydra create ...`. + +```shell +client_id=$(kubectl get secrets -n apps oauth-client -o json | jq -r '.data.client_id' | base64 -d -w 0) +client_secret=$(kubectl get secrets -n apps oauth-client -o json | jq -r '.data.client_secret' | base64 -d -w 0) +auth=$(echo -n "$client_id:$client_secret" | base64 -w 0) +curl -H "Authorization: Basic $auth" http://secure-applications.apigateway.docker.localhost/oauth2-client-credentials +``` + +It should output something like this: + +```shell +Hostname: whoami-6f57d5d6b5-bgmfl +IP: 127.0.0.1 +IP: ::1 +IP: 10.42.0.10 +IP: fe80::c8f6:84ff:fe66:3158 +RemoteAddr: 10.42.0.6:37870 +GET /oauth2-client-credentials HTTP/1.1 +Host: secure-applications.apigateway.docker.localhost +User-Agent: curl/8.5.0 +Accept: */* +Accept-Encoding: gzip +Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjU2ZDdkMzliLTdhNTUtNDFkZi1iNjZkLTRjYzU0YmQ1YmZmOCIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiaHR0cHM6Ly90cmFlZmlrLmlvIl0sImNsaWVudF9pZCI6ImZlZTU2Nzc1LWJhYjMtNDE1Mi1hZDM3LWUyMTE0YWRlNjQ0OSIsImV4cCI6MTcxODgxNzQ5MiwiZXh0Ijp7fSwiaWF0IjoxNzE4ODEzODkyLCJpc3MiOiJodHRwOi8vaHlkcmEuaHlkcmEuc3ZjOjQ0NDQiLCJqdGkiOiI3Y2NmNzAxNC04OWM0LTQ1OTYtYWFjNS0wNTdlZjIzYjVjNjkiLCJuYmYiOjE3MTg4MTM4OTIsInNjcCI6W10sInN1YiI6ImZlZTU2Nzc1LWJhYjMtNDE1Mi1hZDM3LWUyMTE0YWRlNjQ0OSJ9.AwT-3_XQvScKzcfK-HumGZn9AfD9BofzfMxraT4Nmvb7OPamkPwhn6i_hwYtvcxth0TUx6W4gziMX867rw3jPS_KZPeq33GYWkIlmVJbmE90jWcST7MOm5_Pl-KfmV9YKioWD1RFGGM3rkIrobmtH1JM3Oxbxi5bbcPOrdFGlpIiAst5V6LC8e93vwga9mvh86TCT7ZnaxVHNN5Rrz_KdkCnidpUcc5Vev1GlTGOyhK4uolqu7fyQiyckeSNGB_BLB-bk1JBPEApgPWRjKIXLwhmR-xg4WXFl3kWY4nBI7ECmbClMMCfpXa1zYWF_kjDHodWxL7n7dEsrsZuykXRTKT10iT2VxH7QrPS-lHOu_sg6svCCGObB_lkv0rHBP6P9K-nD3tkJi_uPbVEIFkzjxDe6CEIZ2Xrn8H1GVmig-NoNGpflMYVu41wb_6eRv-PPACD_GI-YQOQvpMJXPFIjMIUMmvIWg-vD0bzrd_YUipa0HfruP_ENnHeIXwhJBMCWeVNwGCslsSj8uO7KaTF1NrTDuIHZNBnAp2WxZFQ4RPC5O1T3TJOpPn7dj5PKN-XgUGQlmUCUGPazfvFFBFQymoDSL88ijXDTkbzYGD2TnrbcqyCWV6uJdyEoV1Q8OA_lGtN39XcUuyiMGavSGdQY5yyBoULXo8oAwprOIdAy68 +Sub: fee56775-bab3-4152-ad37-e2114ade6449 +X-Forwarded-For: 10.42.0.1 +X-Forwarded-Host: secure-applications.apigateway.docker.localhost +X-Forwarded-Port: 80 +X-Forwarded-Proto: http +X-Forwarded-Server: traefik-hub-6f5bbd6568-rp882 +X-Real-Ip: 10.42.0.1 +``` + +As we can see: + +1. A JWT Access Token is in the bearer HTTP header +2. The `Sub` header of the authentication has been added to the request. + +We can decode this JWT token easily on https://jwt.io + +![Decode JWT Access Token](./images/oauth2-decode.png) + +There is another mode allowed with this middleware, where we set _ClientId_ and _ClientSecret_ directly on Traefik Hub. Let's try it out. + +```mermaid +--- +title: OAuth2 Client Credentials flow with authentication on Traefik Hub +--- +sequenceDiagram + actor app as M2M Application + participant apigw as Traefik Hub
API Gateway + participant idp as Authorization
Server + participant api as Application + autonumber + + app->>apigw: Send request + apigw->>idp: Validate request. Request access token using Traefik Hub credentials + idp->>apigw: Issue access token + apigw->>api: Send request with access token in HTTP Header +``` + +We will remove the forward headers block and set the client's credentials directly in the middleware. +Another client has been automatically created for that purpose. Let's see how: + +```shell :../../src/manifests/hydra.yaml -s 368 -e 374 -i s1 +hydra create oauth2-client \ + --endpoint http://hydra.hydra.svc:4445 \ + --name oauth-client-nologin \ + --secret traefiklabs \ + --grant-type client_credentials \ + --audience https://traefik.io \ + --format json > /data/oauth-client-nologin.json +``` + +```diff :../../hack/diff.sh -r -a "manifests/whoami-app-oauth2.yaml manifests/whoami-app-oauth2-client-creds-nologin.yaml" +--- manifests/whoami-app-oauth2.yaml ++++ manifests/whoami-app-oauth2-client-creds-nologin.yaml +@@ -0,0 +1,31 @@ ++--- ++apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: oauth2-client-creds-nologin ++ namespace: apps ++spec: ++ plugin: ++ oAuthClientCredentials: ++ url: http://hydra.hydra.svc:4444/oauth2/token ++ audience: https://traefik.io ++ clientId: "urn:k8s:secret:oauth-client-nologin:client_id" ++ clientSecret: "urn:k8s:secret:oauth-client-nologin:client_secret" ++ ++--- ++apiVersion: traefik.io/v1alpha1 ++kind: IngressRoute ++metadata: ++ name: secure-applications-apigateway-oauth2-client-credentials-nologin ++ namespace: apps ++spec: ++ entryPoints: ++ - web ++ routes: ++ - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/oauth2-client-credentials-nologin`) ++ kind: Rule ++ services: ++ - name: whoami ++ port: 80 ++ middlewares: ++ - name: oauth2-client-creds-nologin +``` + +Let's try it: + +```shell +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml +sleep 3 +curl http://secure-applications.apigateway.docker.localhost/oauth2-client-credentials-nologin +``` + +As we can see, there is no authentication required now _and_ there is a JWT access token transmitted to the application: + +```shell +Hostname: whoami-6f57d5d6b5-bgmfl +IP: 127.0.0.1 +IP: ::1 +IP: 10.42.0.10 +IP: fe80::c8f6:84ff:fe66:3158 +RemoteAddr: 10.42.0.6:50004 +GET /oauth2-client-credentials-nologin HTTP/1.1 +Host: secure-applications.apigateway.docker.localhost +User-Agent: curl/8.5.0 +Accept: */* +Accept-Encoding: gzip +Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjU2ZDdkMzliLTdhNTUtNDFkZi1iNjZkLTRjYzU0YmQ1YmZmOCIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiaHR0cHM6Ly90cmFlZmlrLmlvIl0sImNsaWVudF9pZCI6IjNkMzYwNGViLWNkOTItNDAxMi05ODlhLTAxMTY3Mjc3YzRmMSIsImV4cCI6MTcxODgxNzYyMiwiZXh0Ijp7fSwiaWF0IjoxNzE4ODE0MDIyLCJpc3MiOiJodHRwOi8vaHlkcmEuaHlkcmEuc3ZjOjQ0NDQiLCJqdGkiOiJmYTcwZDBlMy1hM2E4LTQ4ZDYtYmNmNi1mMGYyZDEwY2ZkYTUiLCJuYmYiOjE3MTg4MTQwMjIsInNjcCI6W10sInN1YiI6IjNkMzYwNGViLWNkOTItNDAxMi05ODlhLTAxMTY3Mjc3YzRmMSJ9.fVM8ba7cSLBC-2J5t72sp8bzbZF5hGZRvaiWsKeJQxqcslQrB05nzZHhKvkg5Bqsw-0wycFVvYuc_1DWRai0jE7mG73vgiTuuBe38NaAh4hVt9vTORzvKtm_V1Wx4ZAtniGB-6o5ta6TyLc68tq78KHUeWTzZ0f5ugteZVmrflvBgWQWWjM612op2nbA9m6ArDurWRst6FAvJ_lSja60XHjmHSan78Q9ps7PzH1PB2zkmrsaI6Y-c81CtMR6KOdBmO0iD4eRoHGh2GP0iCiozv9r_8pLc7xkdFBDeFoswGIVRQhqEvOLchE5Ca7DnI6PQupX8NtXRrPY-blS8d-WT4UwHVUOc_nEQHhZuIZk3IG7iE6JMmtc_0dOWdBlu5m-XVHe1mC5XSb55McuY0ckp2mb2pbPgQJCcra67prcqQqpXZc0syOCTlvjk6mHBXYMmiISIsunGttKmNuZKFteUiaPsqgRJf_B2JQvG4RknEk1Nl5VKr8ouneP0xunSCRCZyGScZ_qt5XPbhBqLOA4dedATWqtQ7UT8hp5TWOmE0_1bZM3CKSOQeX3aPQWA6NFsJHKxY2IZsohes_QXACCI2qfHa5CVueRzgBYPkECtWC1pXhBWy4E5p4Jp-2mMBvEpTjltcprndfaGbFHxOw1nj1p24ESES7P9IUK73DQca8 +X-Forwarded-For: 10.42.0.1 +X-Forwarded-Host: secure-applications.apigateway.docker.localhost +X-Forwarded-Port: 80 +X-Forwarded-Proto: http +X-Forwarded-Server: traefik-hub-6f5bbd6568-rp882 +X-Real-Ip: 10.42.0.1 +``` diff --git a/api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml b/api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml new file mode 100644 index 0000000..9c27b93 --- /dev/null +++ b/api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: secure-applications-apigateway-no-auth + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/no-auth`) + kind: Rule + services: + - name: whoami + port: 80 diff --git a/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml b/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml new file mode 100644 index 0000000..a78f4bf --- /dev/null +++ b/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: oauth2-client-creds-nologin + namespace: apps +spec: + plugin: + oAuthClientCredentials: + url: http://hydra.hydra.svc:4444/oauth2/token + audience: https://traefik.io + clientId: "urn:k8s:secret:oauth-client-nologin:client_id" + clientSecret: "urn:k8s:secret:oauth-client-nologin:client_secret" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: secure-applications-apigateway-oauth2-client-credentials-nologin + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/oauth2-client-credentials-nologin`) + kind: Rule + services: + - name: whoami + port: 80 + middlewares: + - name: oauth2-client-creds-nologin diff --git a/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml b/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml new file mode 100644 index 0000000..146e081 --- /dev/null +++ b/api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: oauth2-client-creds + namespace: apps +spec: + plugin: + oAuthClientCredentials: + url: http://hydra.hydra.svc:4444/oauth2/token + audience: https://traefik.io + usernameClaim: sub + forwardHeaders: + Sub: sub + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: secure-applications-apigateway-oauth2-client-credentials + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/oauth2-client-credentials`) + kind: Rule + services: + - name: whoami + port: 80 + middlewares: + - name: oauth2-client-creds diff --git a/api-gateway/3-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml b/api-gateway/3-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml new file mode 100644 index 0000000..0a1b388 --- /dev/null +++ b/api-gateway/3-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: oidc-nologin + namespace: apps +spec: + plugin: + oidc: + issuer: http://hydra.hydra.svc:4444 + clientId: "urn:k8s:secret:oidc-client-nologin:client_id" + clientSecret: "urn:k8s:secret:oidc-client-nologin:client_secret" + logoutUrl: /oidc-nologin/logout + redirectUrl: /oidc-nologin/callback + csrf: {} + session: + name: "oidc-session" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: secure-applications-apigateway-whoami-nologin + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`secure-applications.apigateway.docker.localhost`) && (Path(`/oidc-nologin`) || Path(`/oidc-nologin/logout`) || Path(`/oidc-nologin/callback`)) + kind: Rule + services: + - name: whoami + port: 80 + middlewares: + - name: oidc-nologin diff --git a/api-gateway/3-secure-applications/manifests/whoami-app-oidc.yaml b/api-gateway/3-secure-applications/manifests/whoami-app-oidc.yaml new file mode 100644 index 0000000..92da2fb --- /dev/null +++ b/api-gateway/3-secure-applications/manifests/whoami-app-oidc.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: oidc-login + namespace: apps +spec: + plugin: + oidc: + issuer: http://hydra.hydra.svc:4444 + clientId: "urn:k8s:secret:oidc-client:client_id" + clientSecret: "urn:k8s:secret:oidc-client:client_secret" + loginUrl: /oidc/login + logoutUrl: /oidc/logout + redirectUrl: /oidc/callback + csrf: {} + session: + name: "oidc-session" + +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: secure-applications-apigateway-whoami-oidc + namespace: apps +spec: + entryPoints: + - web + routes: + - match: Host(`secure-applications.apigateway.docker.localhost`) && (Path(`/oidc`) || Path(`/oidc/login`) || Path(`/oidc/logout`) || Path(`/oidc/callback`)) + kind: Rule + services: + - name: whoami + port: 80 + middlewares: + - name: oidc-login diff --git a/api-gateway/3-secure-applications/oidc.md b/api-gateway/3-secure-applications/oidc.md new file mode 100644 index 0000000..418d195 --- /dev/null +++ b/api-gateway/3-secure-applications/oidc.md @@ -0,0 +1,263 @@ +## Human authentication with OpenID Connect Authentication + +OpenID Connect Authentication is built on top of the OAuth2 Authorization Code Flow (defined in [OAuth 2.0 RFC 6749, section 4.1](https://tools.ietf.org/html/rfc6749#section-4.1)) +and allows an application to be secured by delegating authentication to an external provider (Google Accounts, LinkedIn, GitHub, etc.) +and obtaining the end user's session claims and scopes for authorization purposes. + +:information_source: To follow this tutorial, you'll need to install Traefik Hub following [getting started](../1-getting-started/README.md) instructions. + +Now that Traefik Hub is deployed, let's see how it works: + +```mermaid +--- +title: OpenID Connect authentication flow +--- +sequenceDiagram + actor user as User + participant idp as OpenID Connect
Provider + participant apigw as Traefik Hub
API Gateway + participant app as Application + autonumber + + user->>apigw: User sends unauthenticated request + apigw->>user: Request denied. Redirect to OpenID Connect Provider + user->>idp: Login to OpenID Connect Provider + idp->>user: Issue authorization code. Redirect to Traefik Hub + user->>apigw: Send authorization code + apigw->>idp: Check authorization code. Request tokens + idp->>apigw: Issue access and identity tokens + apigw->>user: Set session using tokens. Redirect to Traefik Hub + user->>apigw: Re-send request with this session + apigw->>app: Check session validity. Relaying request to application + app->>user: Return response to user +``` + +In this tutorial, we will use [Ory Hydra](https://www.ory.sh/hydra/), an OAuth 2 and OpenID Connect server. We will also use a simple [login/consent app](https://github.com/jlevesy/hail-hydra) that always says yes. + +We can deploy it: + +```shell +kubectl apply -f src/manifests/apps-namespace.yaml +kubectl apply -f src/manifests/hydra.yaml +kubectl wait -n hydra --for=condition=ready pod --selector=app=hydra --timeout=90s +kubectl wait -n hydra --for=condition=ready pod --selector=app=consent --timeout=90s +kubectl wait -n hydra --for=condition=complete job/create-hydra-clients --timeout=90s +``` + +TraefikLabs has open-sourced a simple whoami application displaying technical information about the request. + +First, let's deploy and expose it: + +```shell +kubectl apply -f src/manifests/whoami-app.yaml +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml +sleep 5 +``` + +```shell +namespace/apps unchanged +deployment.apps/whoami unchanged +service/whoami unchanged +ingressroute.traefik.io/secure-applications-apigateway-no-auth unchanged +``` + +It should be accessible with curl on http://secure-applications.apigateway.docker.localhost/no-auth + +```shell +curl http://secure-applications.apigateway.docker.localhost/no-auth +``` + +```shell +Hostname: whoami-6f57d5d6b5-bgmfl +IP: 127.0.0.1 +IP: ::1 +IP: 10.42.0.10 +IP: fe80::c8f6:84ff:fe66:3158 +RemoteAddr: 10.42.0.6:52926 +GET /no-auth HTTP/1.1 +Host: secure-applications.apigateway.docker.localhost +User-Agent: curl/8.5.0 +Accept: */* +Accept-Encoding: gzip +X-Forwarded-For: 10.42.0.1 +X-Forwarded-Host: secure-applications.apigateway.docker.localhost +X-Forwarded-Port: 80 +X-Forwarded-Proto: http +X-Forwarded-Server: traefik-hub-6f5bbd6568-rp882 +X-Real-Ip: 10.42.0.1 +``` + +To secure it with OIDC, we'll need to configure Hydra. + +First, we can create the [JSON Web Key Sets](https://www.ory.sh/docs/hydra/jwks) that hydra uses to sign and verify id-token and access-token. +This step is automated for you in this tutorial but here is how it's created: + +```shell :../../src/manifests/hydra.yaml -s 398 -e 399 -i s1 +hydra create jwks hydra.openid.id-token --alg RS256 --endpoint http://hydra.hydra.svc:4445 +hydra create jwks hydra.jwt.access-token --alg RS256 --endpoint http://hydra.hydra.svc:4445 +``` + +If needed, it can be run manually with `kubectl exec -it -n hydra deploy/hydra -- hydra create ...`. + +And after, we can use the OIDC middleware. Let's see how it works compared to an unprotected IngressRoute: + +```diff :../../hack/diff.sh -r -a "manifests/whoami-app-ingressroute.yaml manifests/whoami-app-oidc.yaml" +--- manifests/whoami-app-ingressroute.yaml ++++ manifests/whoami-app-oidc.yaml +@@ -1,15 +1,36 @@ + --- + apiVersion: traefik.io/v1alpha1 ++kind: Middleware ++metadata: ++ name: oidc-login ++ namespace: apps ++spec: ++ plugin: ++ oidc: ++ issuer: http://hydra.hydra.svc:4444 ++ clientId: "urn:k8s:secret:oidc-client:client_id" ++ clientSecret: "urn:k8s:secret:oidc-client:client_secret" ++ loginUrl: /oidc/login ++ logoutUrl: /oidc/logout ++ redirectUrl: /oidc/callback ++ csrf: {} ++ session: ++ name: "oidc-session" ++ ++--- ++apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: +- name: secure-applications-apigateway-no-auth ++ name: secure-applications-apigateway-whoami-oidc + namespace: apps + spec: + entryPoints: + - web + routes: +- - match: Host(`secure-applications.apigateway.docker.localhost`) && Path(`/no-auth`) ++ - match: Host(`secure-applications.apigateway.docker.localhost`) && (Path(`/oidc`) || Path(`/oidc/login`) || Path(`/oidc/logout`) || Path(`/oidc/callback`)) + kind: Rule + services: + - name: whoami + port: 80 ++ middlewares: ++ - name: oidc-login +``` + +This middleware is configured to redirect `/login`, `/logout`, and `/callback` paths to the OIDC provider. + +We'll use another user in Hydra for that purpose and set it to the OIDC middleware. +It has been created thise way: + +```shell :../../src/manifests/hydra.yaml -s 376 -e 385 -i s1 +hydra create oauth2-client \ + --endpoint http://hydra.hydra.svc:4445 \ + --name oidc-client \ + --secret traefiklabs \ + --grant-type authorization_code,refresh_token \ + --response-type code,id_token \ + --scope openid,offline \ + --redirect-uri http://secure-applications.apigateway.docker.localhost/oidc/callback \ + --post-logout-callback http://secure-applications.apigateway.docker.localhost/oidc/callback \ + --format json > /data/oidc-client.json +``` + +So let's apply it: + +```shell +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-oidc.yaml +``` + +Let's test it: + +```shell +# Protected with OIDC => 401 +curl -I http://secure-applications.apigateway.docker.localhost/oidc +# Let's login and follow the request flow => 204 +rm -f /tmp/cookie +curl -i -L -b /tmp/cookie -c /tmp/cookie http://secure-applications.apigateway.docker.localhost/oidc/login +# Now, with this cookie, we can access => 200 +curl -b /tmp/cookie -c /tmp/cookie http://secure-applications.apigateway.docker.localhost/oidc +``` + +Now, let's say we want the user to log in on the whole domain. This YAML should do the trick: + +```diff :../../hack/diff.sh -r -a "manifests/whoami-app-oidc.yaml manifests/whoami-app-oidc-nologinurl.yaml" +--- manifests/whoami-app-oidc.yaml ++++ manifests/whoami-app-oidc-nologinurl.yaml +@@ -2,17 +2,16 @@ + apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: +- name: oidc-login ++ name: oidc-nologin + namespace: apps + spec: + plugin: + oidc: + issuer: http://hydra.hydra.svc:4444 +- clientId: "urn:k8s:secret:oidc-client:client_id" +- clientSecret: "urn:k8s:secret:oidc-client:client_secret" +- loginUrl: /oidc/login +- logoutUrl: /oidc/logout +- redirectUrl: /oidc/callback ++ clientId: "urn:k8s:secret:oidc-client-nologin:client_id" ++ clientSecret: "urn:k8s:secret:oidc-client-nologin:client_secret" ++ logoutUrl: /oidc-nologin/logout ++ redirectUrl: /oidc-nologin/callback + csrf: {} + session: + name: "oidc-session" +@@ -21,16 +20,16 @@ + apiVersion: traefik.io/v1alpha1 + kind: IngressRoute + metadata: +- name: secure-applications-apigateway-whoami-oidc ++ name: secure-applications-apigateway-whoami-nologin + namespace: apps + spec: + entryPoints: + - web + routes: +- - match: Host(`secure-applications.apigateway.docker.localhost`) && (Path(`/oidc`) || Path(`/oidc/login`) || Path(`/oidc/logout`) || Path(`/oidc/callback`)) ++ - match: Host(`secure-applications.apigateway.docker.localhost`) && (Path(`/oidc-nologin`) || Path(`/oidc-nologin/logout`) || Path(`/oidc-nologin/callback`)) + kind: Rule + services: + - name: whoami + port: 80 + middlewares: +- - name: oidc-login ++ - name: oidc-nologin +``` + +A new ClientID / ClientSecret pair has been created for that purpose: + +```shell :../../src/manifests/hydra.yaml -s 387 -e 396 -i s1 +hydra create oauth2-client \ + --endpoint http://hydra.hydra.svc:4445 \ + --name oidc-client-nologin \ + --secret traefiklabs \ + --grant-type authorization_code,refresh_token \ + --response-type code,id_token \ + --scope openid,offline \ + --redirect-uri http://secure-applications.apigateway.docker.localhost/oidc-nologin/callback \ + --post-logout-callback http://secure-applications.apigateway.docker.localhost/oidc/callback \ + --format json > /data/oidc-client-nologin.json +``` + +Now, test it: + +```shell +kubectl apply -f api-gateway/3-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml +``` + +We can now test it: + +```shell +# First time, it will login: +rm -f /tmp/cookie +curl -L -b /tmp/cookie -c /tmp/cookie http://secure-applications.apigateway.docker.localhost/oidc-nologin +# Second time, it will re-use the cookie: +curl -b /tmp/cookie -c /tmp/cookie http://secure-applications.apigateway.docker.localhost/oidc-nologin +``` diff --git a/src/minica/expose.apigateway.docker.localhost/cert.pem b/src/minica/expose.apigateway.docker.localhost/cert.pem new file mode 100644 index 0000000..0281396 --- /dev/null +++ b/src/minica/expose.apigateway.docker.localhost/cert.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICETCCAZegAwIBAgIIel7GQjjTAYcwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgMmU4NGQ4MB4XDTI0MTAxNDA4NTA0OFoXDTI2MTExMzA5 +NTA0OFowLTErMCkGA1UEAxMiZXhwb3NlLmFwaWdhdGV3YXkuZG9ja2VyLmxvY2Fs +aG9zdDB2MBAGByqGSM49AgEGBSuBBAAiA2IABBg9QESeEtzsH4xxt1O6EMXKVfaN +YHGJSWm5shzHYXKhfCvl9LBBEkNov8K4jWFHFWWogyQBUc8Xr+WbdnpFBXSMMlQS +8hICq6Q5dCgd7yAzaQfbRkxwBpU6YRMoHX0qiaOBkDCBjTAOBgNVHQ8BAf8EBAMC +BaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAw +HwYDVR0jBBgwFoAUnOKb6wawfl+otOL6oMkuljPnhncwLQYDVR0RBCYwJIIiZXhw +b3NlLmFwaWdhdGV3YXkuZG9ja2VyLmxvY2FsaG9zdDAKBggqhkjOPQQDAwNoADBl +AjEAvMaavWl6IQUdKkv6RgOwDnKdM85yjHCx9vUDpyhvKjXqfKjG1Rt9Opw7P1yv +B8hQAjBwdypTKO3RvyoReW3BGwBHtF0HDJcgmxuqEBf+60MR0MtGoCIPLZG2Pz5F +ikPY2Og= +-----END CERTIFICATE----- diff --git a/src/minica/expose.apigateway.docker.localhost/expose.yaml b/src/minica/expose.apigateway.docker.localhost/expose.yaml new file mode 100644 index 0000000..7f33738 --- /dev/null +++ b/src/minica/expose.apigateway.docker.localhost/expose.yaml @@ -0,0 +1,11 @@ +# kubectl create secret tls expose -n apps --cert expose.apigateway.docker.localhost/cert.pem --key expose.apigateway.docker.localhost/key.pem --dry-run=client --output=yaml > expose.apigateway.docker.localhost/expose.yaml +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNFVENDQVplZ0F3SUJBZ0lJZWw3R1FqalRBWWN3Q2dZSUtvWkl6ajBFQXdNd0lERWVNQndHQTFVRUF4TVYKYldsdWFXTmhJSEp2YjNRZ1kyRWdNbVU0TkdRNE1CNFhEVEkwTVRBeE5EQTROVEEwT0ZvWERUSTJNVEV4TXpBNQpOVEEwT0Zvd0xURXJNQ2tHQTFVRUF4TWlaWGh3YjNObExtRndhV2RoZEdWM1lYa3VaRzlqYTJWeUxteHZZMkZzCmFHOXpkREIyTUJBR0J5cUdTTTQ5QWdFR0JTdUJCQUFpQTJJQUJCZzlRRVNlRXR6c0g0eHh0MU82RU1YS1ZmYU4KWUhHSlNXbTVzaHpIWVhLaGZDdmw5TEJCRWtOb3Y4SzRqV0ZIRldXb2d5UUJVYzhYcitXYmRucEZCWFNNTWxRUwo4aElDcTZRNWRDZ2Q3eUF6YVFmYlJreHdCcFU2WVJNb0hYMHFpYU9Ca0RDQmpUQU9CZ05WSFE4QkFmOEVCQU1DCkJhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUhBd0VHQ0NzR0FRVUZCd01DTUF3R0ExVWRFd0VCL3dRQ01BQXcKSHdZRFZSMGpCQmd3Rm9BVW5PS2I2d2F3Zmwrb3RPTDZvTWt1bGpQbmhuY3dMUVlEVlIwUkJDWXdKSUlpWlhodwpiM05sTG1Gd2FXZGhkR1YzWVhrdVpHOWphMlZ5TG14dlkyRnNhRzl6ZERBS0JnZ3Foa2pPUFFRREF3Tm9BREJsCkFqRUF2TWFhdldsNklRVWRLa3Y2UmdPd0RuS2RNODV5akhDeDl2VURweWh2S2pYcWZLakcxUnQ5T3B3N1AxeXYKQjhoUUFqQndkeXBUS08zUnZ5b1JlVzNCR3dCSHRGMEhESmNnbXh1cUVCZis2ME1SME10R29DSVBMWkcyUHo1Rgppa1BZMk9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JRzJBZ0VBTUJBR0J5cUdTTTQ5QWdFR0JTdUJCQUFpQklHZU1JR2JBZ0VCQkRDNmpadW1qSmY5bER6ZFZhWlYKVmh6T3RrV2FWM2RFc2VhbXJDZ0NrSHM4MG9VanZ1Y2dEbHBRYzNIRzdEY0E2U1NoWkFOaUFBUVlQVUJFbmhMYwo3QitNY2JkVHVoREZ5bFgyaldCeGlVbHB1YkljeDJGeW9Yd3I1ZlN3UVJKRGFML0N1STFoUnhWbHFJTWtBVkhQCkY2L2xtM1o2UlFWMGpESlVFdklTQXF1a09YUW9IZThnTTJrSDIwWk1jQWFWT21FVEtCMTlLb2s9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +kind: Secret +metadata: + creationTimestamp: null + name: expose + namespace: apps +type: kubernetes.io/tls diff --git a/src/minica/expose.apigateway.docker.localhost/key.pem b/src/minica/expose.apigateway.docker.localhost/key.pem new file mode 100644 index 0000000..d53a782 --- /dev/null +++ b/src/minica/expose.apigateway.docker.localhost/key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDC6jZumjJf9lDzdVaZV +VhzOtkWaV3dEseamrCgCkHs80oUjvucgDlpQc3HG7DcA6SShZANiAAQYPUBEnhLc +7B+McbdTuhDFylX2jWBxiUlpubIcx2FyoXwr5fSwQRJDaL/CuI1hRxVlqIMkAVHP +F6/lm3Z6RQV0jDJUEvISAqukOXQoHe8gM2kH20ZMcAaVOmETKB19Kok= +-----END PRIVATE KEY----- diff --git a/src/minica/minica-key.pem b/src/minica/minica-key.pem new file mode 100644 index 0000000..5fd7cfa --- /dev/null +++ b/src/minica/minica-key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAfJqkkUCuC8NhQTbcV +IflodflVkGVY/5NEkm5GjlLTcZcdfMq/jOWqSxlb3G2vhXehZANiAARriolMN+2l +UfLZvfi+tsjXl2TydDQ9lB4ewnfOdApC9eCyw1LRDsq8cR+FDcee0up5cXi5J+ZW +FHMF+NAZK8f0xYaAiItSNhDlBW6yQlGjBWUkBaFthaZSFbFI+Jyt/xg= +-----END PRIVATE KEY----- diff --git a/src/minica/minica.pem b/src/minica/minica.pem new file mode 100644 index 0000000..7f5eefb --- /dev/null +++ b/src/minica/minica.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB+zCCAYKgAwIBAgIILoTYEWcRP+wwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgMmU4NGQ4MCAXDTI0MTAxNDA4NTA0OFoYDzIxMjQxMDE0 +MDg1MDQ4WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSAyZTg0ZDgwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAARriolMN+2lUfLZvfi+tsjXl2TydDQ9lB4ewnfOdApC +9eCyw1LRDsq8cR+FDcee0up5cXi5J+ZWFHMF+NAZK8f0xYaAiItSNhDlBW6yQlGj +BWUkBaFthaZSFbFI+Jyt/xijgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud +DgQWBBSc4pvrBrB+X6i04vqgyS6WM+eGdzAfBgNVHSMEGDAWgBSc4pvrBrB+X6i0 +4vqgyS6WM+eGdzAKBggqhkjOPQQDAwNnADBkAjAi7e8xJE8sw7RceDC3WLwWmN2O +N9+R37l+pzE8ChfU8/G1Tcmf3f4bkSP9N4fYn6gCMCJFsLtMjoFPhSWtiqME7YfT +uExSy+FcaU3jwh7a9Kf3vSAY+fjdpgpgOULDFEnn/A== +-----END CERTIFICATE----- diff --git a/src/minica/minica.yaml b/src/minica/minica.yaml new file mode 100644 index 0000000..d02a016 --- /dev/null +++ b/src/minica/minica.yaml @@ -0,0 +1,19 @@ +# kubectl create secret generic minica -n traefik --from-file=minica.pem --output=yaml > minica.yaml +--- +apiVersion: v1 +data: + minica.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUIrekNDQVlLZ0F3SUJBZ0lJTG9UWUVXY1JQK3d3Q2dZSUtvWkl6ajBFQXdNd0lERWVNQndHQTFVRUF4TVYKYldsdWFXTmhJSEp2YjNRZ1kyRWdNbVU0TkdRNE1DQVhEVEkwTVRBeE5EQTROVEEwT0ZvWUR6SXhNalF4TURFMApNRGcxTURRNFdqQWdNUjR3SEFZRFZRUURFeFZ0YVc1cFkyRWdjbTl2ZENCallTQXlaVGcwWkRnd2RqQVFCZ2NxCmhrak9QUUlCQmdVcmdRUUFJZ05pQUFScmlvbE1OKzJsVWZMWnZmaSt0c2pYbDJUeWREUTlsQjRld25mT2RBcEMKOWVDeXcxTFJEc3E4Y1IrRkRjZWUwdXA1Y1hpNUorWldGSE1GK05BWks4ZjB4WWFBaUl0U05oRGxCVzZ5UWxHagpCV1VrQmFGdGhhWlNGYkZJK0p5dC94aWpnWVl3Z1lNd0RnWURWUjBQQVFIL0JBUURBZ0tFTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VBTUIwR0ExVWQKRGdRV0JCU2M0cHZyQnJCK1g2aTA0dnFneVM2V00rZUdkekFmQmdOVkhTTUVHREFXZ0JTYzRwdnJCckIrWDZpMAo0dnFneVM2V00rZUdkekFLQmdncWhrak9QUVFEQXdObkFEQmtBakFpN2U4eEpFOHN3N1JjZURDM1dMd1dtTjJPCk45K1IzN2wrcHpFOENoZlU4L0cxVGNtZjNmNGJrU1A5TjRmWW42Z0NNQ0pGc0x0TWpvRlBoU1d0aXFNRTdZZlQKdUV4U3krRmNhVTNqd2g3YTlLZjN2U0FZK2ZqZHBncGdPVUxERkVubi9BPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +kind: Secret +metadata: + name: minica + namespace: traefik +type: Opaque +--- +apiVersion: v1 +data: + minica.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUIrekNDQVlLZ0F3SUJBZ0lJTG9UWUVXY1JQK3d3Q2dZSUtvWkl6ajBFQXdNd0lERWVNQndHQTFVRUF4TVYKYldsdWFXTmhJSEp2YjNRZ1kyRWdNbVU0TkdRNE1DQVhEVEkwTVRBeE5EQTROVEEwT0ZvWUR6SXhNalF4TURFMApNRGcxTURRNFdqQWdNUjR3SEFZRFZRUURFeFZ0YVc1cFkyRWdjbTl2ZENCallTQXlaVGcwWkRnd2RqQVFCZ2NxCmhrak9QUUlCQmdVcmdRUUFJZ05pQUFScmlvbE1OKzJsVWZMWnZmaSt0c2pYbDJUeWREUTlsQjRld25mT2RBcEMKOWVDeXcxTFJEc3E4Y1IrRkRjZWUwdXA1Y1hpNUorWldGSE1GK05BWks4ZjB4WWFBaUl0U05oRGxCVzZ5UWxHagpCV1VrQmFGdGhhWlNGYkZJK0p5dC94aWpnWVl3Z1lNd0RnWURWUjBQQVFIL0JBUURBZ0tFTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VBTUIwR0ExVWQKRGdRV0JCU2M0cHZyQnJCK1g2aTA0dnFneVM2V00rZUdkekFmQmdOVkhTTUVHREFXZ0JTYzRwdnJCckIrWDZpMAo0dnFneVM2V00rZUdkekFLQmdncWhrak9QUVFEQXdObkFEQmtBakFpN2U4eEpFOHN3N1JjZURDM1dMd1dtTjJPCk45K1IzN2wrcHpFOENoZlU4L0cxVGNtZjNmNGJrU1A5TjRmWW42Z0NNQ0pGc0x0TWpvRlBoU1d0aXFNRTdZZlQKdUV4U3krRmNhVTNqd2g3YTlLZjN2U0FZK2ZqZHBncGdPVUxERkVubi9BPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +kind: Secret +metadata: + name: minica + namespace: pebble +type: Opaque diff --git a/src/minica/pebble.minica.pem b/src/minica/pebble.minica.pem new file mode 100644 index 0000000..a69a4c4 --- /dev/null +++ b/src/minica/pebble.minica.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIIJOLbes8sTr4wDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVbWluaWNhIHJvb3QgY2EgMjRlMmRiMCAXDTE3MTIwNjE5NDIxMFoYDzIxMTcx +MjA2MTk0MjEwWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSAyNGUyZGIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ +alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn +Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu +9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0 +toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3 +Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB +AAGjRTBDMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwEgYDVR0TAQH/BAgwBgEB/wIBADANBgkqhkiG9w0BAQsFAAOCAQEAF85v +d40HK1ouDAtWeO1PbnWfGEmC5Xa478s9ddOd9Clvp2McYzNlAFfM7kdcj6xeiNhF +WPIfaGAi/QdURSL/6C1KsVDqlFBlTs9zYfh2g0UXGvJtj1maeih7zxFLvet+fqll +xseM4P9EVJaQxwuK/F78YBt0tCNfivC6JNZMgxKF59h0FBpH70ytUSHXdz7FKwix +Mfn3qEb9BXSk0Q3prNV5sOV3vgjEtB4THfDxSz9z3+DepVnW3vbbqwEbkXdk3j82 +2muVldgOUgTwK8eT+XdofVdntzU/kzygSAtAQwLJfn51fS1GvEcYGBc1bDryIqmF +p9BI7gVKtWSZYegicA== +-----END CERTIFICATE----- diff --git a/src/minica/pebble.pebble.svc/cert.pem b/src/minica/pebble.pebble.svc/cert.pem new file mode 100644 index 0000000..99290d0 --- /dev/null +++ b/src/minica/pebble.pebble.svc/cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB7DCCAXOgAwIBAgIIdpKC0oe0/5QwCgYIKoZIzj0EAwMwIDEeMBwGA1UEAxMV +bWluaWNhIHJvb3QgY2EgMmU4NGQ4MB4XDTI0MTAxODA4MDcxNloXDTI2MTExNzA5 +MDcxNlowHDEaMBgGA1UEAxMRcGViYmxlLnBlYmJsZS5zdmMwdjAQBgcqhkjOPQIB +BgUrgQQAIgNiAAQ7vr47W03SA6EdhI//F7YRzyzeHPrcxkPLoywluTUbNLfVMNku +dYSq7UrlAtb2JK9PEmUV+pDTq7xkD2OCIz6ZGvhGKHNxxgREXHcmS0SFbUw/2zZM +ruadyhYMnv+Ybw2jfjB8MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEF +BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBSc4pvrBrB+ +X6i04vqgyS6WM+eGdzAcBgNVHREEFTATghFwZWJibGUucGViYmxlLnN2YzAKBggq +hkjOPQQDAwNnADBkAjA6//ypWtLIn2QwBAYTDfdEWLozEtu1wNR5j9bVMZ6gXpm9 +Dyaq5SY/7Z7YLDdUZx0CMGtPZOICLXyfNBqbr22V+VxraIbOcunWguX793sOvLVK +/uep3PKLwwJgajna2bCG/Q== +-----END CERTIFICATE----- diff --git a/src/minica/pebble.pebble.svc/key.pem b/src/minica/pebble.pebble.svc/key.pem new file mode 100644 index 0000000..b8e4983 --- /dev/null +++ b/src/minica/pebble.pebble.svc/key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDXj8A0sUITliBrL38L +Fc/sMAnU8JsRRLYSfZk6JWx8spXRyOHycM3XR0zKHxF+R3ShZANiAAQ7vr47W03S +A6EdhI//F7YRzyzeHPrcxkPLoywluTUbNLfVMNkudYSq7UrlAtb2JK9PEmUV+pDT +q7xkD2OCIz6ZGvhGKHNxxgREXHcmS0SFbUw/2zZMruadyhYMnv+Ybw0= +-----END PRIVATE KEY----- diff --git a/src/minica/pebble.pebble.svc/pebble.yaml b/src/minica/pebble.pebble.svc/pebble.yaml new file mode 100644 index 0000000..5d4951b --- /dev/null +++ b/src/minica/pebble.pebble.svc/pebble.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: pebble + +--- +# kubectl create secret tls pebble -n pebble --cert pebble.pebble.svc/cert.pem --key pebble.pebble.svc/key.pem --dry-run=client --output=yaml >> pebble.pebble.svc/pebble.yaml +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUI3RENDQVhPZ0F3SUJBZ0lJZHBLQzBvZTAvNVF3Q2dZSUtvWkl6ajBFQXdNd0lERWVNQndHQTFVRUF4TVYKYldsdWFXTmhJSEp2YjNRZ1kyRWdNbVU0TkdRNE1CNFhEVEkwTVRBeE9EQTRNRGN4TmxvWERUSTJNVEV4TnpBNQpNRGN4Tmxvd0hERWFNQmdHQTFVRUF4TVJjR1ZpWW14bExuQmxZbUpzWlM1emRtTXdkakFRQmdjcWhrak9QUUlCCkJnVXJnUVFBSWdOaUFBUTd2cjQ3VzAzU0E2RWRoSS8vRjdZUnp5emVIUHJjeGtQTG95d2x1VFViTkxmVk1Oa3UKZFlTcTdVcmxBdGIySks5UEVtVVYrcERUcTd4a0QyT0NJejZaR3ZoR0tITnh4Z1JFWEhjbVMwU0ZiVXcvMnpaTQpydWFkeWhZTW52K1lidzJqZmpCOE1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVUJnZ3JCZ0VGCkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWZCZ05WSFNNRUdEQVdnQlNjNHB2ckJyQisKWDZpMDR2cWd5UzZXTStlR2R6QWNCZ05WSFJFRUZUQVRnaEZ3WldKaWJHVXVjR1ZpWW14bExuTjJZekFLQmdncQpoa2pPUFFRREF3Tm5BREJrQWpBNi8veXBXdExJbjJRd0JBWVREZmRFV0xvekV0dTF3TlI1ajliVk1aNmdYcG05CkR5YXE1U1kvN1o3WUxEZFVaeDBDTUd0UFpPSUNMWHlmTkJxYnIyMlYrVnhyYUliT2N1bldndVg3OTNzT3ZMVksKL3VlcDNQS0x3d0pnYWpuYTJiQ0cvUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JRzJBZ0VBTUJBR0J5cUdTTTQ5QWdFR0JTdUJCQUFpQklHZU1JR2JBZ0VCQkREWGo4QTBzVUlUbGlCckwzOEwKRmMvc01BblU4SnNSUkxZU2ZaazZKV3g4c3BYUnlPSHljTTNYUjB6S0h4RitSM1NoWkFOaUFBUTd2cjQ3VzAzUwpBNkVkaEkvL0Y3WVJ6eXplSFByY3hrUExveXdsdVRVYk5MZlZNTmt1ZFlTcTdVcmxBdGIySks5UEVtVVYrcERUCnE3eGtEMk9DSXo2Wkd2aEdLSE54eGdSRVhIY21TMFNGYlV3LzJ6Wk1ydWFkeWhZTW52K1lidzA9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +kind: Secret +metadata: + creationTimestamp: null + name: pebble + namespace: pebble +type: kubernetes.io/tls diff --git a/tests/apigateway/apigateway_test.go b/tests/apigateway/apigateway_test.go index d096f43..91033af 100644 --- a/tests/apigateway/apigateway_test.go +++ b/tests/apigateway/apigateway_test.go @@ -1,15 +1,20 @@ package apigateway import ( + "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/json" "errors" + "io" "net" "net/http" "net/http/cookiejar" "os" "path/filepath" + "regexp" "strings" "testing" "time" @@ -67,11 +72,14 @@ func (s *APIGatewayTestSuite) SetupSuite() { s.lbIP, err = testhelpers.InstallTraefikHubAPIGW(s.ctx, s.T(), s.k8s) s.Require().NoError(err) + re := regexp.MustCompile(`([-a-z.]+\.docker\.localhost)`) dialer := &net.Dialer{} s.tr = &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if strings.Contains(addr, "docker.localhost") { - addr = s.lbIP + ":80" + new_addr := re.ReplaceAllString(addr, s.lbIP) + // testcontainers.Logger.Printf("addr: %s => new_addr: %s\n", addr, new_addr) + addr = new_addr } return dialer.DialContext(ctx, network, addr) }, @@ -104,7 +112,7 @@ func (s *APIGatewayTestSuite) TestGettingStarted() { s.apply("src/manifests/apps-namespace.yaml") s.apply("src/manifests/weather-app.yaml") time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") s.Require().NoError(err) s.apply("api-gateway/1-getting-started/manifests/weather-app-ingressroute.yaml") @@ -134,15 +142,289 @@ type oAuth2ClientConfig struct { // No interest in other fields for this use case } +func (s *APIGatewayTestSuite) TestExpose() { + var err error + var req *http.Request + + jsonBody := []byte(`{ "query": "{ continents { code name } }" }`) + bodyReader := bytes.NewReader(jsonBody) + req, err = http.NewRequest(http.MethodPost, "https://countries.trevorblades.com/", bodyReader) + s.Require().NoError(err) + req.Header.Add("content-type", "application/json") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + s.apply("src/manifests/apps-namespace.yaml") + s.apply("api-gateway/2-expose/manifests/graphql-service.yaml") + s.apply("api-gateway/2-expose/manifests/graphql-ingressroute.yaml") + // no way to check IngressRoute is loaded, FTM + time.Sleep(1 * time.Second) + + bodyReader = bytes.NewReader(jsonBody) + req, err = http.NewRequest(http.MethodPost, "http://expose.apigateway.docker.localhost/graphql", bodyReader) + s.Require().NoError(err) + req.Header.Add("content-type", "application/json") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusNotFound)) + s.Assert().NoError(err) + + testhelpers.LaunchHelmUpgradeCommand(s.T(), + "--set", "providers.kubernetesCRD.allowExternalNameServices=true", + ) + + time.Sleep(1 * time.Second) + bodyReader = bytes.NewReader(jsonBody) + req, err = http.NewRequest(http.MethodPost, "http://expose.apigateway.docker.localhost/graphql", bodyReader) + s.Require().NoError(err) + req.Header.Add("content-type", "application/json") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusMisdirectedRequest)) + s.Assert().NoError(err) + + s.apply("api-gateway/2-expose/manifests/graphql-ingressroute-complete.yaml") + time.Sleep(1 * time.Second) + bodyReader = bytes.NewReader(jsonBody) + req, err = http.NewRequest(http.MethodPost, "http://expose.apigateway.docker.localhost/graphql", bodyReader) + s.Require().NoError(err) + req.Header.Add("content-type", "application/json") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + // Expose a website + s.apply("api-gateway/2-expose/manifests/webapp-db.yaml") + s.apply("api-gateway/2-expose/manifests/webapp-api.yaml") + s.apply("api-gateway/2-expose/manifests/webapp-front.yaml") + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute.yaml") + time.Sleep(1 * time.Second) + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 30*time.Second, "app in (db,api,web)") + s.Require().NoError(err) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + // Compress static text content + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/app.js", nil) + s.Require().NoError(err) + req.Header.Add("Accept-Encoding", "gzip, deflate, bz, zstd") + + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + testhelpers.HasNotHeader("Content-Encoding"), + ) + s.Assert().NoError(err) + + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-compress.yaml") + time.Sleep(1 * time.Second) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/app.js", nil) + s.Require().NoError(err) + req.Header.Add("Accept-Encoding", "gzip, deflate, bz, zstd") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + try.HasHeaderValue("Content-Encoding", "zstd", true), + ) + s.Assert().NoError(err) + + // Protect with Security Headers and CORS + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + req.Header.Add("Origin", "http://test2.com") + + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + testhelpers.HasNotHeader("Access-Control-Allow-Origin"), + ) + s.Assert().NoError(err) + + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-cors.yaml") + time.Sleep(1 * time.Second) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + req.Header.Add("Origin", "http://test.com") + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + try.HasHeaderValue("Access-Control-Allow-Origin", "http://test.com", true), + ) + s.Assert().NoError(err) + + // Error page + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/doesnotexist", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusNotFound), + try.BodyContains("404 page not found"), // Proxy default content on 404 + ) + s.Assert().NoError(err) + + s.apply("api-gateway/2-expose/manifests/error-page.yaml") + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-error-page.yaml") + time.Sleep(1 * time.Second) + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 30*time.Second, "app=error-page") + s.Require().NoError(err) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/doesnotexist", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusNotFound), + try.BodyContains("Error 404: Not Found"), // Error page default content on 404 + ) + s.Assert().NoError(err) + + // HTTP Caching + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-cache.yaml") + time.Sleep(1 * time.Second) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + try.HasHeaderValue("X-Cache-Status", "MISS", true), + ) + s.Assert().NoError(err) + + err = try.RequestWithTransport(req, 10*time.Second, s.tr, + try.StatusCodeIs(http.StatusOK), + try.HasHeaderValue("X-Cache-Status", "HIT", true), + ) + s.Assert().NoError(err) + + // Configure HTTPS + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-https.yaml") + time.Sleep(2 * time.Second) + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + ServerName: "expose.apigateway.docker.localhost", + NextProtos: []string{"h2", "http/1.1"}, + } + + conn, err := tls.Dial("tcp", s.lbIP+":443", tlsConfig) + s.Require().NoError(err) + defer conn.Close() + + err = conn.Handshake() + s.Require().NoError(err) + cs := conn.ConnectionState() + s.Require().Equal(cs.PeerCertificates[0].Issuer.CommonName, "TRAEFIK DEFAULT CERT") + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 5*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + // TLS with CA certificates + s.apply("src/minica/expose.apigateway.docker.localhost/expose.yaml") + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-https-manual.yaml") + time.Sleep(2 * time.Second) + + caCert, err := os.ReadFile(filepath.Join("..", "..", "src/minica/minica.pem")) + s.Assert().NoError(err) + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + s.tr.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: false, + ServerName: "expose.apigateway.docker.localhost", + RootCAs: caCertPool, + } + + // Check https with CA Cert + req, err = http.NewRequest(http.MethodGet, "https://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + req.Header.Set("Host", s.tr.TLSClientConfig.ServerName) + err = try.RequestWithTransport(req, 3*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + // Check http with CA Cert + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 5*time.Second, s.tr, try.StatusCodeIs(http.StatusNotFound)) + s.Assert().NoError(err) + + // Check https on http with CA Cert + req, err = http.NewRequest(http.MethodGet, "https://expose.apigateway.docker.localhost:80/", nil) + s.Require().NoError(err) + req.Header.Set("Host", s.tr.TLSClientConfig.ServerName) + err = try.RequestWithTransport(req, 5*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-https-split.yaml") + time.Sleep(1 * time.Second) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 5*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + req, err = http.NewRequest(http.MethodGet, "https://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + req.Header.Set("Host", s.tr.TLSClientConfig.ServerName) + err = try.RequestWithTransport(req, 5*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) + + // TLS with automation + s.apply("src/minica/pebble.pebble.svc/pebble.yaml") + s.apply("api-gateway/2-expose/manifests/pebble.yaml") + s.apply("src/minica/minica.yaml") + s.apply("api-gateway/2-expose/manifests/coredns-config.yaml") + + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 30*time.Second, "app=pebble") + s.Require().NoError(err) + err = testhelpers.RestartDeployment(s.ctx, s.T(), s.k8s, "coredns", "kube-system") + s.Require().NoError(err) + + testhelpers.LaunchHelmUpgradeCommand(s.T(), + "--set", "certificatesResolvers.pebble.distributedAcme.caServer=https://pebble.pebble.svc:14000/dir", + "--set", "certificatesResolvers.pebble.distributedAcme.email=test@example.com", + "--set", "certificatesResolvers.pebble.distributedAcme.storage.kubernetes=true", + "--set", "certificatesResolvers.pebble.distributedAcme.tlsChallenge=true", + "--set", "volumes[0].name=minica", + "--set", "volumes[0].mountPath=/minica", + "--set", "volumes[0].type=secret", + "--set", "env[0].name=LEGO_CA_CERTIFICATES", + "--set", "env[0].value=/minica/minica.pem", + ) + + s.apply("api-gateway/2-expose/manifests/webapp-ingressroute-https-auto.yaml") + time.Sleep(25 * time.Second) + + s.apply("api-gateway/2-expose/manifests/pebble-ingressroute.yaml") + time.Sleep(5 * time.Second) + + caCertPool = x509.NewCertPool() + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/pebble/roots/0", nil) + s.Assert().NoError(err) + response, err := http.DefaultClient.Do(req) + s.Assert().NoError(err) + body, err := io.ReadAll(response.Body) + s.Assert().NoError(err) + caCertPool.AppendCertsFromPEM(body) + + req, err = http.NewRequest(http.MethodGet, "http://expose.apigateway.docker.localhost/pebble/intermediates/0", nil) + s.Assert().NoError(err) + response, err = http.DefaultClient.Do(req) + s.Assert().NoError(err) + body, err = io.ReadAll(response.Body) + s.Assert().NoError(err) + caCertPool.AppendCertsFromPEM(body) + + s.tr.TLSClientConfig.RootCAs = caCertPool + // Check with dynamic CA Chain from pebble + req, err = http.NewRequest(http.MethodGet, "https://expose.apigateway.docker.localhost/", nil) + s.Require().NoError(err) + err = try.RequestWithTransport(req, 15*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) + s.Assert().NoError(err) +} + func (s *APIGatewayTestSuite) TestSecureApplications() { var err error var req *http.Request s.apply("src/manifests/hydra.yaml") time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 120*time.Second, "app=hydra") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 120*time.Second, "app=hydra") s.Require().NoError(err) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=consent") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=consent") s.Require().NoError(err) err = testhelpers.WaitForJobCompleted(s.ctx, s.T(), s.k8s, 60*time.Second, "app=create-hydra-clients") s.Require().NoError(err) @@ -151,15 +433,15 @@ func (s *APIGatewayTestSuite) TestSecureApplications() { s.apply("src/manifests/apps-namespace.yaml") s.apply("src/manifests/whoami-app.yaml") time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=whoami") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=whoami") s.Require().NoError(err) - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-ingressroute.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml") req, err = http.NewRequest(http.MethodGet, "http://secure-applications.apigateway.docker.localhost/no-auth", nil) s.Require().NoError(err) err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) s.Assert().NoError(err) - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds.yaml") output := testhelpers.LaunchKubectl(s.T(), "get", "secrets", "-n", "apps", "oauth-client", "-o", "json") s.Require().NotNil(output) oauth2 := k8sSecret{} @@ -180,7 +462,7 @@ func (s *APIGatewayTestSuite) TestSecureApplications() { s.Assert().NoError(err) // M2M with clientID / clientSecret in the mdw - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-oauth2-client-creds-nologin.yaml") req, err = http.NewRequest(http.MethodGet, "http://secure-applications.apigateway.docker.localhost/oauth2-client-credentials-nologin", nil) s.Require().NoError(err) @@ -189,13 +471,13 @@ func (s *APIGatewayTestSuite) TestSecureApplications() { // Test OIDC s.apply("src/manifests/apps-namespace.yaml") - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-ingressroute.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-ingressroute.yaml") req, err = http.NewRequest(http.MethodGet, "http://secure-applications.apigateway.docker.localhost/no-auth", nil) s.Require().NoError(err) err = try.RequestWithTransport(req, 10*time.Second, s.tr, try.StatusCodeIs(http.StatusOK)) s.Assert().NoError(err) - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-oidc.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-oidc.yaml") // FTM: No way to check when oidc has been loaded time.Sleep(5 * time.Second) @@ -232,7 +514,7 @@ func (s *APIGatewayTestSuite) TestSecureApplications() { s.Assert().NoError(err) // Test OIDC No Login - s.apply("api-gateway/2-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml") + s.apply("api-gateway/3-secure-applications/manifests/whoami-app-oidc-nologinurl.yaml") // FTM: No way to check when oidc has been loaded time.Sleep(5 * time.Second) diff --git a/tests/apimanagement/apimanagement_test.go b/tests/apimanagement/apimanagement_test.go index d05e7cb..0207467 100644 --- a/tests/apimanagement/apimanagement_test.go +++ b/tests/apimanagement/apimanagement_test.go @@ -107,7 +107,7 @@ func (s *APIManagementTestSuite) TestGettingStarted() { err = s.apply("src/manifests/weather-app.yaml") s.Require().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") s.Require().NoError(err) err = s.apply("api-management/1-getting-started/manifests/weather-app-ingressroute.yaml") @@ -141,9 +141,9 @@ func (s *APIManagementTestSuite) TestAccessControl() { err = s.apply("src/manifests/admin-app.yaml") s.Require().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") s.Require().NoError(err) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=admin-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=admin-app") s.Require().NoError(err) err = s.apply("api-management/2-access-control/manifests/simple-admin-api.yaml") @@ -204,7 +204,7 @@ func (s *APIManagementTestSuite) TestAPILifeCycleManagement() { s.Require().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") s.Require().NoError(err) err = s.check(http.MethodGet, "http://api.lifecycle.apimanagement.docker.localhost/weather", 5*time.Second, http.StatusUnauthorized) @@ -228,7 +228,7 @@ func (s *APIManagementTestSuite) TestAPILifeCycleManagement() { s.Assert().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app-forecast") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app-forecast") s.Require().NoError(err) var req *http.Request @@ -279,9 +279,9 @@ func (s *APIManagementTestSuite) TestProtectAPIInfrastructure() { s.Require().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=weather-app") s.Require().NoError(err) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=admin-app") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=admin-app") s.Require().NoError(err) // Step 1: Deploy weather and admin APIs @@ -308,11 +308,9 @@ func (s *APIManagementTestSuite) TestProtectAPIInfrastructure() { testcontainers.Logger.Printf("🔐 Found this redis password: %s\n", redisPassword) - testhelpers.LaunchHelmCommand(s.T(), "upgrade", "traefik", "-n", "traefik", "--wait", - "--reuse-values", + testhelpers.LaunchHelmUpgradeCommand(s.T(), "--set", "hub.redis.endpoints=redis-master.traefik.svc.cluster.local:6379", "--set", "hub.redis.password="+redisPassword, - "traefik/traefik", ) // wait for account to be sync'd @@ -400,7 +398,7 @@ func (s *APIManagementTestSuite) TestProtectAPIInfrastructure() { err = s.apply("api-management/4-protect-api-infrastructure/manifests/whoami-apiaccess.yaml") s.Require().NoError(err) time.Sleep(1 * time.Second) - err = testhelpers.WaitForPodReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=whoami") + err = testhelpers.WaitForPodsReady(s.ctx, s.T(), s.k8s, 90*time.Second, "app=whoami") err = s.checkWithBearer(http.MethodGet, "http://api.protect-infrastructure.apimanagement.docker.localhost/whoami", http.NoBody, externalToken, 5*time.Second, http.StatusOK) s.Assert().NoError(err) diff --git a/tests/testhelpers/containers.go b/tests/testhelpers/containers.go index bd556ee..c47d9cc 100644 --- a/tests/testhelpers/containers.go +++ b/tests/testhelpers/containers.go @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/k3s" + "github.com/traefik/traefik/v3/integration/try" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -77,6 +79,13 @@ func LaunchHelmCommand(t *testing.T, arg ...string) { require.NoError(t, err) } +// LaunchHelmUpgradeCommand execute `helm` CLI with arg and display stdout+stder with testcontainer logger +func LaunchHelmUpgradeCommand(t *testing.T, arg ...string) { + upgradeArgs := []string{"upgrade", "traefik", "-n", traefikNamespace, "--wait", "--version", "v33.0.0", "--reuse-values", "traefik/traefik"} + upgradeArgs = append(upgradeArgs, arg...) + LaunchHelmCommand(t, upgradeArgs...) +} + // LaunchKubectl execute `kubectl` CLI with arg and return stdout+stderr in a single string func LaunchKubectl(t *testing.T, arg ...string) *bytes.Buffer { cmd := exec.Command("kubectl", arg...) @@ -178,22 +187,23 @@ func CreateSecretForTraefikHub(ctx context.Context, t *testing.T, k8s client.Cli require.NoError(t, err) } -func WaitForPodReady(ctx context.Context, t *testing.T, k8s client.Client, interval time.Duration, labelSelector string) error { +func WaitForPodsReady(ctx context.Context, t *testing.T, k8s client.Client, interval time.Duration, labelSelector string) error { ctx, cancelFunc := context.WithTimeout(ctx, interval) return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { - state := getPodState(ctx, t, k8s, labelSelector) - if state.Terminated != nil { - cancelFunc() - return false, fmt.Errorf("pod with label %s terminated: %v", labelSelector, state.Terminated) - } + states := getPodsState(ctx, t, k8s, labelSelector) + for id, state := range states { + if state.Terminated != nil { + cancelFunc() + return false, fmt.Errorf("pod %s with label %s terminated: %v", id, labelSelector, state.Terminated) + } - if state.Running != nil { - cancelFunc() - return true, nil + if state.Waiting != nil { + return false, nil + } } - return false, nil + return true, nil }) } @@ -229,7 +239,19 @@ func Delete(ctx context.Context, k8s client.Client, kind, name, ns, group, versi return k8s.Delete(ctx, u) } -func getPodState(ctx context.Context, t *testing.T, k8s client.Client, labelSelector string) corev1.ContainerState { +func RestartDeployment(ctx context.Context, t *testing.T, k8s client.Client, name, ns string) error { + deploy := &appsv1.Deployment{} + err := k8s.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, deploy) + assert.NoError(t, err) + + if deploy.Spec.Template.ObjectMeta.Annotations == nil { + deploy.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + deploy.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + return k8s.Update(ctx, deploy) +} + +func getPodsState(ctx context.Context, t *testing.T, k8s client.Client, labelSelector string) map[string]corev1.ContainerState { podList := &corev1.PodList{} selector, err := labels.Parse(labelSelector) @@ -237,15 +259,16 @@ func getPodState(ctx context.Context, t *testing.T, k8s client.Client, labelSele err = k8s.List(ctx, podList, &client.ListOptions{LabelSelector: selector}) require.NoError(t, err) - if len(podList.Items) != 1 { - log.Fatalf("There should be only one pod with label %s, found %d\n", labelSelector, len(podList.Items)) - } - status := podList.Items[0].Status - if len(status.ContainerStatuses) != 1 { - log.Fatalf("There should be only one container on pod labeled %s, found %d\n", labelSelector, len(status.ContainerStatuses)) + states := map[string]corev1.ContainerState{} + for _, pod := range podList.Items { + s := pod.Status + for _, status := range s.ContainerStatuses { + id := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, status.Name) + states[id] = status.State + } } - return status.ContainerStatuses[0].State + return states } func getJobState(ctx context.Context, t *testing.T, k8s client.Client, labelSelector string) batchv1.JobStatus { @@ -370,3 +393,12 @@ func prepareResources(decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructu return resources, nil } + +func HasNotHeader(header string) try.ResponseCondition { + return func(res *http.Response) error { + if _, ok := res.Header[header]; ok { + return fmt.Errorf(`response should not contains this header: "%s"`, header) + } + return nil + } +} diff --git a/tests/testhelpers/containers_test.go b/tests/testhelpers/containers_test.go index 9fe41cb..197244b 100644 --- a/tests/testhelpers/containers_test.go +++ b/tests/testhelpers/containers_test.go @@ -49,15 +49,23 @@ func Test_WaitForPodReady(t *testing.T) { apply(t, ctx, k8s, "src/manifests/apps-namespace.yaml") apply(t, ctx, k8s, "src/manifests/whoami-app.yaml") - // whoami takes around 30s to be deployed to it should be enough for WaitForPodReady to fail in the context of this test + // whoami takes around 30s to be deployed to it should be enough for WaitForPodsReady to fail in the context of this test timeout := 90 go func() { podRunning := false for i := 0; i < timeout; i++ { time.Sleep(time.Second) - state := getPodState(ctx, t, k8s, "app=whoami") - if state.Running != nil { + states := getPodsState(ctx, t, k8s, "app=whoami") + allContainersRunning := true + for _, state := range states { + if state.Waiting != nil || state.Terminated != nil { + allContainersRunning = false + break + } + } + + if allContainersRunning { podRunning = true break } @@ -72,6 +80,6 @@ func Test_WaitForPodReady(t *testing.T) { time.Sleep(1 * time.Second) - err = WaitForPodReady(ctx, t, k8s, time.Duration(timeout)*time.Second, "app=whoami") + err = WaitForPodsReady(ctx, t, k8s, time.Duration(timeout)*time.Second, "app=whoami") require.NoError(t, err) }