nginx / docker-gen triggering new cert until rate limit reached..

[asaupp]

I'm using Fedora 34 with podman, using nginx-letsencrypt and jwilder/docker-gen. The process (e.g. start git container, automatic cert generation, https functional) is working well. However, when restarting server / containers to often, i get banned by letsencrypt. Note that grafana is still valid / working - but git got banned, probably restarted that one more often latly.

Reloading nginx docker-gen (using separate container nginx-gen)...
Reloading nginx (using separate container 19e76d0ad4337dbb7a40304537bff8d5a202b7cf63e7ff639625726185da1939)...
Sleep for 3600s
Creating/renewal git.xydomain.de certificates... (git.xydomain.de)
[Fri May 14 06:56:55 UTC 2021] Using CA: https://acme-v02.api.letsencrypt.org/directory
[Fri May 14 06:56:55 UTC 2021] Creating domain key
[Fri May 14 06:56:56 UTC 2021] The domain key is here: /etc/acme.sh/[email protected]/git.xydomain.de/git.xydomain.de.key
[Fri May 14 06:56:56 UTC 2021] Single domain='git.xydomain.de'
[Fri May 14 06:56:56 UTC 2021] Getting domain auth token for each domain
[Fri May 14 06:56:57 UTC 2021] Create new order error. Le_OrderFinalize not found. {
  "type": "urn:ietf:params:acme:error:rateLimited",
  "detail": "Error creating new order :: too many certificates already issued for exact set of domains: git.xydomain.de: see https://letsencrypt.org/docs/rate-limits/",
  "status": 429
}
[Fri May 14 06:56:57 UTC 2021] Please check log file for more details: /dev/null
Creating/renewal grafana.xydomain.de certificates... (grafana.xydomain.de)
[Fri May 14 06:56:58 UTC 2021] Domains not changed.
[Fri May 14 06:56:58 UTC 2021] Skip, Next renewal time is: Tue Jul 13 05:56:53 UTC 2021
[Fri May 14 06:56:58 UTC 2021] Add '--force' to force to renew.
Sleep for 3600s
Creating/renewal git.xydomain.de certificates... (git.xydomain.de)
[Fri May 14 07:57:00 UTC 2021] Using CA: https://acme-v02.api.letsencrypt.org/directory
[Fri May 14 07:57:00 UTC 2021] Creating domain key
[Fri May 14 07:57:00 UTC 2021] The domain key is here: /etc/acme.sh/[email protected]/git.xydomain.de/git.xydomain.de.key
[Fri May 14 07:57:00 UTC 2021] Single domain='git.xydomain.de'
[Fri May 14 07:57:00 UTC 2021] Getting domain auth token for each domain
[Fri May 14 07:57:01 UTC 2021] Create new order error. Le_OrderFinalize not found. {
  "type": "urn:ietf:params:acme:error:rateLimited",
  "detail": "Error creating new order :: too many certificates already issued for exact set of domains: git.xydomain.de: see https://letsencrypt.org/docs/rate-limits/",
  "status": 429
}

I haven't understood the process well enough in order to investigate. The certs are available on the host via bind mount

cat docker-compose.yml
version: '3'
services:
  nginx:
    image: docker.io/nginx
    container_name: nginx
    labels:
      com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: "true"
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./conf.d:/etc/nginx/conf.d
      - ./vhost.d:/etc/nginx/vhost.d
      - ./html:/usr/share/nginx/html
      - ./certs:/etc/nginx/certs:ro
      - ./config/usr/local/bin/check_service:/usr/local/bin/check_service

  nginx-gen:
    image: docker.io/jwilder/docker-gen
    container_name: nginx-gen
    command: -notify-sighup nginx -watch -wait 5s:30s /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf
    restart: unless-stopped
    volumes:
      - ./conf.d:/etc/nginx/conf.d
      - ./vhost.d:/etc/nginx/vhost.d
      - ./html:/usr/share/nginx/html
      - ./certs:/etc/nginx/certs:ro
      - ./nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl:ro
      - /var/run/docker.sock:/tmp/docker.sock:ro

  nginx-letsencrypt:
    image: docker.io/jrcs/letsencrypt-nginx-proxy-companion
    container_name: nginx-letsencrypt
    restart: unless-stopped
    volumes:
      - ./conf.d:/etc/nginx/conf.d
      - ./vhost.d:/etc/nginx/vhost.d
      - ./html:/usr/share/nginx/html
      - ./certs:/etc/nginx/certs:rw
      - /var/run/docker.sock:/var/run/docker.sock:ro
    environment:
      NGINX_DOCKER_GEN_CONTAINER: "nginx-gen"
      NGINX_PROXY_CONTAINER: "nginx"
networks:
  default:
    external:
      name: nginx-proxy

I had to tweak my nginx.templ due to IP resolution errors in podman, the second line was marking the server down with the original template.

{{ if .Network.IP }}
                        server {{ .Network.IP }}:{{.Container.Env.VIRTUAL_PORT}};
{{ else }}
{{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }}

{{ define "upstream" }}
        {{ if .Address }}
                {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}}
                {{ if and .Container.Node.ID .Address.HostPort }}
                        # {{ .Container.Node.Name }}/{{ .Container.Name }}
                        server {{ .Container.Node.Address.IP }}:{{ .Address.HostPort }};
                {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}}
                {{ else if .Network }}
                        # {{ .Container.Name }}
                        server {{ .Network.IP }}:{{ .Address.Port }};
                {{ end }}
        {{ else if .Network }}
                # {{ .Container.Name }}
                {{ if .Network.IP }}
                        server {{ .Network.IP }}:{{.Container.Env.VIRTUAL_PORT}};
                {{ else }}
                        server 127.0.0.1 down;
                {{ end }}
        {{ end }}

{{ end }}


# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
# scheme used to connect to this server
map $http_x_forwarded_proto $proxy_x_forwarded_proto {
  default $http_x_forwarded_proto;
  ''      $scheme;
}

# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
# server port the client connected to
map $http_x_forwarded_port $proxy_x_forwarded_port {
  default $http_x_forwarded_port;
  ''      $server_port;
}

# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
# Connection header that may have been passed to this server
map $http_upgrade $proxy_connection {
  default upgrade;
  '' close;
}

[..]

#  docker inspect iot-git |grep NetworkSettings -A 40
Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.
        "NetworkSettings": {
            "EndpointID": "",
            "Gateway": "",
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "MacAddress": "",
            "Bridge": "",
            "SandboxID": "",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {},
            "SandboxKey": "/run/netns/cni-b30f5d52-4b48-f160-15ce-cdcb87aa15f2",
            "Networks": {
                "nginx-proxy": {
                    "EndpointID": "",
                    "Gateway": "172.19.0.1",
                    "IPAddress": "172.19.0.9",
                    "IPPrefixLen": 24,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "3a:e3:e5:82:9c:e4",
                    "NetworkID": "nginx-proxy",
                    "DriverOpts": null,
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "ecf7c5ea38eb",
                        "git"
                    ]
                }
            }
        },

Initially (missplaced?) this friendly support request at nginx-proxy/docker-gen#351

Alex

[buchdag]

Hi, take a look at #796

[asaupp]

the switch of the ACME client used by the Docker image from simp.le to acme.sh. This switch result in some backward incompatible changes, so please read this issue

Thanks a ton - that's it, I believe.

[buchdag]

In case the fix isn't clear, you should add a volume for acme.sh configuration persistence to you acme-companion container.

So something like that in your config:

services:
 
[...]
 
nginx-letsencrypt:
    image: docker.io/jrcs/letsencrypt-nginx-proxy-companion
    [...]
    volumes:
      - ./acme.sh:/etc/acme.sh
    [...]