diff --git a/githubactions-dovecot/Dockerfile b/githubactions-dovecot/Dockerfile
index cde8b33..0d93e2f 100644
--- a/githubactions-dovecot/Dockerfile
+++ b/githubactions-dovecot/Dockerfile
@@ -29,6 +29,18 @@ RUN \
 COPY ./files/etc/dovecot/local.conf /etc/dovecot/local.conf
 COPY ./files/etc/dovecot/passwd /etc/dovecot/passwd
 
+# Debian's default config puts a PAM passdb in front of our passwd-file one.
+# Test users are not system users, so PAM always fails, which add a 2 seconds delay
+# before Dovecot falls back to passwd-file.
+# Disable it and fail the build if PAM is still configured (e.g. if Debian changes this line).
+RUN \
+  sed -i 's/^!include auth-system.conf.ext/#&/' /etc/dovecot/conf.d/10-auth.conf \
+  \
+  # Check the resulting config.
+  && doveconf -n > /tmp/doveconf.txt \
+  && ! grep -qw pam /tmp/doveconf.txt \
+  && rm /tmp/doveconf.txt
+
 HEALTHCHECK --interval=10s --retries=5 --timeout=5s --start-period=30s --start-interval=1s \
   CMD doveadm service status
 
diff --git a/githubactions-dovecot/files/etc/dovecot/local.conf b/githubactions-dovecot/files/etc/dovecot/local.conf
index f70623b..3230419 100644
--- a/githubactions-dovecot/files/etc/dovecot/local.conf
+++ b/githubactions-dovecot/files/etc/dovecot/local.conf
@@ -4,6 +4,9 @@ mail_path = /home/glpi/Maildir
 mail_inbox_path = /home/glpi/Maildir
 mail_max_userip_connections = 10000
 
+# Do not slow down failed logins, tests use wrong credentials on purpose.
+auth_failure_delay = 0
+
 passdb passwd-file {
   passwd_file_path = /etc/dovecot/passwd
 }
diff --git a/githubactions-dovecot/files/etc/dovecot/passwd b/githubactions-dovecot/files/etc/dovecot/passwd
index 7a1dd29..3bf0df6 100644
--- a/githubactions-dovecot/files/etc/dovecot/passwd
+++ b/githubactions-dovecot/files/etc/dovecot/passwd
@@ -1 +1,5 @@
 testuser:{plain}applesauce:1000:1000::/home/glpi::
+# Some fixtures scripts are included with `doveadm -u glpi`.
+# Since we disabled the PAM passdb we now need a dedicated user here so these
+# scripts do not break.
+glpi:{plain}glpi:1000:1000::/home/glpi::