diff --git a/Dockerfile b/Dockerfile index e24bd05..187fd78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,11 @@ ENV KEYCLOAK_IMPORT /opt/jboss/realms/example.json # Add module COPY vendor/modules/ /opt/jboss/modules/ -# TODO don't know if we need this actually +# Add custom inifinispan jdbc-string store key mapper module +RUN mkdir -p ${JBOSS_HOME}/modules/de/coliquio/keycloak/main +COPY module/extended-keymapper.jar ${JBOSS_HOME}/modules/de/coliquio/keycloak/main/extended-keymapper.jar +COPY configuration/infinispan-module.xml ${JBOSS_HOME}/modules/system/layers/base/org/jboss/as/clustering/infinispan/main/module.xml +COPY configuration/extended-keymapper-module.xml ${JBOSS_HOME}/modules/de/coliquio/keycloak/main/module.xml COPY configuration/jgroups-module.xml ${JBOSS_HOME}/modules/system/layers/base/org/jgroups/main/module.xml # add customized tools (docker-entrypoint.sh and jgroups configuration cli) diff --git a/README.md b/README.md index 5725fa6..4b7660e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # Keycloak Cluster based on Docker + JDBC_PING -Note: this demonstrates issues creating session on keycloak cluster +Note: this demonstrates issues persisting sessions using Keycloak Infinispan JDBC Store -Related discussions: +## Sandbox Environment -- https://issues.jboss.org/browse/KEYCLOAK-9855 -- http://lists.jboss.org/pipermail/keycloak-user/2019-March/017511.html +Requirements: docker and docker-compose are installed -## Start the cluster +### Start the cluster ```bash # just start @@ -17,34 +16,67 @@ docker-compose -f docker-compose.yml -f docker-compose.ports.yml up docker-compose down && docker-compose -f docker-compose.yml -f docker-compose.ports.yml up --build ``` -## Create Sessions +### Create Sessions -### Cluster (this fails and demonstrates the issue) +#### Cluster ```bash open http://localhost:8000/auth/realms/example/account -# Username `user` -# Password `password` - -# => fails with the following error on the UI: -# 1st try (or cleared cookies) => "An error occurred, please login again through your application." -# 2nd try (with existing cookies) => "You are already logged in." +# Username `admin` +# Password `admin` ``` -### Single Node +#### Single Node ```bash open http://localhost:8081/auth/realms/example/account -# Username `user` -# Password `password` +# Username `admin` +# Password `admin` # => works, the "Edit Account" view is shown ``` +### Problem + +#### Expectation + +Keycloak cluster nodes should persist all sessions in the JDBC mysql database, because the caches are configured like that [./startup-scripts/cache_owners.cli](./startup-scripts/cache_owners.cli) + +So after stopping and restarting all cluster nodes, the expectation is that Keycloak nodes get their sessions again from the mysql database. + +#### Observation + +After stopping all cluster nodes and restarting them, all sessions are gone. + +#### Steps to reproduce + +1. Stop instance 1 `docker stop keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_1` +2. Check in browser if you are still logged in -> works, because sessions are spread in cluster memory (**CORRECT**) +3. Stop instance 2 `docker stop keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_2` +4. Of course, now no web UI is available +5. Start instance 1 again `docker start keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_1` (wait some minute to until it comes up again) +6. Check in browser if you are still logged in -> works, because sessions were not persisted in mysql db (**WRONG**) + ## Analyse Internals +### JDBC Sessions + +No client sessions (does not match expectations because user just logged in) + +```bash +mysql --host 127.0.0.1 --user root --password=root --database keycloak --execute "select * from ISPN_clientSessions;" +``` + +No sessions (does not match expectations because user just logged in) + +```bash +mysql --host 127.0.0.1 --user root --password=root --database keycloak --execute "select * from ISPN_sessions;" +``` + +Interestingly for the tables `ISPN_authenticationSessions` or `ISPN_actionTokens` the persistence works immediately (e.g. after opening a new login page without being logged in)! + ### Container Logs ```bash diff --git a/configuration/extended-keymapper-module.xml b/configuration/extended-keymapper-module.xml new file mode 100644 index 0000000..5329b1f --- /dev/null +++ b/configuration/extended-keymapper-module.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/configuration/infinispan-module.xml b/configuration/infinispan-module.xml new file mode 100644 index 0000000..af5f2bd --- /dev/null +++ b/configuration/infinispan-module.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/module/extended-keymapper.jar b/module/extended-keymapper.jar new file mode 100644 index 0000000..075bc9f Binary files /dev/null and b/module/extended-keymapper.jar differ diff --git a/module/extended-keymapper/pom.xml b/module/extended-keymapper/pom.xml new file mode 100644 index 0000000..02f6b80 --- /dev/null +++ b/module/extended-keymapper/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + de.coliquio + keycloak + 1.0-SNAPSHOT + jar + + + extended-keymapper + + + + UTF-8 + + + + org.infinispan + infinispan-core + 9.4.3.Final + + + \ No newline at end of file diff --git a/module/extended-keymapper/src/main/java/de/coliquio/keycloak/ExtendedKeyMapper.java b/module/extended-keymapper/src/main/java/de/coliquio/keycloak/ExtendedKeyMapper.java new file mode 100644 index 0000000..10d787c --- /dev/null +++ b/module/extended-keymapper/src/main/java/de/coliquio/keycloak/ExtendedKeyMapper.java @@ -0,0 +1,39 @@ +package de.coliquio.keycloak; + +import org.infinispan.persistence.keymappers.DefaultTwoWayKey2StringMapper; +import org.jboss.logging.Logger; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.UUID; + +public class ExtendedKeyMapper extends DefaultTwoWayKey2StringMapper { + private final static Logger logger = Logger.getLogger(ExtendedKeyMapper.class); + private static final char NON_STRING_PREFIX = '\uFEFF'; + + @Override + public String getStringMapping(Object key) { + logger.info("getStringMapping(" + key.toString() + "<" + key.getClass().toString() + ">)"); + if (key.getClass().equals(UUID.class)) { + return NON_STRING_PREFIX + "u" + ((UUID) key).toString(); + } + + return super.getStringMapping(key); + } + + @Override + public Object getKeyMapping(String key) { + logger.info("getKeyMapping(" + key + ")"); + + if (key.length() > 0 && key.charAt(0) == NON_STRING_PREFIX && key.charAt(1) == 'u') { + return UUID.fromString(key.substring(2)); + } + + return super.getKeyMapping(key); + } + + @Override + public boolean isSupportedType(Class keyType) { + return keyType == UUID.class || super.isSupportedType(keyType); + } +} diff --git a/startup-scripts/cache_owners.cli b/startup-scripts/cache_owners.cli index 59f66ab..9c45700 100644 --- a/startup-scripts/cache_owners.cli +++ b/startup-scripts/cache_owners.cli @@ -1,24 +1,51 @@ embed-server --server-config=standalone-ha.xml --std-out=echo batch -/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:remove -/subsystem=infinispan/cache-container=keycloak/replicated-cache=sessions:add() -/subsystem=infinispan/cache-container=keycloak/replicated-cache=sessions:write-attribute(name="mode",value="SYNC") +cd /subsystem=infinispan/cache-container=keycloak/ -/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:remove -/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2}) +./distributed-cache=sessions:remove +./distributed-cache=sessions:add() +./distributed-cache=authenticationSessions:remove +./distributed-cache=authenticationSessions:add() +./distributed-cache=offlineSessions:remove +./distributed-cache=offlineSessions:add() +./distributed-cache=clientSessions:remove +./distributed-cache=clientSessions:add() +./distributed-cache=offlineClientSessions:remove +./distributed-cache=offlineClientSessions:add() +./distributed-cache=loginFailures:remove +./distributed-cache=loginFailures:add() +./distributed-cache=actionTokens:remove +./distributed-cache=actionTokens:add() -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:remove -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2}) +./distributed-cache=sessions:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=authenticationSessions:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=offlineSessions:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=clientSessions:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=offlineClientSessions:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=loginFailures:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) +./distributed-cache=actionTokens:write-attribute (name=owners, value=${env.CACHE_OWNERS:2}) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:remove -/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2}) +./distributed-cache=sessions/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=sessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:remove -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2}) +./distributed-cache=authenticationSessions/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=authenticationSessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") -/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:remove -/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(mode="SYNC",owners=${env.CACHE_OWNERS:2}) +./distributed-cache=offlineSessions/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=offlineSessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") + +./distributed-cache=clientSessions/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=clientSessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") + +./distributed-cache=offlineClientSessions/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=offlineClientSessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") + +./distributed-cache=loginFailures/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=loginFailures/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") + +./distributed-cache=actionTokens/store=string-jdbc:add (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"}) +./distributed-cache=actionTokens/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN") run-batch -stop-embedded-server +stop-embedded-server \ No newline at end of file