Session persistence

Dockerfile

@@ -12,7 +12,11 @@ ENV KEYCLOAK_IMPORT /opt/jboss/realms/example.json
 # Add module
 COPY vendor/modules/ /opt/jboss/modules/
 
-# TODO don't know if we need this actually
+# Add custom inifinispan jdbc-string store key mapper module
+RUN mkdir -p ${JBOSS_HOME}/modules/de/coliquio/keycloak/main
+COPY module/extended-keymapper.jar ${JBOSS_HOME}/modules/de/coliquio/keycloak/main/extended-keymapper.jar
+COPY configuration/infinispan-module.xml ${JBOSS_HOME}/modules/system/layers/base/org/jboss/as/clustering/infinispan/main/module.xml
+COPY configuration/extended-keymapper-module.xml ${JBOSS_HOME}/modules/de/coliquio/keycloak/main/module.xml
 COPY configuration/jgroups-module.xml ${JBOSS_HOME}/modules/system/layers/base/org/jgroups/main/module.xml
 
 # add customized tools (docker-entrypoint.sh and jgroups configuration cli)

README.md

@@ -1,13 +1,12 @@
 # Keycloak Cluster based on Docker + JDBC_PING
 
-Note: this demonstrates issues creating session on keycloak cluster
+Note: this demonstrates issues persisting sessions using Keycloak Infinispan JDBC Store
 
-Related discussions:
+## Sandbox Environment
 
-- https://issues.jboss.org/browse/KEYCLOAK-9855
-- http://lists.jboss.org/pipermail/keycloak-user/2019-March/017511.html
+Requirements: docker and docker-compose are installed
 
-## Start the cluster
+### Start the cluster
 
 ```bash
 # just start
@@ -17,34 +16,67 @@ docker-compose -f docker-compose.yml -f docker-compose.ports.yml up
 docker-compose down && docker-compose -f docker-compose.yml -f docker-compose.ports.yml up --build
 ```
 
-## Create Sessions
+### Create Sessions
 
-### Cluster (this fails and demonstrates the issue)
+#### Cluster
 
 ```bash
 open http://localhost:8000/auth/realms/example/account
 
-# Username `user`
-# Password `password`
-
-# => fails with the following error on the UI:
-# 1st try (or cleared cookies) => "An error occurred, please login again through your application."
-# 2nd try (with existing cookies) => "You are already logged in."
+# Username `admin`
+# Password `admin`
 ```
 
-### Single Node
+#### Single Node
 
 ```bash
 open http://localhost:8081/auth/realms/example/account
 
-# Username `user`
-# Password `password`
+# Username `admin`
+# Password `admin`
 
 # => works, the "Edit Account" view is shown
 ```
 
+### Problem
+
+#### Expectation
+
+Keycloak cluster nodes should persist all sessions in the JDBC mysql database, because the caches are configured like that [./startup-scripts/cache_owners.cli](./startup-scripts/cache_owners.cli)
+
+So after stopping and restarting all cluster nodes, the expectation is that Keycloak nodes get their sessions again from the mysql database.
+
+#### Observation
+
+After stopping all cluster nodes and restarting them, all sessions are gone.
+
+#### Steps to reproduce
+
+1. Stop instance 1 `docker stop keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_1`
+2. Check in browser if you are still logged in -> works, because sessions are spread in cluster memory (**CORRECT**)
+3. Stop instance 2 `docker stop keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_2`
+4. Of course, now no web UI is available
+5. Start instance 1 again `docker start keycloak-docker-jdbcping-cluster-example_mysql_jdbcping_1` (wait some minute to until it comes up again)
+6. Check in browser if you are still logged in -> works, because sessions were not persisted in mysql db (**WRONG**)
+
 ## Analyse Internals
 
+### JDBC Sessions
+
+No client sessions (does not match expectations because user just logged in)
+
+```bash
+mysql --host 127.0.0.1 --user root --password=root --database keycloak --execute "select * from ISPN_clientSessions;"
+```
+
+No sessions (does not match expectations because user just logged in)
+
+```bash
+mysql --host 127.0.0.1 --user root --password=root --database keycloak --execute "select * from ISPN_sessions;"
+```
+
+Interestingly for the tables `ISPN_authenticationSessions` or `ISPN_actionTokens` the persistence works immediately (e.g. after opening a new login page without being logged in)!
+
 ### Container Logs
 
 ```bash

configuration/extended-keymapper-module.xml

@@ -0,0 +1,13 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<module xmlns="urn:jboss:module:1.1" name="de.coliquio.keycloak">
+
+    <resources>
+        <resource-root path="extended-keymapper.jar"/>
+    </resources>
+
+    <dependencies>
+        <module name="org.infinispan"/>
+        <module name="org.infinispan.persistence.jdbc"/>
+        <module name="org.jboss.logging"/>
+    </dependencies>
+</module>

configuration/infinispan-module.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2010, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+<module name="org.jboss.as.clustering.infinispan" xmlns="urn:jboss:module:1.5">
+    <properties>
+        <property name="jboss.api" value="private"/>
+    </properties>
+
+    <exports>
+        <exclude path="org/wildfly/clustering/ejb/infinispan/logging"/>
+    </exports>
+
+    <resources>
+        <resource-root path="wildfly-clustering-infinispan-extension-15.0.1.Final.jar"/>
+    </resources>
+
+    <dependencies>
+        <module name="javax.api"/>
+        <module name="javax.transaction.api"/>
+        <module name="io.reactivex.rxjava2.rxjava"/>
+        <module name="net.jcip"/>
+        <module name="de.coliquio.keycloak" />
+        <module name="org.infinispan" services="import"/>
+        <module name="org.infinispan.persistence.jdbc"/>
+        <module name="org.infinispan.persistence.remote"/>
+        <module name="org.infinispan.client.hotrod"/>
+        <module name="org.infinispan.commons"/>
+        <module name="org.jboss.as.clustering.common"/>
+        <module name="org.jboss.as.clustering.jgroups"/>
+        <module name="org.jboss.as.controller"/>
+        <module name="org.jboss.as.naming"/>
+        <module name="org.jboss.as.network"/>
+        <module name="org.jboss.as.server"/>
+        <module name="org.jboss.jboss-transaction-spi"/>
+        <module name="org.jboss.logging"/>
+        <module name="org.jboss.marshalling"/>
+        <module name="org.jboss.modules"/>
+        <module name="org.jboss.msc"/>
+        <module name="org.jboss.staxmapper"/>
+        <module name="org.jboss.threads"/>
+        <module name="org.jgroups"/>
+        <module name="org.reactivestreams"/>
+        <module name="org.wildfly.clustering.ee.infinispan"/>
+        <module name="org.wildfly.clustering.ee.spi"/>
+        <module name="org.wildfly.clustering.infinispan.spi"/>
+        <module name="org.wildfly.clustering.jgroups.spi"/>
+        <module name="org.wildfly.clustering.marshalling.api"/>
+        <module name="org.wildfly.clustering.marshalling.spi"/>
+        <module name="org.wildfly.clustering.marshalling.infinispan"/>
+        <module name="org.wildfly.clustering.marshalling.jboss"/>
+        <module name="org.wildfly.clustering.service"/>
+        <module name="org.wildfly.clustering.spi"/>
+        <module name="org.wildfly.common"/>
+        <module name="org.wildfly.security.elytron-private"/>
+        <module name="org.wildfly.transaction.client"/>
+    </dependencies>
+</module>

module/extended-keymapper/pom.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>de.coliquio</groupId>
+    <artifactId>keycloak</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <build>
+        <finalName>extended-keymapper</finalName>
+    </build>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.infinispan</groupId>
+            <artifactId>infinispan-core</artifactId>
+            <version>9.4.3.Final</version>
+        </dependency>
+    </dependencies>
+</project>

module/extended-keymapper/src/main/java/de/coliquio/keycloak/ExtendedKeyMapper.java

@@ -0,0 +1,39 @@
+package de.coliquio.keycloak;
+
+import org.infinispan.persistence.keymappers.DefaultTwoWayKey2StringMapper;
+import org.jboss.logging.Logger;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.UUID;
+
+public class ExtendedKeyMapper extends DefaultTwoWayKey2StringMapper {
+    private final static Logger logger = Logger.getLogger(ExtendedKeyMapper.class);
+    private static final char NON_STRING_PREFIX = '\uFEFF';
+
+    @Override
+    public String getStringMapping(Object key) {
+        logger.info("getStringMapping(" + key.toString() + "<" + key.getClass().toString() + ">)");
+        if (key.getClass().equals(UUID.class)) {
+            return NON_STRING_PREFIX + "u" + ((UUID) key).toString();
+        }
+
+        return super.getStringMapping(key);
+    }
+
+    @Override
+    public Object getKeyMapping(String key) {
+        logger.info("getKeyMapping(" + key + ")");
+
+        if (key.length() > 0 && key.charAt(0) == NON_STRING_PREFIX && key.charAt(1) == 'u') {
+            return UUID.fromString(key.substring(2));
+        }
+
+        return super.getKeyMapping(key);
+    }
+
+    @Override
+    public boolean isSupportedType(Class<?> keyType) {
+        return keyType == UUID.class || super.isSupportedType(keyType);
+    }
+}

startup-scripts/cache_owners.cli

@@ -1,24 +1,51 @@
 embed-server --server-config=standalone-ha.xml --std-out=echo
 batch
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:remove
-/subsystem=infinispan/cache-container=keycloak/replicated-cache=sessions:add()
-/subsystem=infinispan/cache-container=keycloak/replicated-cache=sessions:write-attribute(name="mode",value="SYNC")
+cd /subsystem=infinispan/cache-container=keycloak/
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:remove
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2})
+./distributed-cache=sessions:remove
+./distributed-cache=sessions:add()
+./distributed-cache=authenticationSessions:remove
+./distributed-cache=authenticationSessions:add()
+./distributed-cache=offlineSessions:remove
+./distributed-cache=offlineSessions:add()
+./distributed-cache=clientSessions:remove
+./distributed-cache=clientSessions:add()
+./distributed-cache=offlineClientSessions:remove
+./distributed-cache=offlineClientSessions:add()
+./distributed-cache=loginFailures:remove
+./distributed-cache=loginFailures:add()
+./distributed-cache=actionTokens:remove
+./distributed-cache=actionTokens:add()
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:remove
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2})
+./distributed-cache=sessions:write-attribute                                  (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=authenticationSessions:write-attribute                    (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=offlineSessions:write-attribute                           (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=clientSessions:write-attribute                            (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=offlineClientSessions:write-attribute                     (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=loginFailures:write-attribute                             (name=owners, value=${env.CACHE_OWNERS:2})
+./distributed-cache=actionTokens:write-attribute                              (name=owners, value=${env.CACHE_OWNERS:2})
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:remove
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2})
+./distributed-cache=sessions/store=string-jdbc:add                            (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=sessions/store=string-jdbc/table=string:add               (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:remove
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:add(mode="SYNC",owners=${env.CACHE_OWNERS:2})
+./distributed-cache=authenticationSessions/store=string-jdbc:add              (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=authenticationSessions/store=string-jdbc/table=string:add (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
 
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:remove
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(mode="SYNC",owners=${env.CACHE_OWNERS:2})
+./distributed-cache=offlineSessions/store=string-jdbc:add                     (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=offlineSessions/store=string-jdbc/table=string:add        (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
+
+./distributed-cache=clientSessions/store=string-jdbc:add                      (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=clientSessions/store=string-jdbc/table=string:add         (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
+
+./distributed-cache=offlineClientSessions/store=string-jdbc:add               (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=offlineClientSessions/store=string-jdbc/table=string:add  (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
+
+./distributed-cache=loginFailures/store=string-jdbc:add                       (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=loginFailures/store=string-jdbc/table=string:add          (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
+
+./distributed-cache=actionTokens/store=string-jdbc:add                        (data-source=KeycloakDS, preload=false, passivation=false, purge=false, shared=false, properties={key2StringMapper="de.coliquio.keycloak.ExtendedKeyMapper"})
+./distributed-cache=actionTokens/store=string-jdbc/table=string:add           (id-column={name="id", type="VARCHAR(255)"}, data-column={name="data", type="BLOB"},timestamp-column={name="timestamp", type="BIGINT"}, prefix="ISPN")
 
 run-batch
-stop-embedded-server
+stop-embedded-server