diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml index 26a3323144..a0d2bf027d 100644 --- a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml @@ -855,5 +855,23 @@ + + + db-failover-mariadb + + org.mariadb.jdbc + mariadb-java-client + 2.0.3 + keycloak + keycloak + + replication + localhost:3316,localhost:3326 + keycloak + + jdbc:mariadb:${mariadb.ha.mode}://${mariadb.hosts}/${mariadb.database}${mariadb.options} + + + diff --git a/testsuite/performance/db-failover/README.md b/testsuite/performance/db-failover/README.md new file mode 100644 index 0000000000..c378484cd2 --- /dev/null +++ b/testsuite/performance/db-failover/README.md @@ -0,0 +1,67 @@ +# DB Failover Testing Utilities + +A set of scripts for testing DB failover scenarios. + +The scripts expect to be run with relative execution prefix `./` from within the `db-failover` directory. + +Provisioned services are defined in `../docker-compose-db-failover.yml` template. + + +## Set the size of DB cluster + +Default size is 2 nodes. For a 3-node cluster run: `export NODES=3` before executing any of the scripts. + +For more than 3 nodes more service definitions need to be added to the docker-compose template. + +## Set up the environment + +Run `./setup.sh` + +This script will: +1. Start a bootstrap DB instance of MariaDB cluster +2. Start additional DB instances connected to the bootstrapped cluster +3. Stop the bootstrap DB instance +4. Optionally start Keycloak server + +Parameterized by environment variables: +- `MARIADB_HA_MODE` See: [MariaDB HA parameters](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-parameters) + Defaults to `replication`. +- `MARIADB_OPTIONS` See: [MariaDB HA options](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-options). + Use format: `?option1=value1[&option2=value2]...`. Default is an empty string. +- `START_KEYCLOAK` Default is `false`. Use `export START_KEYCLOAK=true` to enable. + +More options relevant to MariaDB clustering can be found in `../db/mariadb/wsrep.cnf`. + + +## Test the failover + +### Manual failover + +To induce a failure of specific DB node run: `./kill-node.sh X` where `X ∈ {1..3}` + +To reconnect the node back run: `./reconnect-node.sh X` + + +### Automated failover loop + +Run `./loop.sh` + +This script will run an infinite loop of failover/failback of DB nodes, switching to the next node in each loop. + +Parameterized by environment variables: +- `TIME_BETWEEN_FAILURES` Default is `60` (seconds). +- `FAILURE_DURATION` Default is `60` (seconds). + +To exit the script press `Ctrl+C`. + + +### Check number of table rows across the cluster + +Run: `./check-rows.sh` + + +## Tear down the environment + +Run `./teardown.sh` + +This will stop all services and delete the database. diff --git a/testsuite/performance/db-failover/check-rows.sh b/testsuite/performance/db-failover/check-rows.sh new file mode 100755 index 0000000000..215d387864 --- /dev/null +++ b/testsuite/performance/db-failover/check-rows.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +. ./common.sh + +CONCAT_SQL=$(< db-failover/concat.sql) +CONCAT_SQL_COMMAND='mysql -N -B -u keycloak --password=keycloak -e "$CONCAT_SQL" keycloak' +ROWS_SQL=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_1 $CONCAT_SQL_COMMAND | tr -dc '[:print:]') +ROWS_SQL=${ROWS_SQL%UNION } +ROWS_SQL_COMMAND='mysql -u keycloak --password=keycloak -e "$ROWS_SQL" keycloak' + +for (( i=1; i <= $NODES; i++)); do + ROWS[i]=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_$i $ROWS_SQL_COMMAND) +done + +DIFF=0 +for (( i=2; i <= $NODES; i++)); do + echo Node 1 vs Node $(( i )): + diff -y --suppress-common-lines <(echo "${ROWS[1]}") <(echo "${ROWS[i]}") + if [ $? -eq 0 ]; then echo No difference.; else DIFF=1; fi +done + +exit $DIFF diff --git a/testsuite/performance/db-failover/common.sh b/testsuite/performance/db-failover/common.sh new file mode 100755 index 0000000000..abdb8f9836 --- /dev/null +++ b/testsuite/performance/db-failover/common.sh @@ -0,0 +1,18 @@ +function killNode { + echo Killing mariadb_${1} + docker-compose -f docker-compose-db-failover.yml kill mariadb_${1} +} + +function reconnectNode { + N=$1 + NR=$(( N + 1 )); if [ "$NR" -gt "$NODES" ]; then NR=1; fi + export MARIADB_RUNNING_HOST=mariadb_${NR} + echo Attempting failback of mariadb_${N}, connecting to running cluster member mariadb_${NR} + docker-compose -f docker-compose-db-failover.yml up -d mariadb_${N} +} + +if [ -z $NODES ]; then export NODES=2; fi +if [ -z $MARIADB_OPTIONS ]; then export MARIADB_OPTIONS=""; fi +if [ -z $START_KEYCLOAK ]; then export START_KEYCLOAK=false; fi + +cd .. diff --git a/testsuite/performance/db-failover/concat.sql b/testsuite/performance/db-failover/concat.sql new file mode 100644 index 0000000000..f329ae8590 --- /dev/null +++ b/testsuite/performance/db-failover/concat.sql @@ -0,0 +1,3 @@ +SELECT CONCAT( + 'SELECT "', table_name, '" AS table_name, COUNT(*) AS exact_row_count FROM `', table_schema, '`.`', table_name, '` UNION ' +) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'keycloak'; diff --git a/testsuite/performance/db-failover/kill-node.sh b/testsuite/performance/db-failover/kill-node.sh new file mode 100755 index 0000000000..b47e39f3b4 --- /dev/null +++ b/testsuite/performance/db-failover/kill-node.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. ./common.sh + +if [ -z $1 ]; then echo "Specify DB node to kill."; exit 1; fi + +killNode $1 \ No newline at end of file diff --git a/testsuite/performance/db-failover/loop.sh b/testsuite/performance/db-failover/loop.sh new file mode 100755 index 0000000000..79813b24e3 --- /dev/null +++ b/testsuite/performance/db-failover/loop.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +. ./common.sh + +if [ -z "$TIME_BETWEEN_FAILURES" ]; then export TIME_BETWEEN_FAILURES=60; fi +if [ -z "$FAILURE_DURATION" ]; then export FAILURE_DURATION=60; fi + +echo Running DB failover loop with the following parameters: +echo NODES=$NODES +echo TIME_BETWEEN_FAILURES=$TIME_BETWEEN_FAILURES +echo FAILURE_DURATION=$FAILURE_DURATION +echo +echo Press Ctrl+C to interrupt. +echo + +N=1 + +while : +do + + killNode $N + + echo Waiting $FAILURE_DURATION s before attempting to reconnect mariadb_${N} + sleep $FAILURE_DURATION + + reconnectNode $N + + echo Waiting $TIME_BETWEEN_FAILURES s before inducing another failure. + echo + sleep $TIME_BETWEEN_FAILURES + + N=$((N+1)) + if [ "$N" -gt "$NODES" ]; then N=1; fi + +done \ No newline at end of file diff --git a/testsuite/performance/db-failover/reconnect-node.sh b/testsuite/performance/db-failover/reconnect-node.sh new file mode 100755 index 0000000000..6788a79ebd --- /dev/null +++ b/testsuite/performance/db-failover/reconnect-node.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. ./common.sh + +if [ -z $1 ]; then echo "Specify DB node to reconnect to cluster."; exit 1; fi + +reconnectNode $1 \ No newline at end of file diff --git a/testsuite/performance/db-failover/setup.sh b/testsuite/performance/db-failover/setup.sh new file mode 100755 index 0000000000..6f407ae055 --- /dev/null +++ b/testsuite/performance/db-failover/setup.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +. ./common.sh + +if [ -z "$DB_BOOTSTRAP_TIMEOUT" ]; then DB_BOOTSTRAP_TIMEOUT=10; fi +if [ -z "$DB_JOIN_TIMEOUT" ]; then DB_JOIN_TIMEOUT=5; fi + +echo Setting up Keycloak DB failover environment: + +echo Starting DB bootstrap instance. +docker-compose -f docker-compose-db-failover.yml up -d --build mariadb_bootstrap +echo Waiting $DB_BOOTSTRAP_TIMEOUT s for the DB to initialize. +sleep $DB_BOOTSTRAP_TIMEOUT + +MARIADB_HOSTS="" +for (( i=1; i<=$NODES; i++ )); do + + MARIADB_HOSTS=$MARIADB_HOSTS,mariadb_$i:3306 + + echo Starting DB node $i. + docker-compose -f docker-compose-db-failover.yml up -d mariadb_$i + echo Waiting $DB_JOIN_TIMEOUT s for the DB node to join + echo + sleep $DB_JOIN_TIMEOUT + +done + +echo Turning off the DB bootstrap instance. +docker-compose -f docker-compose-db-failover.yml stop mariadb_bootstrap + +export MARIADB_HOSTS=${MARIADB_HOSTS/,/} +echo MARIADB_HOSTS=$MARIADB_HOSTS + +if $START_KEYCLOAK; then + echo Starting Keycloak server. + docker-compose -f docker-compose-db-failover.yml up -d --build keycloak + ./healthcheck.sh +fi diff --git a/testsuite/performance/db-failover/teardown.sh b/testsuite/performance/db-failover/teardown.sh new file mode 100755 index 0000000000..ac472980f3 --- /dev/null +++ b/testsuite/performance/db-failover/teardown.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +. ./common.sh + +echo Stopping Keycloak DB failover environment. + +docker-compose -f docker-compose-db-failover.yml down -v + diff --git a/testsuite/performance/db/mariadb/wsrep.cnf b/testsuite/performance/db/mariadb/wsrep.cnf index 6a461c8246..52f09796c0 100644 --- a/testsuite/performance/db/mariadb/wsrep.cnf +++ b/testsuite/performance/db/mariadb/wsrep.cnf @@ -3,7 +3,7 @@ general_log=OFF bind-address=0.0.0.0 innodb_flush_log_at_trx_commit=0 -query_cache_size=0 +query_cache_size=0 query_cache_type=0 binlog_format=ROW @@ -15,7 +15,12 @@ innodb_buffer_pool_size=122M wsrep_on=ON wsrep_provider=/usr/lib/galera/libgalera_smm.so + wsrep_provider_options="gcache.size=300M; gcache.page_size=300M" +#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES" +#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES; pc.ignore_sb=TRUE" +# See: http://galeracluster.com/documentation-webpages/twonode.html + wsrep_cluster_address="gcomm://" wsrep_cluster_name="galera_cluster_keycloak" wsrep_sst_method=rsync diff --git a/testsuite/performance/docker-compose-cluster.yml b/testsuite/performance/docker-compose-cluster.yml index 4ecd95aaf6..490987893d 100644 --- a/testsuite/performance/docker-compose-cluster.yml +++ b/testsuite/performance/docker-compose-cluster.yml @@ -41,7 +41,7 @@ services: CONFIGURATION: standalone-ha.xml PUBLIC_SUBNET: 10.0.1.0/24 PRIVATE_SUBNET: 10.0.1.0/24 - MARIADB_HOST: mariadb + MARIADB_HOSTS: mariadb:3306 MARIADB_DATABASE: keycloak MARIADB_USER: keycloak MARIADB_PASSWORD: keycloak diff --git a/testsuite/performance/docker-compose-crossdc.yml b/testsuite/performance/docker-compose-crossdc.yml index 5075ad999f..4115efe67b 100644 --- a/testsuite/performance/docker-compose-crossdc.yml +++ b/testsuite/performance/docker-compose-crossdc.yml @@ -131,7 +131,7 @@ services: CONFIGURATION: standalone-ha.xml PUBLIC_SUBNET: 10.1.1.0/24 PRIVATE_SUBNET: 10.1.1.0/24 - MARIADB_HOST: mariadb_dc1 + MARIADB_HOSTS: mariadb_dc1:3306 MARIADB_DATABASE: keycloak MARIADB_USER: keycloak MARIADB_PASSWORD: keycloak @@ -172,7 +172,7 @@ services: CONFIGURATION: standalone-ha.xml PUBLIC_SUBNET: 10.2.1.0/24 PRIVATE_SUBNET: 10.2.1.0/24 - MARIADB_HOST: mariadb_dc2 + MARIADB_HOSTS: mariadb_dc2:3306 MARIADB_DATABASE: keycloak MARIADB_USER: keycloak MARIADB_PASSWORD: keycloak diff --git a/testsuite/performance/docker-compose-db-failover.yml b/testsuite/performance/docker-compose-db-failover.yml new file mode 100644 index 0000000000..23b0b714e4 --- /dev/null +++ b/testsuite/performance/docker-compose-db-failover.yml @@ -0,0 +1,101 @@ +version: "2.2" + +networks: + keycloak: + ipam: + config: + - subnet: 10.0.1.0/24 + + db_replication: + ipam: + config: + - subnet: 10.0.3.0/24 + +services: + + mariadb_bootstrap: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + networks: + - db_replication + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + MYSQL_DATABASE: keycloak + MYSQL_USER: keycloak + MYSQL_PASSWORD: keycloak + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep-new-cluster + + mariadb_1: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + networks: + - db_replication + - keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap} + ports: + - "3316:3306" + + mariadb_2: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + networks: + - db_replication + - keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap} + ports: + - "3326:3306" + + mariadb_3: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + networks: + - db_replication + - keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap} + ports: + - "3336:3306" + + + keycloak: + build: keycloak + image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} + networks: + - keycloak + environment: + MARIADB_HA_MODE: ${MARIADB_HA_MODE:-replication} + MARIADB_HOSTS: ${MARIADB_HOSTS:-mariadb_1:3306,mariadb_2:3306} + MARIADB_OPTIONS: ${MARIADB_OPTIONS} + MARIADB_DATABASE: keycloak + MARIADB_USER: keycloak + MARIADB_PASSWORD: keycloak + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: admin + # docker-compose syntax note: ${ENV_VAR:-} + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} + WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} + DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} + DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} + DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" + DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} + ports: + - "8080:8080" + - "8443:8443" + - "7989:7989" + - "9990:9990" + - "9999:9999" diff --git a/testsuite/performance/docker-compose.yml b/testsuite/performance/docker-compose.yml index 503f0d4191..93cc97ff54 100644 --- a/testsuite/performance/docker-compose.yml +++ b/testsuite/performance/docker-compose.yml @@ -35,7 +35,7 @@ services: networks: - keycloak environment: - MARIADB_HOST: mariadb + MARIADB_HOSTS: mariadb:3306 MARIADB_DATABASE: keycloak MARIADB_USER: keycloak MARIADB_PASSWORD: keycloak diff --git a/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli b/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli index cffb6d8425..35df67e2b7 100644 --- a/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli +++ b/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli @@ -2,7 +2,7 @@ cd /subsystem=datasources/data-source=KeycloakDS -:write-attribute(name=connection-url, value=jdbc:mariadb://${env.MARIADB_HOST:mariadb}:${env.MARIADB_PORT:3306}/${env.MARIADB_DATABASE:keycloak}) +:write-attribute(name=connection-url, value=jdbc:mariadb:${env.MARIADB_HA_MODE:}://${env.MARIADB_HOSTS:mariadb:3306}/${env.MARIADB_DATABASE:keycloak}${env.MARIADB_OPTIONS:}) :write-attribute(name=driver-name, value=mariadb) :write-attribute(name=user-name, value=${env.MARIADB_USER:keycloak}) :write-attribute(name=password, value=${env.MARIADB_PASSWORD:keycloak}) diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml index 9c46b175ac..fb84672d6a 100644 --- a/testsuite/performance/keycloak/pom.xml +++ b/testsuite/performance/keycloak/pom.xml @@ -34,6 +34,12 @@ Uses maven-dependency-plugin to unpack keycloak-server-dist artifact into `target/keycloak` which is then added to Docker image. + + org.keycloak + keycloak-server-dist + keycloak-${product.version} + + @@ -51,8 +57,8 @@ - org.keycloak - keycloak-server-dist + ${server.groupId} + ${server.artifactId} ${project.version} zip ${project.build.directory} @@ -77,7 +83,7 @@ - + @@ -87,4 +93,16 @@ + + + + integration-testsuite-server + + org.keycloak.testsuite + integration-arquillian-servers-auth-server-wildfly + auth-server-wildfly + + + + \ No newline at end of file