From 83136c1bd8c79621a5c3fea31cabc35313f5456d Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Tue, 14 Nov 2017 19:58:55 +0100 Subject: [PATCH] KEYCLOAK-5897 Support for multiple provisioning options in performance testsuite --- .../performance/README.docker-compose.md | 41 +- testsuite/performance/README.log-tool.md | 8 +- testsuite/performance/README.md | 46 +- .../README.provisioning-parameters.md | 140 +++-- .../performance/docker-compose-cluster.yml | 100 ---- .../performance/docker-compose-crossdc.yml | 236 -------- .../docker-compose-db-failover.yml | 2 +- testsuite/performance/healthcheck.sh | 34 -- testsuite/performance/keycloak/Dockerfile | 40 -- testsuite/performance/keycloak/configure.xml | 79 +++ testsuite/performance/keycloak/pom.xml | 102 +++- .../{ => src/main/resources}/module.xml | 4 +- .../keycloak/src/main/scripts/Dockerfile | 28 + .../main/scripts}/docker-entrypoint.sh | 0 .../{ => src/main/scripts}/get-ips.sh | 0 .../jboss-cli}/add-remote-cache-stores.cli | 0 .../jboss-cli}/distributed-cache-owners.cli | 0 .../jboss-cli}/infinispan-debug-logging.cli | 0 .../jboss-cli}/io-worker-threads-batch.cli | 0 .../scripts/jboss-cli}/io-worker-threads.cli | 0 .../modcluster-simple-load-provider.cli | 0 .../jboss-cli}/set-keycloak-ds-batch.cli | 0 .../scripts/jboss-cli}/set-keycloak-ds.cli | 0 .../scripts/jboss-cli}/undertow-batch.cli | 2 +- .../main/scripts/jboss-cli}/undertow.cli | 2 +- .../main/scripts}/keycloak-healthcheck.sh | 0 testsuite/performance/load-dump.sh | 26 - testsuite/performance/log-tool.sh | 5 - .../monitoring/cadvisor/Dockerfile | 1 + .../performance/monitoring/grafana/Dockerfile | 1 + testsuite/performance/prepare-dump.sh | 18 - testsuite/performance/tests/common.sh | 6 + .../tests/datasets/200ku200c.properties | 8 + .../tests/datasets/500ku500c.properties | 8 + testsuite/performance/tests/docker-compose.sh | 371 ++++++++++++ testsuite/performance/tests/healthcheck.sh | 49 ++ testsuite/performance/tests/log-tool.sh | 6 + testsuite/performance/tests/pom.xml | 545 ++++++++++-------- .../cluster/docker-compose-base.yml | 51 ++ .../cluster/docker-compose-keycloak.yml | 34 ++ .../crossdc/docker-compose-base.yml | 154 +++++ .../crossdc/docker-compose-keycloak_dc1.yml | 40 ++ .../crossdc/docker-compose-keycloak_dc2.yml | 35 ++ .../docker-compose-monitoring.yml | 0 .../main/docker-compose}/docker-compose.yml | 4 +- .../RealmsConfigurationLoader.java | 1 + .../org/keycloak/performance/TestConfig.java | 19 +- .../keycloak/AdminConsoleSimulation.scala | 2 +- .../scala/keycloak/DefaultSimulation.scala | 3 + 49 files changed, 1441 insertions(+), 810 deletions(-) delete mode 100644 testsuite/performance/docker-compose-cluster.yml delete mode 100644 testsuite/performance/docker-compose-crossdc.yml delete mode 100755 testsuite/performance/healthcheck.sh delete mode 100644 testsuite/performance/keycloak/Dockerfile create mode 100644 testsuite/performance/keycloak/configure.xml rename testsuite/performance/keycloak/{ => src/main/resources}/module.xml (91%) create mode 100644 testsuite/performance/keycloak/src/main/scripts/Dockerfile rename testsuite/performance/keycloak/{ => src/main/scripts}/docker-entrypoint.sh (100%) rename testsuite/performance/keycloak/{ => src/main/scripts}/get-ips.sh (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/add-remote-cache-stores.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/distributed-cache-owners.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/infinispan-debug-logging.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/io-worker-threads-batch.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/io-worker-threads.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/modcluster-simple-load-provider.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/set-keycloak-ds-batch.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/set-keycloak-ds.cli (100%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/undertow-batch.cli (95%) rename testsuite/performance/keycloak/{configs => src/main/scripts/jboss-cli}/undertow.cli (97%) rename testsuite/performance/keycloak/{ => src/main/scripts}/keycloak-healthcheck.sh (100%) delete mode 100755 testsuite/performance/load-dump.sh delete mode 100755 testsuite/performance/log-tool.sh delete mode 100755 testsuite/performance/prepare-dump.sh create mode 100755 testsuite/performance/tests/common.sh create mode 100644 testsuite/performance/tests/datasets/200ku200c.properties create mode 100644 testsuite/performance/tests/datasets/500ku500c.properties create mode 100755 testsuite/performance/tests/docker-compose.sh create mode 100755 testsuite/performance/tests/healthcheck.sh create mode 100755 testsuite/performance/tests/log-tool.sh create mode 100644 testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml create mode 100644 testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-keycloak.yml create mode 100644 testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml create mode 100644 testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc1.yml create mode 100644 testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc2.yml rename testsuite/performance/{ => tests/src/main/docker-compose}/docker-compose-monitoring.yml (100%) rename testsuite/performance/{ => tests/src/main/docker-compose}/docker-compose.yml (97%) diff --git a/testsuite/performance/README.docker-compose.md b/testsuite/performance/README.docker-compose.md index f0256e9971..b92e545696 100644 --- a/testsuite/performance/README.docker-compose.md +++ b/testsuite/performance/README.docker-compose.md @@ -1,35 +1,20 @@ -# Keycloak Performance Testsuite - Docker Compose +# Keycloak Performance Testsuite - Docker Compose Provisioner -## Requirements: -- Maven 3.1.1+ -- Unpacked Keycloak server distribution in `keycloak/target` folder. -- Docker 1.13+ -- Docker Compose 1.14+ +## Supported Deployments -### Keycloak Server Distribution -To unpack the current Keycloak server distribution into `keycloak/target` folder: -1. Build and install the distribution by running `mvn install -Pdistribution` from the root of the Keycloak project. -2. Unpack the installed artifact by running `mvn process-resources` from the Performance Testsuite module. +| Deployment | Available Operations | Orchestration Template | +| ------------ | ----------------------------------------------------- | ------------------------------- | +| `singlenode` | `provision`, `teardown`, `export-dump`, `import-dump` | `docker-compose.yml` | +| `cluster` | `provision`, `teardown`, `export-dump`, `import-dump` | `docker-compose-cluster.yml`* | +| `crossdc` | `provision`, `teardown`, `export-dump`, `import-dump` | `docker-compose-crossdc.yml`* | +| `monitoring` | `provision`, `teardown` | `docker-compose-monitoring.yml` | -## Deployments +The docker-compose orchestration templates are located in `tests/src/main/docker-compose` directory. -### Singlenode Deployment -- Build / rebuild: `docker-compose build` -- Start services: `docker-compose up -d --build` - Note: The `--build` parameter triggers a rebuild/restart of changed services if they are already running. -- Stop services: `docker-compose down -v`. If you wish to keep the container volumes skip the `-v` option. - -### Keycloak Cluster Deployment -- Build / rebuild: `docker-compose -f docker-compose-cluster.yml build` -- Start services: `docker-compose -f docker-compose-cluster.yml up -d --build` -- Scaling KC nodes: `docker-compose -f docker-compose-cluster.yml up -d --build --scale keycloak=2` -- Stop services: `docker-compose -f docker-compose-cluster.yml down -v`. If you wish to keep the container volumes skip the `-v` option. - -### Cross-DC Deployment -- Build / rebuild: `docker-compose -f docker-compose-crossdc.yml build` -- Start services: `docker-compose -f docker-compose-crossdc.yml up -d --build` -- Scaling KC nodes: `docker-compose -f docker-compose-crossdc.yml up -d --build --scale keycloak_dc1=2 --scale keycloak_dc2=3` -- Stop services: `docker-compose -f docker-compose-crossdc.yml down -v`. If you wish to keep the container volumes skip the `-v` option. +**[*]** The cluster and crossdc templates are generated dynamically during the `provision` operation based on provided `cpusets` parameter. +One Keycloak service entry is generated for each cpuset. This is a workaround for limitations of the default docker-compose scaling mechanism +which only allows setting `cpuset` per service, not per container. For more predictable performance results it is necessary for each +Keycloak server to have an exclusive access to specific CPU cores. ## Debugging docker containers: - List started containers: `docker ps`. It's useful to watch continuously: `watch docker ps`. diff --git a/testsuite/performance/README.log-tool.md b/testsuite/performance/README.log-tool.md index 07c23cf7fd..25c85b1a50 100644 --- a/testsuite/performance/README.log-tool.md +++ b/testsuite/performance/README.log-tool.md @@ -18,8 +18,8 @@ LOG_DIR=$HOME/devel/keycloak/keycloak/testsuite/performance/tests/target/gatling Get general statistics about the run to help with deciding about the interval to extract: ``` -./log-tool.sh -s -f $LOG_DIR/simulation.log -./log-tool.sh -s -f $LOG_DIR/simulation.log --lastRequest "Browser logout" +tests/log-tool.sh -s -f $LOG_DIR/simulation.log +tests/log-tool.sh -s -f $LOG_DIR/simulation.log --lastRequest "Browser logout" ``` Set start and end times for the extraction, and create new directory for results: @@ -34,9 +34,9 @@ mkdir $RESULT_DIR Extract a portion of the original log, and inspect statistics of resulting log: ``` -./log-tool.sh -f $LOG_DIR/simulation.log -o $RESULT_DIR/simulation-$FROM\_$TO.log -e --start $FROM --end $TO +tests/log-tool.sh -f $LOG_DIR/simulation.log -o $RESULT_DIR/simulation-$FROM\_$TO.log -e --start $FROM --end $TO -./log-tool.sh -f $RESULT_DIR/simulation-$FROM\_$TO.log -s +tests/log-tool.sh -f $RESULT_DIR/simulation-$FROM\_$TO.log -s ``` Generate another set of reports from extracted log: diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index 3eadbccef0..6884e4ab13 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -1,11 +1,13 @@ # Keycloak Performance Testsuite ## Requirements: +- Bash 2.05+ - Maven 3.1.1+ - Keycloak server distribution installed in the local Maven repository. To do this run `mvn install -Pdistribution` from the root of the Keycloak project. + +### Docker Compose Provisioner - Docker 1.13+ - Docker Compose 1.14+ -- Bash ## Getting started for the impatient @@ -47,30 +49,50 @@ Keep reading for more information. ## Provisioning +### Available provisioners: + +- `docker-compose` **Default.** See [`README.docker-compose.md`](README.docker-compose.md) for more details. + ### Provision -Usage: `mvn verify -Pprovision[,cluster] [-D= ...]`. +Usage: `mvn verify -Pprovision [-Dprovisioner=] [-D=] …`. -- Single node deployment: `mvn verify -Pprovision` -- Cluster deployment: `mvn verify -Pprovision,cluster [-Dkeycloak.scale=N]`. Default `N=1`. +#### Deployment Types -Available parameters are described in [README.provisioning-parameters](README.provisioning-parameters.md). +- Single node: `mvn verify -Pprovision` +- Cluster: `mvn verify -Pprovision,cluster [-Dkeycloak.scale=N] [-Dkeycloak.cpusets="cpuset1 cpuset2 … cpusetM"]`. `N ∈ {1 .. M}`. +- Cross-DC: `mvn verify -Pprovision,crossdc [-Dkeycloak.dc1.scale=K] [-Dkeycloak.dc2.scale=L] [-Dkeycloak.dc1.cpusets=…] [-Dkeycloak.dc2.cpusets=…]` + +All available parameters are described in [`README.provisioning-parameters.md`](README.provisioning-parameters.md). + +#### Provisioned System + +The `provision` operation will produce a `provisioned-system.properties` inside the `tests/target` directory +with information about the provisioned system such as the type of deployment and URLs of Keycloak servers and load balancers. +This information is then used by operations `generate-data`, `import-dump`, `test`, `teardown`. + +Provisioning can be run multiple times with different parameters. The system will be updated/reprovisioned based on the new parameters. +However when switching between different deployment types (e.g. from `singlenode` to `cluster`) it is always necessary +to tear down the currently running system. + +**Note:** When switching deployment type from `singlenode` or `cluster` to `crossdc` (or the other way around) +it is necessary to update the generated Keycloak server configuration (inside `keycloak/target` directory) by +adding a `clean` goal to the provisioning command like so: `mvn clean verify -Pprovision …`. It is *not* necessary to update this configuration +when switching between `singlenode` and `cluster` deployments. ### Teardown -Usage: `mvn verify -Pteardown[,cluster]` +Usage: `mvn verify -Pteardown [-Dprovisioner=]` -- Single node deployment: `mvn verify -Pteardown` -- Cluster deployment: `mvn verify -Pteardown,cluster` - -Provisioning/teardown is performed via `docker-compose` tool. More details in [README.docker-compose](README.docker-compose.md). +**Note:** Unless the provisioned system has been properly torn down the maven build will not allow a cleanup of the `tests/target` directory +because it contains the `provisioned-system.properties` with information about the still-running system. ## Testing ### Generate Test Data -Usage: `mvn verify -Pgenerate-data[,cluster] [-Ddataset=DATASET] [-Dexport-dump] [-D=]`. +Usage: `mvn verify -Pgenerate-data [-Ddataset=DATASET] [-D=]`. Dataset properties are loaded from `datasets/${dataset}.properties` file. Individual properties can be overriden by specifying `-D` params. @@ -97,7 +119,7 @@ Usage: `mvn verify -Pimport-dump [-Ddataset=DATASET]` ### Run Tests -Usage: `mvn verify -Ptest[,cluster] [-DrunUsers=N] [-DrampUpPeriod=SECONDS] [-DnumOfIterations=N] [-Ddataset=DATASET] [-D=]* [-D=]* `. +Usage: `mvn verify -Ptest [-DrunUsers=N] [-DrampUpPeriod=SECONDS] [-DnumOfIterations=N] [-Ddataset=DATASET] [-D=]* [-D=]* `. _*Note:* The same dataset properties which were used for data generation/import should be supplied to the `test` phase._ diff --git a/testsuite/performance/README.provisioning-parameters.md b/testsuite/performance/README.provisioning-parameters.md index eaea691f06..b291159099 100644 --- a/testsuite/performance/README.provisioning-parameters.md +++ b/testsuite/performance/README.provisioning-parameters.md @@ -1,48 +1,122 @@ # Keycloak Performance Testsuite - Provisioning Parameters -## Keycloak Server Settings: +## Overview of Provisioned Services -| Category | Setting | Property | Default value | -|-------------|-------------------------------|------------------------------------|------------------------------------------------------------------| -| JVM | Memory settings | `keycloak.jvm.memory` | -Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m | -| Undertow | HTTP Listener max connections | `keycloak.http.max-connections` | 500 | -| | AJP Listener max connections | `keycloak.ajp.max-connections` | 500 | -| IO | Worker IO thread pool | `keycloak.worker.io-threads` | 2 | -| | Worker Task thread pool | `keycloak.worker.task-max-threads` | 16 | -| Datasources | Connection pool min size | `keycloak.ds.min-pool-size` | 10 | -| | Connection pool max size | `keycloak.ds.max-pool-size` | 100 | -| | Connection pool prefill | `keycloak.ds.pool-prefill` | true | -| | Prepared statement cache size | `keycloak.ds.ps-cache-size` | 100 | +### Testing -## Load Balancer Settings: +| Deployment | Keycloak Server | Database | Load Balancer | Infinispan Server | +|-----------------|------------------------------------------|--------------------|--------------------|--------------------| +| *Singlenode* | 1 instance | 1 instance | - | - | +| *Cluster* | N instances | 1 instance | 1 instance | - | +| *Cross-DC* | K instances in DC1 + L instances in DC2 | 1 instance per DC | 1 instance per DC | 1 instance per DC | -| Category | Setting | Property | Default value | -|-------------|-------------------------------|---------------------------------------|------------------------------------------------------------------| -| JVM | Memory settings | `keycloak-lb.jvm.memory` | -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m | -| Undertow | HTTP Listener max connections | `keycloak-lb.http.max-connections` | 500 | -| IO | Worker IO thread pool | `keycloak-lb.worker.io-threads` | 2 | -| | Worker Task thread pool | `keycloak-lb.worker.task-max-threads` | 16 | +### Monitoring -## Infinispan Server Settings +| Deployment | CAdvisor | Influx DB | Grafana | +|-----------------|-------------|-------------|-------------| +| *Monitoring* | 1 instance | 1 instance | 1 instance | -| Category | Setting | Property | Default value | -|-------------|-------------------------------|-------------------------|-----------------------------------------------------------------------------------------| -| JVM | Memory settings | `infinispan.jvm.memory` | -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC | -## Docker settings +## Service Parameters + +### Keycloak Server + +| Category | Setting | Property | Default Value | +|-------------|-------------------------------|------------------------------------|--------------------------------------------------------------------| +| Scaling[1] | Scale for cluster | `keycloak.scale` | Maximum size[2] of cluster. | +| | Scale for DC1 | `keycloak.dc1.scale` | Maximum size of DC1. | +| | Scale for DC2 | `keycloak.dc2.scale` | Maximum size of DC2. | +| Docker | Allocated CPUs | `keycloak.docker.cpusets` | `2-3` for singlenode, `2 3` for cluster deployment | +| | Allocated CPUs for DC1 | `keycloak.dc1.docker.cpusets` | `2` | +| | Allocated CPUs for DC2 | `keycloak.dc2.docker.cpusets` | `3` | +| | Available memory | `keycloak.docker.memlimit` | `2500m` | +| JVM | Memory settings | `keycloak.jvm.memory` | `-Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m` | +| Undertow | HTTP Listener max connections | `keycloak.http.max-connections` | `50000` | +| | AJP Listener max connections | `keycloak.ajp.max-connections` | `50000` | +| IO | Worker IO thread pool | `keycloak.worker.io-threads` | `2` | +| | Worker Task thread pool | `keycloak.worker.task-max-threads` | `16` | +| Datasources | Connection pool min size | `keycloak.ds.min-pool-size` | `10` | +| | Connection pool max size | `keycloak.ds.max-pool-size` | `100` | +| | Connection pool prefill | `keycloak.ds.pool-prefill` | `true` | +| | Prepared statement cache size | `keycloak.ds.ps-cache-size` | `100` | + +**[ 1 ]** The scaling parameters are optional. They can be set within interval from 1 to the maximum cluster size]. +If not set they are automatically set to the maximum size of the cluster (DC1/DC2 respectively). + +**[ 2 ]** Maximum cluster size is determined by provisioner-specific parameter such as `keycloak.docker.cpusets` for the default *docker-compose* provisioner. +The maximum cluster size corresponds to the number of cpusets. + +### Database + +| Category | Setting | Property | Default Value | +|-------------|-------------------------------|------------------------------------|--------------------------------------------------------------------| +| Docker | Allocated CPUs | `db.docker.cpusets` | `1` | +| | Allocated CPUs for DC1 | `db.dc1.docker.cpusets` | `1` | +| | Allocated CPUs for DC2 | `db.dc2.docker.cpusets` | `1` | +| | Available memory | `db.docker.memlimit` | `2g` | + +### Load Balancer + +| Category | Setting | Property | Default Value | +|-------------|-------------------------------|------------------------------|--------------------------------------------------------------------| +| Docker | Allocated CPUs | `lb.docker.cpusets` | `1` | +| | Allocated CPUs for DC1 | `lb.dc1.docker.cpusets` | `1` | +| | Allocated CPUs for DC2 | `lb.dc2.docker.cpusets` | `1` | +| | Available memory | `lb.docker.memlimit` | `1g` | +| JVM | Memory settings | `lb.jvm.memory` | `-Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m` | +| Undertow | HTTP Listener max connections | `lb.http.max-connections` | `50000` | +| IO | Worker IO thread pool | `lb.worker.io-threads` | `2` | +| | Worker Task thread pool | `lb.worker.task-max-threads` | `16` | + +### Infinispan Server + +| Category | Setting | Property | Default Value | +|-------------|-------------------------------|---------------------------------|-------------------------------------------------------------------------------------------| +| Docker | Allocated CPUs for DC1 | `infinispan.dc1.docker.cpusets` | `1` | +| | Allocated CPUs for DC2 | `infinispan.dc2.docker.cpusets` | `1` | +| | Available memory | `infinispan.docker.memlimit` | `1500m` | +| JVM | Memory settings | `infinispan.jvm.memory` | `-Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC` | + +### Monitoring + +| Category | Setting | Property | Default Value | +|-------------|-------------------------------|-----------------------------|-----------------| +| Docker | Allocated CPUs | `monitoring.docker.cpusets` | `0` | + + +## Note on Docker settings By default, there are 4 CPU cores allocated: core 0 for monitoring, core 1 for database (MariaDB), and cores 2 and 3 for Keycloak server. Default memory limits for database and Keycloak server are 2g. The `cpuset` and `memlimit` parameters set here are set to `cpuset` and `mem_limit` parameters of docker-compose configuration. See docker-compose documentation for meaning of the values. How to set the parameters correctly depends on number of factors - number of cpu cores, NUMA, available memory etc., hence it is out of scope of this document. -| Container | Setting | Property | Default value | -|-------------|-------------------------------|---------------------------------|-------------------------------------------------------| -| Keycloak | Allocated CPUs | `keycloak.docker.cpuset` | 2-3 | -| | Allocated CPUs for DC1 | `keycloak.dc1.docker.cpuset` | 2-3 | -| | Allocated CPUs for DC2 | `keycloak.dc2.docker.cpuset` | 2-3 | -| | Available memory | `keycloak.docker.memlimit` | 2g | -| MariaDB | Allocated CPUs | `db.docker.cpuset` | 1 | -| | Available memory | `db.docker.memlimit` | 2g | -| Monitoring | Allocated CPUs | `monitoring.docker.cpuset` | 0 | +### Example CPU Settings +| HW | Development Machine | "Fat Box" | +|-------------|----------------------|--------------| +| Cores | 4 | 48 | +| NUMA Nodes | 0-3 | 0-23, 24-47 | + +#### Cluster + +| Setting | Development Machine | "Fat Box" | +|------------------------------------|----------------------|-----------------------------| +| `monitoring.docker.cpusets` | 0 | 0 | +| `db.docker.cpusets` | 1 | 1 | +| `lb.docker.cpusets` | 1 | 2 | +| `keycloak.docker.cpusets` | 2-3 | 3-6 7-10 11-16 … 43-46 | + +#### Cross-DC + +| Setting | Development Machine | "Fat Box" | +|------------------------------------|----------------------|--------------------------------| +| `monitoring.docker.cpusets` | 0 | 0 | +| `db.dc1.docker.cpusets` | 1 | 1 | +| `lb.dc1.docker.cpusets` | 1 | 2 | +| `infinispan.dc1.docker.cpusets` | 1 | 3 | +| `keycloak.dc1.docker.cpusets` | 2 | 4-7 8-11 12-15 16-19 20-23 | +| `db.dc2.docker.cpusets` | 1 | 24 | +| `lb.dc2.docker.cpusets` | 1 | 25 | +| `infinispan.dc2.docker.cpusets` | 1 | 26 | +| `keycloak.dc2.docker.cpusets` | 3 | 27-30 31-34 35-38 39-42 43-46 | diff --git a/testsuite/performance/docker-compose-cluster.yml b/testsuite/performance/docker-compose-cluster.yml deleted file mode 100644 index 490987893d..0000000000 --- a/testsuite/performance/docker-compose-cluster.yml +++ /dev/null @@ -1,100 +0,0 @@ -version: "2.2" - -networks: - keycloak: - ipam: - config: - - subnet: 10.0.1.0/24 - loadbalancing: - ipam: - config: - - subnet: 10.0.2.0/24 - -services: - - mariadb: - build: db/mariadb - image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} - cpuset: ${DB_CPUSET:-1} - mem_limit: ${DB_MEMLIMIT:-1g} - networks: - - keycloak - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: keycloak - MYSQL_USER: keycloak - MYSQL_PASSWORD: keycloak - ports: - - "3306:3306" - - keycloak: - build: keycloak - image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} - depends_on: - mariadb: - condition: service_healthy - cpuset: ${KEYCLOAK_CPUSET:-2-3} - mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} - networks: - - keycloak - environment: - CONFIGURATION: standalone-ha.xml - PUBLIC_SUBNET: 10.0.1.0/24 - PRIVATE_SUBNET: 10.0.1.0/24 - MARIADB_HOSTS: mariadb:3306 - MARIADB_DATABASE: keycloak - MARIADB_USER: keycloak - MARIADB_PASSWORD: keycloak - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} - AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} - DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} - DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} - DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" - DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} - ports: - - "8080" - - "9990" - - keycloak_lb: - build: load-balancer/wildfly-modcluster - image: keycloak_test_wildfly_modcluster:${KEYCLOAK_VERSION:-latest} - depends_on: - keycloak: - condition: service_healthy - cpus: 1 - networks: - - keycloak -# - loadbalancing - environment: - PRIVATE_SUBNET: 10.0.1.0/24 -# PUBLIC_SUBNET: 10.0.2.0/24 - JAVA_OPTS: ${KEYCLOAK_LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_LB_HTTP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_LB_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_LB_WORKER_TASK_MAX_THREADS:-16} - ports: - - "8080:8080" - -# advertize_server: -# build: load-balancer/debug-advertise -# image: keycloak_test_modcluster_advertize_server:${KEYCLOAK_VERSION:-latest} -# networks: -# - keycloak -# - loadbalancing -# command: 0.0.0.0 224.0.1.105 23365 -# -# advertize_client: -# build: -# context: load-balancer/debug-advertise -# dockerfile: Dockerfile-client -# image: keycloak_test_modcluster_advertize_client:${KEYCLOAK_VERSION:-latest} -# networks: -# - keycloak -# - loadbalancing -# command: 224.0.1.105 23364 diff --git a/testsuite/performance/docker-compose-crossdc.yml b/testsuite/performance/docker-compose-crossdc.yml deleted file mode 100644 index b20a6ebdca..0000000000 --- a/testsuite/performance/docker-compose-crossdc.yml +++ /dev/null @@ -1,236 +0,0 @@ -version: "2.2" - -networks: - # DC 1 - dc1_keycloak: - ipam: - config: - - subnet: 10.1.1.0/24 - # DC 2 - dc2_keycloak: - ipam: - config: - - subnet: 10.2.1.0/24 - # cross-DC - loadbalancing: - ipam: - config: - - subnet: 10.0.2.0/24 - # cross-DC - db_replication: - ipam: - config: - - subnet: 10.0.3.0/24 - # cross-DC - ispn_replication: - ipam: - config: - - subnet: 10.0.4.0/24 - -services: - - infinispan_dc1: - build: - context: infinispan - args: - LOCAL_SITE: dc1 - REMOTE_SITE: dc2 - image: keycloak_test_infinispan_dc1:${KEYCLOAK_VERSION:-latest} - cpus: 1 - networks: - - ispn_replication - - dc1_keycloak - environment: - PUBLIC_SUBNET: 10.1.1.0/24 - PRIVATE_SUBNET: 10.0.4.0/24 - MGMT_USER: admin - MGMT_USER_PASSWORD: admin - TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600] - JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - ports: - - "9991:9990" - - infinispan_dc2: - build: - context: infinispan - args: - LOCAL_SITE: dc2 - REMOTE_SITE: dc1 - image: keycloak_test_infinispan_dc2:${KEYCLOAK_VERSION:-latest} - depends_on: - infinispan_dc1: - condition: service_healthy - cpus: 1 - networks: - - ispn_replication - - dc2_keycloak - environment: - PUBLIC_SUBNET: 10.2.1.0/24 - PRIVATE_SUBNET: 10.0.4.0/24 - MGMT_USER: admin - MGMT_USER_PASSWORD: admin - TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600] - JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - ports: - - "9992:9990" - - - mariadb_dc1: - build: db/mariadb - image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} - cpus: 1 - networks: - - db_replication - - dc1_keycloak - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_INITDB_SKIP_TZINFO: foo - MYSQL_DATABASE: keycloak - MYSQL_USER: keycloak - MYSQL_PASSWORD: keycloak - entrypoint: docker-entrypoint-wsrep.sh - command: --wsrep-new-cluster - ports: - - "3306:3306" - - mariadb_dc2: - build: db/mariadb - image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} - depends_on: - mariadb_dc1: - condition: service_healthy - cpuset: ${DB_CPUSET:-1} - mem_limit: ${DB_MEMLIMIT:-1g} - networks: - - db_replication - - dc2_keycloak - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_INITDB_SKIP_TZINFO: foo - entrypoint: docker-entrypoint-wsrep.sh - command: --wsrep_cluster_address=gcomm://mariadb_dc1 - ports: - - "3307:3306" - - - - keycloak_dc1: - build: - context: ./keycloak - args: - REMOTE_CACHES: "true" - image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} - depends_on: - # wait for the db cluster to be ready before starting keycloak - mariadb_dc2: - condition: service_healthy - # wait for the ispn cluster to be ready before starting keycloak - infinispan_dc2: - condition: service_healthy - cpuset: ${KEYCLOAK_DC1_CPUSET:-2} - mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} - networks: - - dc1_keycloak - environment: - CONFIGURATION: standalone-ha.xml - PUBLIC_SUBNET: 10.1.1.0/24 - PRIVATE_SUBNET: 10.1.1.0/24 - MARIADB_HOSTS: mariadb_dc1:3306 - MARIADB_DATABASE: keycloak - MARIADB_USER: keycloak - MARIADB_PASSWORD: keycloak - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - INFINISPAN_HOST: infinispan_dc1 - SITE: dc1 - - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} - AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} - DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} - DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} - DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" - DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} - ports: - - "8080" - - "9990" - - - keycloak_dc2: - build: - context: ./keycloak - args: - REMOTE_CACHES: "true" - image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} - depends_on: - # wait for first kc instance to be ready before starting another - keycloak_dc1: - condition: service_healthy - cpuset: ${KEYCLOAK_DC2_CPUSET:-3} - mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} - networks: - - dc2_keycloak - environment: - CONFIGURATION: standalone-ha.xml - PUBLIC_SUBNET: 10.2.1.0/24 - PRIVATE_SUBNET: 10.2.1.0/24 - MARIADB_HOSTS: mariadb_dc2:3306 - MARIADB_DATABASE: keycloak - MARIADB_USER: keycloak - MARIADB_PASSWORD: keycloak - INFINISPAN_HOST: infinispan_dc2 - SITE: dc2 - - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} - AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} - DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} - DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} - DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" - DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} - ports: - - "8080" - - "9990" - - - - - - keycloak_lb_dc1: - build: load-balancer/wildfly-modcluster - image: keycloak_test_keycloak_lb:${KEYCLOAK_VERSION:-latest} - cpus: 1 - networks: - - dc1_keycloak -# - loadbalancing - environment: - PRIVATE_SUBNET: 10.1.1.0/24 -# PUBLIC_SUBNET: 10.0.2.0/24 - JAVA_OPTS: ${KEYCLOAK_LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_LB_HTTP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_LB_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_LB_WORKER_TASK_MAX_THREADS:-16} - ports: - - "8081:8080" - - keycloak_lb_dc2: - build: load-balancer/wildfly-modcluster - image: keycloak_test_keycloak_lb:${KEYCLOAK_VERSION:-latest} - cpus: 1 - networks: - - dc2_keycloak -# - loadbalancing - environment: - PRIVATE_SUBNET: 10.2.1.0/24 -# PUBLIC_SUBNET: 10.0.2.0/24 - JAVA_OPTS: ${KEYCLOAK_LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_LB_HTTP_MAX_CONNECTIONS:-500} - WORKER_IO_THREADS: ${KEYCLOAK_LB_WORKER_IO_THREADS:-2} - WORKER_TASK_MAX_THREADS: ${KEYCLOAK_LB_WORKER_TASK_MAX_THREADS:-16} - ports: - - "8082:8080" - diff --git a/testsuite/performance/docker-compose-db-failover.yml b/testsuite/performance/docker-compose-db-failover.yml index 23b0b714e4..81e7231e33 100644 --- a/testsuite/performance/docker-compose-db-failover.yml +++ b/testsuite/performance/docker-compose-db-failover.yml @@ -71,7 +71,7 @@ services: keycloak: - build: keycloak + build: keycloak/target/docker image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} networks: - keycloak diff --git a/testsuite/performance/healthcheck.sh b/testsuite/performance/healthcheck.sh deleted file mode 100755 index 870d79fdf8..0000000000 --- a/testsuite/performance/healthcheck.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -function isKeycloakHealthy { - echo Checking instance $1 - CODE=`curl -s -o /dev/null -w "%{http_code}" $1/realms/master` - [ "$CODE" -eq "200" ] -} - -function waitUntilSystemHealthy { - echo Waiting until all Keycloak instances are healthy... - if [ -z $1 ]; then ITERATIONS=60; else ITERATIONS=$1; fi - C=0 - SYSTEM_HEALTHY=false - while ! $SYSTEM_HEALTHY; do - C=$((C+1)) - if [ $C -gt $ITERATIONS ]; then - echo System healthcheck failed. - exit 1 - fi - sleep 2 - SYSTEM_HEALTHY=true - for URI in $KEYCLOAK_SERVER_URIS ; do - if ! isKeycloakHealthy $URI; then - echo Instance is not healthy. - SYSTEM_HEALTHY=false - fi - done - done - echo System is healthy. -} - -if [ -z "$KEYCLOAK_SERVER_URIS" ]; then KEYCLOAK_SERVER_URIS=http://localhost:8080/auth; fi - -waitUntilSystemHealthy diff --git a/testsuite/performance/keycloak/Dockerfile b/testsuite/performance/keycloak/Dockerfile deleted file mode 100644 index b89df12ef0..0000000000 --- a/testsuite/performance/keycloak/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM jboss/base-jdk:8 - -ENV JBOSS_HOME /opt/jboss/keycloak -ARG REMOTE_CACHES=false -WORKDIR $JBOSS_HOME - -ENV CONFIGURATION standalone.xml -ENV DEBUG_USER admin -ENV DEBUG_USER_PASSWORD admin - -# Enables signals getting passed from startup script to JVM -# ensuring clean shutdown when container is stopped. -ENV LAUNCH_JBOSS_IN_BACKGROUND 1 -ENV PROXY_ADDRESS_FORWARDING false - -ADD target/keycloak configs/ ./ -ADD *.sh /usr/local/bin/ - -USER root -RUN chown -R jboss .; chgrp -R jboss .; \ - chmod -R -v +x /usr/local/bin/ ; \ - yum install -y epel-release jq iproute && yum clean all - -USER jboss -# install mariadb JDBC driver -RUN curl --create-dirs --output modules/system/layers/base/org/mariadb/jdbc/main/mariadb-java-client-2.0.3.jar http://central.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.0.3/mariadb-java-client-2.0.3.jar ; \ - $JBOSS_HOME/bin/jboss-cli.sh --file=set-keycloak-ds.cli && \ - $JBOSS_HOME/bin/jboss-cli.sh --file=io-worker-threads.cli && \ - $JBOSS_HOME/bin/jboss-cli.sh --file=undertow.cli && \ - $JBOSS_HOME/bin/jboss-cli.sh --file=modcluster-simple-load-provider.cli && \ - if [ "$REMOTE_CACHES" == "true" ]; then $JBOSS_HOME/bin/jboss-cli.sh --file=add-remote-cache-stores.cli; fi && \ - cd $JBOSS_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp ; \ - $JBOSS_HOME/bin/add-user.sh -u $DEBUG_USER -p $DEBUG_USER_PASSWORD - -ADD module.xml modules/system/layers/base/org/mariadb/jdbc/main/ - -EXPOSE 8080 -EXPOSE 9990 -HEALTHCHECK --interval=5s --timeout=5s --retries=12 CMD ["keycloak-healthcheck.sh"] -ENTRYPOINT ["docker-entrypoint.sh"] \ No newline at end of file diff --git a/testsuite/performance/keycloak/configure.xml b/testsuite/performance/keycloak/configure.xml new file mode 100644 index 0000000000..b6572ff0d2 --- /dev/null +++ b/testsuite/performance/keycloak/configure.xml @@ -0,0 +1,79 @@ + + + + + performance.configured: ${performance.configured} + + + + keycloak-performance-configuration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + crossdc.configured: ${crossdc.configured} + + + + keycloak-crossdc-configuration + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml index 7c6ecb0983..8e520bb121 100644 --- a/testsuite/performance/keycloak/pom.xml +++ b/testsuite/performance/keycloak/pom.xml @@ -37,24 +37,43 @@ org.keycloak keycloak-server-dist - keycloak-${product.version} + ${project.build.directory}/keycloak-${product.version} + + org.mariadb.jdbc + mariadb-java-client + 2.0.3 + org/mariadb/jdbc + + sh + jboss-cli.${script.extension} + add-user.${script.extension} + + true + + false + false + + admin + admin + + ${project.build.scriptSourceDirectory} + ${project.basedir}/src/main/resources - + maven-dependency-plugin - unpack-keycloak-server-dist - generate-resources unpack + true ${server.groupId} @@ -66,6 +85,25 @@ + + copy-jdbc-driver + generate-resources + + copy + + + true + + + ${jdbc.driver.groupId} + ${jdbc.driver.artifactId} + ${jdbc.driver.version} + jar + ${server.unpacked.home}/modules/system/layers/base/${jdbc.driver.module.path}/main + + + + @@ -73,18 +111,40 @@ maven-antrun-plugin - - rename-keycloak-folder - + keycloak-performance-configuration process-resources run - - - + + + + + + keycloak-crossdc-configuration + process-resources + + run + + + ${skip.crossdc.configuration} + + + + + + + keycloak-docker + process-resources + + run + + + ${skip.keycloak.docker} + + @@ -95,14 +155,34 @@ + + windows + + + windows + + + + ps1 + + + + + crossdc + + false + + + integration-testsuite-server org.keycloak.testsuite integration-arquillian-servers-auth-server-wildfly - auth-server-wildfly + auth-server-wildfly + \ No newline at end of file diff --git a/testsuite/performance/keycloak/module.xml b/testsuite/performance/keycloak/src/main/resources/module.xml similarity index 91% rename from testsuite/performance/keycloak/module.xml rename to testsuite/performance/keycloak/src/main/resources/module.xml index d928fda293..3ba535904a 100644 --- a/testsuite/performance/keycloak/module.xml +++ b/testsuite/performance/keycloak/src/main/resources/module.xml @@ -20,9 +20,9 @@ ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org. --> - + - + diff --git a/testsuite/performance/keycloak/src/main/scripts/Dockerfile b/testsuite/performance/keycloak/src/main/scripts/Dockerfile new file mode 100644 index 0000000000..836ba08199 --- /dev/null +++ b/testsuite/performance/keycloak/src/main/scripts/Dockerfile @@ -0,0 +1,28 @@ +FROM jboss/base-jdk:8 + +ENV JBOSS_HOME /opt/jboss/keycloak +WORKDIR $JBOSS_HOME + +ENV CONFIGURATION standalone.xml + +# Enables signals getting passed from startup script to JVM +# ensuring clean shutdown when container is stopped. +ENV LAUNCH_JBOSS_IN_BACKGROUND 1 +ENV PROXY_ADDRESS_FORWARDING false + +USER root +RUN yum install -y epel-release jq iproute && yum clean all + +ADD keycloak ./ +ADD *.sh /usr/local/bin/ + +USER root +RUN chown -R jboss .; chgrp -R jboss .; chmod -R -v ug+x bin/*.sh ; \ + chmod -R -v ug+x /usr/local/bin/ + +USER jboss + +EXPOSE 8080 +EXPOSE 9990 +HEALTHCHECK --interval=5s --timeout=5s --retries=12 CMD ["keycloak-healthcheck.sh"] +ENTRYPOINT ["docker-entrypoint.sh"] \ No newline at end of file diff --git a/testsuite/performance/keycloak/docker-entrypoint.sh b/testsuite/performance/keycloak/src/main/scripts/docker-entrypoint.sh similarity index 100% rename from testsuite/performance/keycloak/docker-entrypoint.sh rename to testsuite/performance/keycloak/src/main/scripts/docker-entrypoint.sh diff --git a/testsuite/performance/keycloak/get-ips.sh b/testsuite/performance/keycloak/src/main/scripts/get-ips.sh similarity index 100% rename from testsuite/performance/keycloak/get-ips.sh rename to testsuite/performance/keycloak/src/main/scripts/get-ips.sh diff --git a/testsuite/performance/keycloak/configs/add-remote-cache-stores.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/add-remote-cache-stores.cli similarity index 100% rename from testsuite/performance/keycloak/configs/add-remote-cache-stores.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/add-remote-cache-stores.cli diff --git a/testsuite/performance/keycloak/configs/distributed-cache-owners.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/distributed-cache-owners.cli similarity index 100% rename from testsuite/performance/keycloak/configs/distributed-cache-owners.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/distributed-cache-owners.cli diff --git a/testsuite/performance/keycloak/configs/infinispan-debug-logging.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/infinispan-debug-logging.cli similarity index 100% rename from testsuite/performance/keycloak/configs/infinispan-debug-logging.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/infinispan-debug-logging.cli diff --git a/testsuite/performance/keycloak/configs/io-worker-threads-batch.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/io-worker-threads-batch.cli similarity index 100% rename from testsuite/performance/keycloak/configs/io-worker-threads-batch.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/io-worker-threads-batch.cli diff --git a/testsuite/performance/keycloak/configs/io-worker-threads.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/io-worker-threads.cli similarity index 100% rename from testsuite/performance/keycloak/configs/io-worker-threads.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/io-worker-threads.cli diff --git a/testsuite/performance/keycloak/configs/modcluster-simple-load-provider.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/modcluster-simple-load-provider.cli similarity index 100% rename from testsuite/performance/keycloak/configs/modcluster-simple-load-provider.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/modcluster-simple-load-provider.cli diff --git a/testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/set-keycloak-ds-batch.cli similarity index 100% rename from testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/set-keycloak-ds-batch.cli diff --git a/testsuite/performance/keycloak/configs/set-keycloak-ds.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/set-keycloak-ds.cli similarity index 100% rename from testsuite/performance/keycloak/configs/set-keycloak-ds.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/set-keycloak-ds.cli diff --git a/testsuite/performance/keycloak/configs/undertow-batch.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow-batch.cli similarity index 95% rename from testsuite/performance/keycloak/configs/undertow-batch.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow-batch.cli index 11e9a9187a..dac5833c5e 100644 --- a/testsuite/performance/keycloak/configs/undertow-batch.cli +++ b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow-batch.cli @@ -1,2 +1,2 @@ /subsystem=undertow/server=default-server/http-listener=default:write-attribute(name=proxy-address-forwarding, value=${env.PROXY_ADDRESS_FORWARDING}) -/subsystem=undertow/server=default-server/http-listener=default:write-attribute(name=max-connections,value=${env.HTTP_MAX_CONNECTIONS:500}) +/subsystem=undertow/server=default-server/http-listener=default:write-attribute(name=max-connections,value=${env.HTTP_MAX_CONNECTIONS:50000}) diff --git a/testsuite/performance/keycloak/configs/undertow.cli b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow.cli similarity index 97% rename from testsuite/performance/keycloak/configs/undertow.cli rename to testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow.cli index df04771905..edfe8eeb62 100644 --- a/testsuite/performance/keycloak/configs/undertow.cli +++ b/testsuite/performance/keycloak/src/main/scripts/jboss-cli/undertow.cli @@ -6,4 +6,4 @@ stop-embedded-server embed-server --server-config=standalone-ha.xml run-batch --file=undertow-batch.cli -/subsystem=undertow/server=default-server/ajp-listener=ajp:write-attribute(name=max-connections,value=${env.AJP_MAX_CONNECTIONS:500}) +/subsystem=undertow/server=default-server/ajp-listener=ajp:write-attribute(name=max-connections,value=${env.AJP_MAX_CONNECTIONS:50000}) diff --git a/testsuite/performance/keycloak/keycloak-healthcheck.sh b/testsuite/performance/keycloak/src/main/scripts/keycloak-healthcheck.sh similarity index 100% rename from testsuite/performance/keycloak/keycloak-healthcheck.sh rename to testsuite/performance/keycloak/src/main/scripts/keycloak-healthcheck.sh diff --git a/testsuite/performance/load-dump.sh b/testsuite/performance/load-dump.sh deleted file mode 100755 index c4363e1eba..0000000000 --- a/testsuite/performance/load-dump.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -DIRNAME=`dirname "$0"` -GATLING_HOME=$DIRNAME/tests - -if [ -z "$DATASET" ]; then - echo "Please specify DATASET env variable."; - exit 1; -fi - -if [ ! -f $GATLING_HOME/datasets/$DATASET.sql ]; then - if [ ! -f $GATLING_HOME/datasets/$DATASET.sql.gz ]; then - echo "Data dump file does not exist: $GATLING_HOME/datasets/$DATASET.sql(.gz)"; - echo "Usage: DATASET=[name] $0" - exit 1; - else - FILE=$GATLING_HOME/datasets/$DATASET.sql.gz - CMD="zcat $FILE" - fi -else - FILE=$GATLING_HOME/datasets/$DATASET.sql - CMD="cat $FILE" -fi - -echo "Importing dump file: $FILE" -echo "\$ $CMD | docker exec -i performance_mariadb_1 /usr/bin/mysql -u root --password=root keycloak" -$CMD | docker exec -i performance_mariadb_1 /usr/bin/mysql -u root --password=root keycloak diff --git a/testsuite/performance/log-tool.sh b/testsuite/performance/log-tool.sh deleted file mode 100755 index 0b9aa3c08e..0000000000 --- a/testsuite/performance/log-tool.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -DIRNAME=`dirname "$0"` -GATLING_HOME=$DIRNAME/tests - -java -cp $GATLING_HOME/target/classes org.keycloak.performance.log.LogProcessor "$@" \ No newline at end of file diff --git a/testsuite/performance/monitoring/cadvisor/Dockerfile b/testsuite/performance/monitoring/cadvisor/Dockerfile index 56cc630fa9..fa12b1523b 100644 --- a/testsuite/performance/monitoring/cadvisor/Dockerfile +++ b/testsuite/performance/monitoring/cadvisor/Dockerfile @@ -1,4 +1,5 @@ FROM google/cadvisor:v0.26.1 RUN apk add --no-cache bash curl ADD entrypoint.sh /entrypoint.sh +RUN chmod +x -v /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/testsuite/performance/monitoring/grafana/Dockerfile b/testsuite/performance/monitoring/grafana/Dockerfile index 545e19b583..20d2893859 100644 --- a/testsuite/performance/monitoring/grafana/Dockerfile +++ b/testsuite/performance/monitoring/grafana/Dockerfile @@ -4,4 +4,5 @@ ENV GF_DASHBOARDS_JSON_PATH /etc/grafana/dashboards/ COPY resource-usage-per-container.json /etc/grafana/dashboards/ COPY resource-usage-combined.json /etc/grafana/dashboards/ ADD entrypoint.sh /entrypoint.sh +RUN chmod +x -v /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/testsuite/performance/prepare-dump.sh b/testsuite/performance/prepare-dump.sh deleted file mode 100755 index 23bd99c9ce..0000000000 --- a/testsuite/performance/prepare-dump.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -DIRNAME=`dirname "$0"` -GATLING_HOME=$DIRNAME/tests - -if [ -z "$DATASET" ]; then - echo "This script requires DATASET env variable to be set" - exit 1 -fi - -echo "Exporting dump file" -docker exec performance_mariadb_1 /usr/bin/mysqldump -u root --password=root keycloak > $DATASET.sql -if [ $? -ne 0 ]; then - echo "Failed!" - exit $? -fi - -gzip $DATASET.sql -mv $DATASET.sql.gz $GATLING_HOME/datasets/ \ No newline at end of file diff --git a/testsuite/performance/tests/common.sh b/testsuite/performance/tests/common.sh new file mode 100755 index 0000000000..b64f46a897 --- /dev/null +++ b/testsuite/performance/tests/common.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +PROJECT_BASEDIR=${PROJECT_BASEDIR:-$(cd "$(dirname "$0")"; pwd)} +PROJECT_BUILD_DIRECTORY=$PROJECT_BASEDIR/target + +export PROVISIONED_SYSTEM_PROPERTIES_FILE="$PROJECT_BUILD_DIRECTORY/provisioned-system.properties" diff --git a/testsuite/performance/tests/datasets/200ku200c.properties b/testsuite/performance/tests/datasets/200ku200c.properties new file mode 100644 index 0000000000..9228099bcd --- /dev/null +++ b/testsuite/performance/tests/datasets/200ku200c.properties @@ -0,0 +1,8 @@ +numOfRealms=1 +usersPerRealm=200000 +clientsPerRealm=200 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/datasets/500ku500c.properties b/testsuite/performance/tests/datasets/500ku500c.properties new file mode 100644 index 0000000000..88dc49cd40 --- /dev/null +++ b/testsuite/performance/tests/datasets/500ku500c.properties @@ -0,0 +1,8 @@ +numOfRealms=1 +usersPerRealm=500000 +clientsPerRealm=500 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/docker-compose.sh b/testsuite/performance/tests/docker-compose.sh new file mode 100755 index 0000000000..9e0aebfc4c --- /dev/null +++ b/testsuite/performance/tests/docker-compose.sh @@ -0,0 +1,371 @@ +#!/bin/bash + +### FUNCTIONS ### + +function runCommand() { + echo "$1" + if ! eval "$1" ; then + echo "Execution of command failed." + echo "Command: \"$1\"" + exit 1; + fi +} + +function generateDockerComposeFile() { + echo "Generating $( basename "$DOCKER_COMPOSE_FILE" )" + local TEMPLATES_PATH=$DEPLOYMENT + cat $TEMPLATES_PATH/docker-compose-base.yml > $DOCKER_COMPOSE_FILE + case "$DEPLOYMENT" in + cluster) + I=0 + for CPUSET in $KEYCLOAK_CPUSETS ; do + I=$((I+1)) + sed -e s/%I%/$I/ -e s/%CPUSET%/$CPUSET/ $TEMPLATES_PATH/docker-compose-keycloak.yml >> $DOCKER_COMPOSE_FILE + done + ;; + crossdc) + I=0 + for CPUSET in $KEYCLOAK_DC1_CPUSETS ; do + I=$((I+1)) + sed -e s/%I%/$I/ -e s/%CPUSET%/$CPUSET/ $TEMPLATES_PATH/docker-compose-keycloak_dc1.yml >> $DOCKER_COMPOSE_FILE + done + I=0 + for CPUSET in $KEYCLOAK_DC2_CPUSETS ; do + I=$((I+1)) + sed -e s/%I%/$I/ -e s/%CPUSET%/$CPUSET/ $TEMPLATES_PATH/docker-compose-keycloak_dc2.yml >> $DOCKER_COMPOSE_FILE + done + ;; + esac +} + +function inspectDockerPortMapping() { + local PORT="$1" + local CONTAINER="$2" + local INSPECT_COMMAND="docker inspect --format='{{(index (index .NetworkSettings.Ports \"$PORT\") 0).HostPort}}' $CONTAINER" + MAPPED_PORT="$( eval "$INSPECT_COMMAND" )" + if [ -z "$MAPPED_PORT" ]; then + echo "Error finding mapped port for $CONTAINER." + exit 1 + fi +} + +function generateProvisionedSystemProperties() { + echo "Generating $PROVISIONED_SYSTEM_PROPERTIES_FILE" + echo "deployment=$DEPLOYMENT" > $PROVISIONED_SYSTEM_PROPERTIES_FILE + echo "keycloak.docker.services=$KEYCLOAK_SERVICES" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + case "$DEPLOYMENT" in + singlenode) + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_keycloak_1 + echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + ;; + cluster) + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_1 + echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + BACKEND_URLS="" + for SERVICE in $KEYCLOAK_SERVICES ; do + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_${SERVICE}_1 + BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth" + done + echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + ;; + crossdc) + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc1_1 + KC_DC1_PORT=$MAPPED_PORT + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc2_1 + KC_DC2_PORT=$MAPPED_PORT + echo "keycloak.frontend.servers=http://localhost:$KC_DC1_PORT/auth http://localhost:$KC_DC2_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + BACKEND_URLS="" + for SERVICE in $KEYCLOAK_SERVICES ; do + inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_${SERVICE}_1 + BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth" + done + echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + ;; + esac +} + +function loadProvisionedSystemProperties() { + if [ -f $PROVISIONED_SYSTEM_PROPERTIES_FILE ]; then + echo "Loading $PROVISIONED_SYSTEM_PROPERTIES_FILE" + export DEPLOYMENT=$( grep -Po "(?<=^deployment=).*" $PROVISIONED_SYSTEM_PROPERTIES_FILE ) + export KEYCLOAK_SERVICES=$( grep -Po "(?<=^keycloak.docker.services=).*" $PROVISIONED_SYSTEM_PROPERTIES_FILE ) + else + echo "$PROVISIONED_SYSTEM_PROPERTIES_FILE not found." + fi +} + +function removeProvisionedSystemProperties() { + rm -f $PROVISIONED_SYSTEM_PROPERTIES_FILE +} + +function isTestDeployment() { + IS_TEST_DEPLOYMENT=false + case "$DEPLOYMENT" in + singlenode|cluster|crossdc) IS_TEST_DEPLOYMENT=true ;; + esac + $IS_TEST_DEPLOYMENT +} + +function validateNotEmpty() { + VARIABLE_NAME=$1 + VARIABLE_VALUE=$2 +# echo "$VARIABLE_NAME: $VARIABLE_VALUE" + if [ -z "$VARIABLE_VALUE" ]; then echo "$VARIABLE_NAME must contain at least one item."; exit 1; fi +} + + +### SCRIPT ### + +cd "$(dirname "$0")" +. ./common.sh + +cd $PROJECT_BUILD_DIRECTORY/docker-compose + +PROJECT_NAME=performance + + +export DEPLOYMENT="${DEPLOYMENT:-singlenode}" +if isTestDeployment ; then loadProvisionedSystemProperties; fi +export OPERATION="${OPERATION:-provision}" + +echo "DEPLOYMENT: $DEPLOYMENT" + +case "$DEPLOYMENT" in + singlenode) DOCKER_COMPOSE_FILE=docker-compose.yml ;; + cluster) DOCKER_COMPOSE_FILE=docker-compose-cluster.yml ;; + crossdc) DOCKER_COMPOSE_FILE=docker-compose-crossdc.yml ;; + monitoring) DOCKER_COMPOSE_FILE=docker-compose-monitoring.yml ; DELETE_DATA="${DELETE_DATA:-false}" ;; + *) + echo "Deployment '$DEPLOYMENT' not supported by provisioner '$PROVISIONER'." + exit 1 + ;; +esac + + +echo "OPERATION: $OPERATION" + +case "$OPERATION" in + + provision) + + case "$DEPLOYMENT" in + + singlenode) + BASE_SERVICES="mariadb" + KEYCLOAK_SERVICES="keycloak" + + validateNotEmpty DB_CPUSETS $DB_CPUSETS + DB_CPUSETS_ARRAY=( $DB_CPUSETS ) + DB_CPUSET=${DB_CPUSETS_ARRAY[0]} + + validateNotEmpty KEYCLOAK_CPUSETS $KEYCLOAK_CPUSETS + KEYCLOAK_CPUSETS_ARRAY=( $KEYCLOAK_CPUSETS ) + KEYCLOAK_CPUSET=${KEYCLOAK_CPUSETS_ARRAY[0]} + + echo "DB_CPUSET: $DB_CPUSET" + echo "KEYCLOAK_CPUSET: $KEYCLOAK_CPUSET" + echo "BASE_SERVICES: $BASE_SERVICES" + echo "KEYCLOAK_SERVICES: $KEYCLOAK_SERVICES" + + ;; + + cluster) + BASE_SERVICES="mariadb loadbalancer" + + validateNotEmpty DB_CPUSETS $DB_CPUSETS + DB_CPUSETS_ARRAY=( $DB_CPUSETS ) + DB_CPUSET=${DB_CPUSETS_ARRAY[0]} + + validateNotEmpty LB_CPUSETS $LB_CPUSETS + LB_CPUSETS_ARRAY=( $LB_CPUSETS ) + LB_CPUSET=${LB_CPUSETS_ARRAY[0]} + + validateNotEmpty KEYCLOAK_CPUSETS $KEYCLOAK_CPUSETS + KEYCLOAK_CPUSETS_ARRAY=( $KEYCLOAK_CPUSETS ) + + KEYCLOAK_MAX_SCALE=${#KEYCLOAK_CPUSETS_ARRAY[@]} + KEYCLOAK_SCALE="${KEYCLOAK_SCALE:-$KEYCLOAK_MAX_SCALE}" + if [ $KEYCLOAK_SCALE -gt $KEYCLOAK_MAX_SCALE ]; then KEYCLOAK_SCALE=$KEYCLOAK_MAX_SCALE; fi + if [ $KEYCLOAK_SCALE -lt 1 ]; then KEYCLOAK_SCALE=1; fi + + echo "DB_CPUSET: $DB_CPUSET" + echo "KEYCLOAK_CPUSETS: $KEYCLOAK_CPUSETS" + echo "KEYCLOAK_SCALE: ${KEYCLOAK_SCALE} (max ${KEYCLOAK_MAX_SCALE})" + + KEYCLOAK_SERVICES="" + STOPPED_KEYCLOAK_SERVICES="" + for ((i=1; i<=$KEYCLOAK_MAX_SCALE; i++)) ; do + if (( $i <= $KEYCLOAK_SCALE )) ; then + KEYCLOAK_SERVICES="$KEYCLOAK_SERVICES keycloak_$i" + else + STOPPED_KEYCLOAK_SERVICES="$STOPPED_KEYCLOAK_SERVICES keycloak_$i" + fi + done + echo "BASE_SERVICES: $BASE_SERVICES" + echo "KEYCLOAK_SERVICES: $KEYCLOAK_SERVICES" + + generateDockerComposeFile + ;; + + crossdc) + BASE_SERVICES="mariadb_dc1 mariadb_dc2 infinispan_dc1 infinispan_dc2 loadbalancer_dc1 loadbalancer_dc2" + + validateNotEmpty DB_DC1_CPUSETS $DB_DC1_CPUSETS + validateNotEmpty DB_DC2_CPUSETS $DB_DC2_CPUSETS + DB_DC1_CPUSETS_ARRAY=( $DB_DC1_CPUSETS ) + DB_DC2_CPUSETS_ARRAY=( $DB_DC2_CPUSETS ) + DB_DC1_CPUSET=${DB_DC1_CPUSETS_ARRAY[0]} + DB_DC2_CPUSET=${DB_DC2_CPUSETS_ARRAY[0]} + echo "DB_DC1_CPUSET: $DB_DC1_CPUSET" + echo "DB_DC2_CPUSET: $DB_DC2_CPUSET" + + validateNotEmpty LB_DC1_CPUSETS $LB_DC1_CPUSETS + validateNotEmpty LB_DC2_CPUSETS $LB_DC2_CPUSETS + LB_DC1_CPUSETS_ARRAY=( $LB_DC1_CPUSETS ) + LB_DC2_CPUSETS_ARRAY=( $LB_DC2_CPUSETS ) + LB_DC1_CPUSET=${LB_DC1_CPUSETS_ARRAY[0]} + LB_DC2_CPUSET=${LB_DC2_CPUSETS_ARRAY[0]} + echo "LB_DC1_CPUSET: $LB_DC1_CPUSET" + echo "LB_DC2_CPUSET: $LB_DC2_CPUSET" + + validateNotEmpty INFINISPAN_DC1_CPUSETS $INFINISPAN_DC1_CPUSETS + validateNotEmpty INFINISPAN_DC2_CPUSETS $INFINISPAN_DC2_CPUSETS + INFINISPAN_DC1_CPUSETS_ARRAY=( $INFINISPAN_DC1_CPUSETS ) + INFINISPAN_DC2_CPUSETS_ARRAY=( $INFINISPAN_DC2_CPUSETS ) + INFINISPAN_DC1_CPUSET=${INFINISPAN_DC1_CPUSETS_ARRAY[0]} + INFINISPAN_DC2_CPUSET=${INFINISPAN_DC2_CPUSETS_ARRAY[0]} + echo "INFINISPAN_DC1_CPUSET: $INFINISPAN_DC1_CPUSET" + echo "INFINISPAN_DC2_CPUSET: $INFINISPAN_DC2_CPUSET" + + validateNotEmpty KEYCLOAK_DC1_CPUSETS $KEYCLOAK_DC1_CPUSETS + validateNotEmpty KEYCLOAK_DC2_CPUSETS $KEYCLOAK_DC2_CPUSETS + KEYCLOAK_DC1_CPUSETS_ARRAY=( $KEYCLOAK_DC1_CPUSETS ) + KEYCLOAK_DC2_CPUSETS_ARRAY=( $KEYCLOAK_DC2_CPUSETS ) + KEYCLOAK_DC1_MAX_SCALE=${#KEYCLOAK_DC1_CPUSETS_ARRAY[@]} + KEYCLOAK_DC2_MAX_SCALE=${#KEYCLOAK_DC2_CPUSETS_ARRAY[@]} + KEYCLOAK_DC1_SCALE="${KEYCLOAK_DC1_SCALE:-$KEYCLOAK_DC1_MAX_SCALE}" + KEYCLOAK_DC2_SCALE="${KEYCLOAK_DC2_SCALE:-$KEYCLOAK_DC2_MAX_SCALE}" + + if [ $KEYCLOAK_DC1_SCALE -gt $KEYCLOAK_DC1_MAX_SCALE ]; then KEYCLOAK_DC1_SCALE=$KEYCLOAK_DC1_MAX_SCALE; fi + if [ $KEYCLOAK_DC1_SCALE -lt 1 ]; then KEYCLOAK_DC1_SCALE=1; fi + if [ $KEYCLOAK_DC2_SCALE -gt $KEYCLOAK_DC2_MAX_SCALE ]; then KEYCLOAK_DC2_SCALE=$KEYCLOAK_DC2_MAX_SCALE; fi + if [ $KEYCLOAK_DC2_SCALE -lt 1 ]; then KEYCLOAK_DC2_SCALE=1; fi + + echo "KEYCLOAK_DC1_CPUSETS: ${KEYCLOAK_DC1_CPUSETS}" + echo "KEYCLOAK_DC2_CPUSETS: ${KEYCLOAK_DC2_CPUSETS}" + echo "KEYCLOAK_DC1_SCALE: ${KEYCLOAK_DC1_SCALE} (max ${KEYCLOAK_DC1_MAX_SCALE})" + echo "KEYCLOAK_DC2_SCALE: ${KEYCLOAK_DC2_SCALE} (max ${KEYCLOAK_DC2_MAX_SCALE})" + + KEYCLOAK_SERVICES="" + STOPPED_KEYCLOAK_SERVICES="" + for ((i=1; i<=$KEYCLOAK_DC1_MAX_SCALE; i++)) ; do + if (( $i <= $KEYCLOAK_DC1_SCALE )) ; then + KEYCLOAK_SERVICES="$KEYCLOAK_SERVICES keycloak_dc1_$i" + else + STOPPED_KEYCLOAK_SERVICES="$STOPPED_KEYCLOAK_SERVICES keycloak_dc1_$i" + fi + done + for ((i=1; i<=$KEYCLOAK_DC2_MAX_SCALE; i++)) ; do + if (( $i <= $KEYCLOAK_DC2_SCALE )) ; then + KEYCLOAK_SERVICES="$KEYCLOAK_SERVICES keycloak_dc2_$i" + else + STOPPED_KEYCLOAK_SERVICES="$STOPPED_KEYCLOAK_SERVICES keycloak_dc2_$i" + fi + done + echo "BASE_SERVICES: $BASE_SERVICES" + echo "KEYCLOAK_SERVICES: $KEYCLOAK_SERVICES" + + generateDockerComposeFile + + ;; + + esac + + runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} up -d --build $BASE_SERVICES $KEYCLOAK_SERVICES" + if [ ! -z "$STOPPED_KEYCLOAK_SERVICES" ] ; then + echo "STOPPED_KEYCLOAK_SERVICES: $STOPPED_KEYCLOAK_SERVICES" + runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} stop $STOPPED_KEYCLOAK_SERVICES" + fi + + if isTestDeployment ; then + generateProvisionedSystemProperties; + $PROJECT_BASEDIR/healthcheck.sh + fi + + ;; + + + teardown) + + DELETE_DATA="${DELETE_DATA:-true}" + echo "DELETE_DATA: $DELETE_DATA" + if "$DELETE_DATA" ; then VOLUMES_ARG="-v"; else VOLUMES_ARG=""; fi + + runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} down $VOLUMES_ARG" + + if isTestDeployment ; then removeProvisionedSystemProperties; fi + ;; + + + export-dump|import-dump) + + loadProvisionedSystemProperties + + echo "KEYCLOAK_SERVICES: $KEYCLOAK_SERVICES" + if [ -z "$KEYCLOAK_SERVICES" ]; then echo "Unable to load KEYCLOAK_SERVICES"; exit 1; fi + + case "$DEPLOYMENT" in + singlenode|cluster) export DB_CONTAINER=${PROJECT_NAME}_mariadb_1 ;; + crossdc) export DB_CONTAINER=${PROJECT_NAME}_mariadb_dc1_1 ;; + *) echo "Deployment '$DEPLOYMENT' doesn't support operation '$OPERATION'." ; exit 1 ;; + esac + if [ -z "$DATASET" ]; then echo "Operation '$OPERATION' requires DATASET parameter."; exit 1; fi + echo "DATASET: $DATASET" + + echo "Stopping Keycloak services." + runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} stop $KEYCLOAK_SERVICES" + + cd $PROJECT_BASEDIR/datasets + case "$OPERATION" in + export-dump) + echo "Exporting $DATASET.sql." + if docker exec $DB_CONTAINER /usr/bin/mysqldump -u root --password=root keycloak > $DATASET.sql ; then + echo "Compressing $DATASET.sql." + gzip $DATASET.sql + fi + ;; + import-dump) + DUMP_DOWNLOAD_SITE=${DUMP_DOWNLOAD_SITE:-https://downloads.jboss.org/keycloak-qe} + if [ ! -f "$DATASET.sql.gz" ]; then + echo "Downloading dump file." + if ! curl -f -O $DUMP_DOWNLOAD_SITE/$DATASET.properties -O $DUMP_DOWNLOAD_SITE/$DATASET.sql.gz ; then + echo Download failed. + exit 1 + fi + fi + echo "Importing $DATASET.sql.gz" + set -o pipefail + if ! zcat $DATASET.sql.gz | docker exec -i $DB_CONTAINER /usr/bin/mysql -u root --password=root keycloak ; then + echo Import failed. + exit 1 + fi + ;; + esac + cd $PROJECT_BUILD_DIRECTORY/docker-compose + + echo "Starting Keycloak services." + runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} up -d --no-recreate $KEYCLOAK_SERVICES" + # need to update mapped ports + generateProvisionedSystemProperties + + $PROJECT_BASEDIR/healthcheck.sh + + ;; + + *) + echo "Unsupported operation: '$OPERATION'" + exit 1 + ;; + +esac + diff --git a/testsuite/performance/tests/healthcheck.sh b/testsuite/performance/tests/healthcheck.sh new file mode 100755 index 0000000000..cf14634275 --- /dev/null +++ b/testsuite/performance/tests/healthcheck.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +cd "$(dirname "$0")" +. ./common.sh + +CHECK_TIMEOUT=${CHECK_TIMEOUT:-5} +function isKeycloakServerReady { + CODE=`curl --connect-timeout $CHECK_TIMEOUT -m $CHECK_TIMEOUT -s -o /dev/null -w "%{http_code}" $1/realms/master` + [ "$CODE" -eq "200" ] +} + + +if [ -f "$PROVISIONED_SYSTEM_PROPERTIES_FILE" ] ; then + + FRONTEND_SERVERS=$( grep -Po "(?<=^keycloak.frontend.servers=).*" "$PROVISIONED_SYSTEM_PROPERTIES_FILE" ) + BACKEND_SERVERS=$( grep -Po "(?<=^keycloak.backend.servers=).*" "$PROVISIONED_SYSTEM_PROPERTIES_FILE" ) + KEYCLOAK_SERVERS="$FRONTEND_SERVERS $BACKEND_SERVERS" + + HEALTHCHECK_ITERATIONS=${HEALTHCHECK_ITERATIONS:-20} + HEALTHCHECK_WAIT=${HEALTHCHECK_WAIT:-6s} + + echo "Waiting for Keycloak servers to be ready." + echo "Check intervals: $HEALTHCHECK_ITERATIONS x $HEALTHCHECK_WAIT." + + C=0 + READY=false + while ! $READY; do + C=$((C+1)) + if [ $C -gt $HEALTHCHECK_ITERATIONS ]; then + echo System healthcheck failed. + exit 1 + fi + echo $( date -Iseconds ) + READY=true + for SERVER in $KEYCLOAK_SERVERS ; do + if isKeycloakServerReady $SERVER; then + echo -e "Keycloak server: $SERVER\tREADY" + else + echo -e "Keycloak server: $SERVER\tNOT READY" + READY=false + fi + done + if ! $READY; then sleep $HEALTHCHECK_WAIT ; fi + done + echo All servers ready. + +else + echo Healthcheck skipped. +fi diff --git a/testsuite/performance/tests/log-tool.sh b/testsuite/performance/tests/log-tool.sh new file mode 100755 index 0000000000..c36cfe2ece --- /dev/null +++ b/testsuite/performance/tests/log-tool.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd "$(dirname "$0")" +. ./common.sh + +java -cp $PROJECT_BUILD_DIRECTORY/classes org.keycloak.performance.log.LogProcessor "$@" \ No newline at end of file diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml index fa5cef842c..970a1499f3 100644 --- a/testsuite/performance/tests/pom.xml +++ b/testsuite/performance/tests/pom.xml @@ -29,19 +29,23 @@ performance-tests Keycloak Performance TestSuite - Tests - - - - docker-compose.yml - - keycloak - http://localhost:8080/auth - jdbc:mariadb://keycloak:keycloak@localhost:3306/keycloak + docker-compose + singlenode + + ${project.build.directory}/provisioned-system.properties + + + + + 2-3 + 2 + 3 + 2500m -Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m - 500 - 500 + 50000 + 50000 2 16 10 @@ -49,24 +53,32 @@ true 100 - -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m - 500 - 2 - 16 - - - 1 - 2-3 - 2 - 3 - 0 - + + 1 + 1 + 1 2g - 2g - - -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC + + 1 + 1 + 1 + 1g + -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m + 500 + 2 + 16 + + 1 + 1 + 1500m + -Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC + + + 0 + + default 1 @@ -156,6 +168,58 @@ + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-teardown-before-clean + + enforce + + pre-clean + + + + + + ${provisioned.system.properties.file} + + + + true + + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + read-existing-provisioned-system-properties + initialize + + read-project-properties + + + + ${project.build.directory}/provisioned-system.properties + + true + + + + + net.alchim31.maven scala-maven-plugin @@ -191,12 +255,11 @@ - @@ -213,6 +276,7 @@ keycloak.DemoSimulation2 + -Dkeycloak.server.uris=${keycloak.frontend.servers} -DnumOfRealms=${numOfRealms} -DusersPerRealm=${usersPerRealm} -DclientsPerRealm=${clientsPerRealm} @@ -233,30 +297,163 @@ - + org.codehaus.mojo - properties-maven-plugin - 1.0.0 - - - read-dataset-properties - initialize - - read-project-properties - - - - ${project.basedir}/datasets/${dataset}.properties - - - - - + exec-maven-plugin + + ${project.basedir} + + + + + docker-compose + + + !provisioner + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + copy-dockerfiles-etc + generate-resources + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + cluster + + cluster + 2 3 + + + + crossdc + + crossdc + + + + + provision + + + + org.codehaus.mojo + exec-maven-plugin + + + provision + process-test-resources + + exec + + + ./${provisioner}.sh + + ${provisioner} + ${deployment} + provision + + ${project.version} + + ${keycloak.scale} + ${keycloak.dc1.scale} + ${keycloak.dc2.scale} + ${keycloak.docker.cpusets} + ${keycloak.dc1.docker.cpusets} + ${keycloak.dc2.docker.cpusets} + ${keycloak.docker.memlimit} + ${keycloak.jvm.memory} + ${keycloak.http.max-connections} + ${keycloak.ajp.max-connections} + ${keycloak.worker.io-threads} + ${keycloak.worker.task-max-threads} + ${keycloak.ds.min-pool-size} + ${keycloak.ds.max-pool-size} + ${keycloak.ds.pool-prefill} + ${keycloak.ds.ps-cache-size} + + ${db.docker.cpusets} + ${db.dc1.docker.cpusets} + ${db.dc2.docker.cpusets} + ${db.docker.memlimit} + + ${lb.docker.cpusets} + ${lb.dc1.docker.cpusets} + ${lb.dc2.docker.cpusets} + ${lb.docker.memlimit} + ${lb.jvm.memory} + ${lb.http.max-connections} + ${lb.worker.io-threads} + ${lb.worker.task-max-threads} + + ${infinispan.dc1.docker.cpusets} + ${infinispan.dc2.docker.cpusets} + ${infinispan.docker.memlimit} + ${infinispan.jvm.memory} + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0.0 + + + read-new-provisioned-system-properties + + read-project-properties + + pre-integration-test + + + ${project.build.directory}/provisioned-system.properties + + + + + + + + + initialize-dataset-properties @@ -273,127 +470,19 @@ initialize-dataset-properties - initialize + pre-integration-test read-project-properties - ${project.basedir}/datasets/${dataset}.properties + ${project.basedir}/../datasets/${dataset}.properties + true - - - - - - - cluster - - docker-compose-cluster.yml - 1 - --scale keycloak=${keycloak.scale} - http://localhost:8080/auth - - - - crossdc - - docker-compose-crossdc.yml - 1 - 1 - --scale keycloak_dc1=${keycloak.dc1.scale} --scale keycloak_dc2=${keycloak.dc2.scale} - keycloak_dc1 keycloak_dc2 - http://localhost:8081/auth http://localhost:8082/auth - - - - - - provision - - - - org.codehaus.mojo - exec-maven-plugin - - - docker-compose-up - pre-integration-test - - exec - - - ${project.basedir}/.. - docker-compose - -f ${compose.file} up -d --build ${compose.up.params} - - ${project.version} - - ${keycloak.docker.cpuset} - ${keycloak.dc1.docker.cpuset} - ${keycloak.dc2.docker.cpuset} - ${keycloak.docker.memlimit} - ${db.docker.cpuset} - ${db.docker.memlimit} - - ${keycloak.jvm.memory} - ${keycloak.http.max-connections} - ${keycloak.ajp.max-connections} - ${keycloak.worker.io-threads} - ${keycloak.worker.task-max-threads} - ${keycloak.ds.min-pool-size} - ${keycloak.ds.max-pool-size} - ${keycloak.ds.pool-prefill} - ${keycloak.ds.ps-cache-size} - - ${keycloak-lb.jvm.memory} - ${keycloak-lb.http.max-connections} - ${keycloak-lb.worker.io-threads} - ${keycloak-lb.worker.task-max-threads} - - ${infinispan.jvm.memory} - - - - - healthcheck - pre-integration-test - - exec - - - ./healthcheck.sh - ${project.basedir}/.. - - ${keycloak.server.uris} - - - - - - - maven-antrun-plugin - - - write-provisioned-system-properties - pre-integration-test - - run - - - - - - - - - - - @@ -442,6 +531,7 @@ -classpath + -Dkeycloak.server.uris=${keycloak.frontend.servers} -DnumOfWorkers=${numOfWorkers} org.keycloak.performance.RealmsConfigurationLoader benchmark-realms.json @@ -450,48 +540,34 @@ - - maven-antrun-plugin - - - write-imported-dataset-properties - pre-integration-test - - run - - - - - - - - - - - - - - - - - - - export-dump-after-generation - - - false - - export-dump - - - + export-dump + + maven-enforcer-plugin + + + enforce-nondefault-dataset + + enforce + + + + + dataset + (?!default).* + For the "export-dump" task property "dataset" cannot be set to "default". + + + + + + org.codehaus.mojo exec-maven-plugin @@ -503,10 +579,11 @@ exec - ${project.basedir}/.. - ./prepare-dump.sh - + ./${provisioner}.sh + ${provisioner} + ${deployment} + export-dump ${dataset} @@ -526,46 +603,21 @@ exec-maven-plugin - load-dump + import-dump pre-integration-test exec - ${project.basedir}/.. - ./load-dump.sh - + ./${provisioner}.sh + ${provisioner} + ${deployment} + import-dump ${dataset} - - restart-keycloak - pre-integration-test - - exec - - - ${project.basedir}/.. - docker-compose - -f ${compose.file} restart ${compose.restart.params} - - - - healthcheck - pre-integration-test - - exec - - - ./healthcheck.sh - ${project.basedir}/.. - - ${keycloak.server.uris} - - - @@ -582,7 +634,7 @@ teardown - -v + true @@ -591,29 +643,30 @@ exec-maven-plugin - docker-compose-down + teardown post-integration-test exec - ${project.basedir}/.. - docker-compose - -f ${compose.file} down ${volumes.arg} + ./${provisioner}.sh - ${project.version} + ${provisioner} + ${deployment} + teardown + ${delete.data} - + keep-data - + false @@ -627,29 +680,29 @@ exec-maven-plugin - monitoring-docker-compose-up + monitoring-on pre-integration-test exec - ${project.basedir}/.. - docker-compose - -f docker-compose-monitoring.yml up -d --build + ./${provisioner}.sh - ${monitoring.docker.cpuset} + ${provisioner} + monitoring + provision - + monitoring-off - + false @@ -658,15 +711,19 @@ exec-maven-plugin - monitoring-docker-compose-down + monitoring-off post-integration-test exec - ${project.basedir}/.. - docker-compose - -f docker-compose-monitoring.yml down ${monitoring.volumes.arg} + ./${provisioner}.sh + + ${provisioner} + monitoring + teardown + ${delete.monitoring.data} + @@ -677,7 +734,7 @@ delete-monitoring-data - -v + true diff --git a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml new file mode 100644 index 0000000000..7d152c7d4b --- /dev/null +++ b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml @@ -0,0 +1,51 @@ +version: "2.2" + +networks: + keycloak: + ipam: + config: + - subnet: 10.0.1.0/24 +# loadbalancing: +# ipam: +# config: +# - subnet: 10.0.2.0/24 + +services: + + mariadb: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + cpuset: ${DB_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-1g} + networks: + - keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: keycloak + MYSQL_USER: keycloak + MYSQL_PASSWORD: keycloak + ports: + - "3306:3306" + + loadbalancer: + build: load-balancer/wildfly-modcluster + image: keycloak_test_loadbalancer:${KEYCLOAK_VERSION:-latest} +# depends_on: +# keycloak: +# condition: service_healthy + cpuset: ${LB_CPUSET:-1} + mem_limit: ${LB_MEMLIMIT:-1g} + networks: + - keycloak +# - loadbalancing + environment: + PRIVATE_SUBNET: 10.0.1.0/24 +# PUBLIC_SUBNET: 10.0.2.0/24 + JAVA_OPTS: ${LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-500} + WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} + ports: + - "8080:8080" + + diff --git a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-keycloak.yml b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-keycloak.yml new file mode 100644 index 0000000000..fdc549cea3 --- /dev/null +++ b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-keycloak.yml @@ -0,0 +1,34 @@ + keycloak_%I%: + build: keycloak + image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} + depends_on: + mariadb: + condition: service_healthy + cpuset: "%CPUSET%" + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} + networks: + - keycloak + environment: + CONFIGURATION: standalone-ha.xml + PUBLIC_SUBNET: 10.0.1.0/24 + PRIVATE_SUBNET: 10.0.1.0/24 + MARIADB_HOSTS: mariadb:3306 + MARIADB_DATABASE: keycloak + MARIADB_USER: keycloak + MARIADB_PASSWORD: keycloak + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: admin + + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000} + AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-50000} + WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} + DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} + DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} + DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" + DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} + ports: + - "8080" + - "9990" + diff --git a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml new file mode 100644 index 0000000000..e51e192dcb --- /dev/null +++ b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml @@ -0,0 +1,154 @@ +version: "2.2" + +networks: + # DC 1 + dc1_keycloak: + ipam: + config: + - subnet: 10.1.1.0/24 + # DC 2 + dc2_keycloak: + ipam: + config: + - subnet: 10.2.1.0/24 +# # cross-DC +# loadbalancing: +# ipam: +# config: +# - subnet: 10.0.2.0/24 + # cross-DC + db_replication: + ipam: + config: + - subnet: 10.0.3.0/24 + # cross-DC + ispn_replication: + ipam: + config: + - subnet: 10.0.4.0/24 + +services: + + infinispan_dc1: + build: + context: infinispan + args: + LOCAL_SITE: dc1 + REMOTE_SITE: dc2 + image: keycloak_test_infinispan_dc1:${KEYCLOAK_VERSION:-latest} + cpuset: ${INFINISPAN_DC1_CPUSET:-1} + mem_limit: ${INFINISPAN_MEMLIMIT:-1500m} + networks: + - ispn_replication + - dc1_keycloak + environment: + PUBLIC_SUBNET: 10.1.1.0/24 + PRIVATE_SUBNET: 10.0.4.0/24 + MGMT_USER: admin + MGMT_USER_PASSWORD: admin + TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600] + JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + ports: + - "9991:9990" + + infinispan_dc2: + build: + context: infinispan + args: + LOCAL_SITE: dc2 + REMOTE_SITE: dc1 + image: keycloak_test_infinispan_dc2:${KEYCLOAK_VERSION:-latest} + depends_on: + infinispan_dc1: + condition: service_healthy + cpuset: ${INFINISPAN_DC2_CPUSET:-1} + mem_limit: ${INFINISPAN_MEMLIMIT:-1500m} + networks: + - ispn_replication + - dc2_keycloak + environment: + PUBLIC_SUBNET: 10.2.1.0/24 + PRIVATE_SUBNET: 10.0.4.0/24 + MGMT_USER: admin + MGMT_USER_PASSWORD: admin + TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600] + JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + ports: + - "9992:9990" + + + mariadb_dc1: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + cpuset: ${DB_DC1_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-2g} + networks: + - db_replication + - dc1_keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + MYSQL_DATABASE: keycloak + MYSQL_USER: keycloak + MYSQL_PASSWORD: keycloak + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep-new-cluster + ports: + - "3306:3306" + + mariadb_dc2: + build: db/mariadb + image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} + depends_on: + mariadb_dc1: + condition: service_healthy + cpuset: ${DB_DC2_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-2g} + networks: + - db_replication + - dc2_keycloak + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_INITDB_SKIP_TZINFO: foo + entrypoint: docker-entrypoint-wsrep.sh + command: --wsrep_cluster_address=gcomm://mariadb_dc1 + ports: + - "3307:3306" + + + loadbalancer_dc1: + build: load-balancer/wildfly-modcluster + image: keycloak_test_loadbalancer:${KEYCLOAK_VERSION:-latest} + cpuset: ${LB_DC1_CPUSET:-1} + mem_limit: ${LB_MEMLIMIT:-1g} + networks: + - dc1_keycloak +# - loadbalancing + environment: + PRIVATE_SUBNET: 10.1.1.0/24 +# PUBLIC_SUBNET: 10.0.2.0/24 + JAVA_OPTS: ${LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-500} + WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} + ports: + - "8081:8080" + + loadbalancer_dc2: + build: load-balancer/wildfly-modcluster + image: keycloak_test_loadbalancer:${KEYCLOAK_VERSION:-latest} + cpuset: ${LB_DC2_CPUSET:-1} + mem_limit: ${LB_MEMLIMIT:-1g} + networks: + - dc2_keycloak +# - loadbalancing + environment: + PRIVATE_SUBNET: 10.2.1.0/24 +# PUBLIC_SUBNET: 10.0.2.0/24 + JAVA_OPTS: ${LB_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-500} + WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} + ports: + - "8082:8080" + diff --git a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc1.yml b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc1.yml new file mode 100644 index 0000000000..cd72df1226 --- /dev/null +++ b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc1.yml @@ -0,0 +1,40 @@ + keycloak_dc1_%I%: + build: ./keycloak + image: keycloak_test_keycloak_dc1:${KEYCLOAK_VERSION:-latest} + depends_on: + # wait for the db cluster to be ready before starting keycloak + mariadb_dc2: + condition: service_healthy + # wait for the ispn cluster to be ready before starting keycloak + infinispan_dc2: + condition: service_healthy + cpuset: "%CPUSET%" + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} + networks: + - dc1_keycloak + environment: + CONFIGURATION: standalone-ha.xml + PUBLIC_SUBNET: 10.1.1.0/24 + PRIVATE_SUBNET: 10.1.1.0/24 + MARIADB_HOSTS: mariadb_dc1:3306 + MARIADB_DATABASE: keycloak + MARIADB_USER: keycloak + MARIADB_PASSWORD: keycloak + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: admin + INFINISPAN_HOST: infinispan_dc1 + SITE: dc1 + + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000} + AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-50000} + WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} + DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} + DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} + DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" + DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} + ports: + - "8080" + - "9990" + diff --git a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc2.yml b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc2.yml new file mode 100644 index 0000000000..b8fadf2327 --- /dev/null +++ b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc2.yml @@ -0,0 +1,35 @@ + keycloak_dc2_%I%: + build: ./keycloak + image: keycloak_test_keycloak_dc2:${KEYCLOAK_VERSION:-latest} + depends_on: + # wait for first kc instance to be ready before starting another + keycloak_dc1_1: + condition: service_healthy + cpuset: "%CPUSET%" + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} + networks: + - dc2_keycloak + environment: + CONFIGURATION: standalone-ha.xml + PUBLIC_SUBNET: 10.2.1.0/24 + PRIVATE_SUBNET: 10.2.1.0/24 + MARIADB_HOSTS: mariadb_dc2:3306 + MARIADB_DATABASE: keycloak + MARIADB_USER: keycloak + MARIADB_PASSWORD: keycloak + INFINISPAN_HOST: infinispan_dc2 + SITE: dc2 + + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000} + AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-50000} + WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} + WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} + DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} + DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100} + DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}" + DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} + ports: + - "8080" + - "9990" + diff --git a/testsuite/performance/docker-compose-monitoring.yml b/testsuite/performance/tests/src/main/docker-compose/docker-compose-monitoring.yml similarity index 100% rename from testsuite/performance/docker-compose-monitoring.yml rename to testsuite/performance/tests/src/main/docker-compose/docker-compose-monitoring.yml diff --git a/testsuite/performance/docker-compose.yml b/testsuite/performance/tests/src/main/docker-compose/docker-compose.yml similarity index 97% rename from testsuite/performance/docker-compose.yml rename to testsuite/performance/tests/src/main/docker-compose/docker-compose.yml index 93cc97ff54..17ee9f628e 100644 --- a/testsuite/performance/docker-compose.yml +++ b/testsuite/performance/tests/src/main/docker-compose/docker-compose.yml @@ -12,7 +12,7 @@ services: build: db/mariadb image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} cpuset: ${DB_CPUSET:-1} - mem_limit: ${DB_MEMLIMIT:-1g} + mem_limit: ${DB_MEMLIMIT:-2g} networks: - keycloak environment: @@ -43,7 +43,7 @@ services: KEYCLOAK_PASSWORD: admin # docker-compose syntax note: ${ENV_VAR:-} JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true - HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} + HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000} WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10} diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/RealmsConfigurationLoader.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/RealmsConfigurationLoader.java index 764f435227..52d1fde934 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/RealmsConfigurationLoader.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/RealmsConfigurationLoader.java @@ -58,6 +58,7 @@ public class RealmsConfigurationLoader { static boolean realmCreated; public static void main(String [] args) throws IOException { + System.out.println("Keycloak servers: "+TestConfig.serverUrisList); if (args.length == 0) { args = new String[] {EXPORT_FILENAME}; diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java index 8a28ea1a57..e95ca8782a 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java @@ -4,6 +4,7 @@ import org.keycloak.performance.util.FilteredIterator; import org.keycloak.performance.util.LoopingIterator; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -77,21 +78,17 @@ public class TestConfig { static { // if KEYCLOAK_SERVER_URIS env var is set, and system property serverUris is not set - String servers = System.getProperty("serverUris"); - if (servers == null) { - String env = System.getenv("KEYCLOAK_SERVER_URIS"); - serverUris = env != null ? env : "http://localhost:8080/auth"; + String serversProp = System.getProperty("keycloak.server.uris"); + if (serversProp == null) { + String serversEnv = System.getenv("KEYCLOAK_SERVERS"); + serverUris = serversEnv != null ? serversEnv : "http://localhost:8080/auth"; } else { - serverUris = servers; + serverUris = serversProp; } // initialize serverUrisList and serverUrisIterator - ArrayList uris = new ArrayList<>(); - for (String uri: serverUris.split(" ")) { - uris.add(uri); - } - serverUrisList = uris; - serverUrisIterator = new LoopingIterator<>(uris); + serverUrisList = Arrays.asList(serverUris.split(" ")); + serverUrisIterator = new LoopingIterator<>(serverUrisList); } // Users iterators by realm diff --git a/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala index e5465ca3ea..97e323fedf 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala @@ -14,7 +14,7 @@ import SimulationsHelper._ class AdminConsoleSimulation extends Simulation { println() - println("Using server: " + TestConfig.serverUrisList.get(0)) + println("Target server: " + TestConfig.serverUrisList.get(0)) println() println("Using test parameters:") println(" runUsers: " + TestConfig.runUsers) diff --git a/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala index 86dd29a201..a2ed6d4830 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala @@ -25,6 +25,9 @@ class DefaultSimulation extends Simulation { + println() + println("Taget servers: " + TestConfig.serverUrisList) + println() println("Using test parameters:") println(" runUsers: " + TestConfig.runUsers) println(" numOfIterations: " + TestConfig.numOfIterations)