混部隔离示例

# 查看/boot/config-<kernel version>系统配置是否开启混部隔离功能
# 若 CONFIG_QOS_SCHED=y 则说明使能了混部隔离功能，例如：
cat /boot/config-5.10.0-60.18.0.50.oe2203.x86_64 | grep CONFIG_QOS
CONFIG_QOS_SCHED=y

yum install -y docker-engine
docker version
# 如下为 docker version 显示结果
Client:
 Version:           18.09.0
 EulerVersion:      18.09.0.300
 API version:       1.39
 Go version:        go1.17.3
 Git commit:        aa1eee8
 Built:             Wed Mar 30 05:07:38 2022
 OS/Arch:           linux/amd64
 Experimental:      false

Server:
 Engine:
  Version:          18.09.0
  EulerVersion:     18.09.0.300
  API version:      1.39 (minimum version 1.12)
  Go version:       go1.17.3
  Git commit:       aa1eee8
  Built:            Tue Mar 22 00:00:00 2022
  OS/Arch:          linux/amd64
  Experimental:     false

#!/bin/bash

with_offline=${1:-no_offline}
enable_isolation=${2:-no_isolation}
stress_num=${3:-10}
concurrency=10
timeout=30
output=/tmp/result.json
online_container=
offline_container=

exec_sql="echo \"SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000\" | clickhouse-benchmark -i 10000 -c $concurrency -t $timeout"

function prepare() {
    echo "Launch clickhouse container."
    online_container=$(docker run -itd \
            -v /tmp:/tmp:rw \
            --ulimit nofile=262144:262144 \
            -p 34424:34424 \
            yandex/clickhouse-server)

    sleep 3
    echo "Clickhouse container lauched."
}

function clickhouse() {
    echo "Start clickhouse benchmark test."
    docker exec $online_container bash -c "$exec_sql --json $output"
    echo "Clickhouse benchmark test done."
}

function stress() {
    echo "Launch stress container."
    offline_container=$(docker run -itd joedval/stress --cpu $stress_num)
    echo "Stress container launched."

    if [ $enable_isolation == "enable_isolation" ]; then
        echo "Set stress container qos level to -1."
        echo -1 > /sys/fs/cgroup/cpu/docker/$offline_container/cpu.qos_level
    fi
}

function benchmark() {
    if [ $with_offline == "with_offline" ]; then
    	stress
    	sleep 3
	fi
    clickhouse
    echo "Remove test containers."
    docker rm -f $online_container
    docker rm -f $offline_container
    echo "Finish benchmark test for clickhouse(online) and stress(offline) colocation."
    echo "===============================clickhouse benchmark=================================================="
    cat $output
    echo "===============================clickhouse benchmark=================================================="
}

prepare
benchmark

sh test_demo.sh no_offline no_isolation

{
    "localhost:9000": {
        "statistics": {
            "QPS": 1.8853412284364512,
            ......
        }
    },
    "query_time_percentiles": {
        ......
        "50": 0.484905256,
        "60": 0.519641313,
        "70": 0.570876148,
        "80": 0.632544937,
        "90": 0.728295525,
        "95": 0.808700418,
        "99": 0.873945121,
        ......
    }
}

# with_offline 参数表示启用 stress 离线业务
# no_isolation 参数表示未开启混部隔离功能
sh test_demo.sh with_offline no_isolation

{
    "localhost:9000": {
        "statistics": {
            "QPS": 0.9424028693636205,
            ......
        }
    },
    "query_time_percentiles": {
        ......
        "50": 0.840476774,
        "60": 1.304607373,
        "70": 1.393591017,
        "80": 1.41277543,
        "90": 1.430316688,
        "95": 1.457534764,
        "99": 1.555646855,
        ......
    }
}

# with_offline 参数表示启用 stress 离线业务
# enable_isolation 参数表示开启混部隔离功能
sh test_demo.sh with_offline enable_isolation

{
    "localhost:9000": {
        "statistics": {
            "QPS": 1.8825798759270718,
            ......
        }
    },
    "query_time_percentiles": {
        ......
        "50": 0.485725185,
        "60": 0.512629901,
        "70": 0.55656488,
        "80": 0.636395956,
        "90": 0.734695906,
        "95": 0.804118275,
        "99": 0.887807409,
        ......
    }
}