Implementing Resource Isolation in Doris Offline Analysis Data Warehouse Cluster

1
2
3

[doris_dabe_nodes]
# If nodes have different CGroup versions, they can be grouped. Here only specifying offline analysis warehouse BE nodes
10.18.7.160

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108

---
- name: Setup Doris CGroup Configuration
  hosts: doris_dabe_nodes
  become: true
  vars:
    # CGroup configuration path (automatically selected based on version)
    cgroup_v1_path: "/sys/fs/cgroup/cpu/doris"
    cgroup_v2_path: "/sys/fs/cgroup/doris"
    doris_user: "root"
    doris_group: "root"
    
  tasks:
    - name: Detect CGroup version
      stat:
        path: /sys/fs/cgroup/cpu/
      register: cgroup_v1_check
      
    - name: Set CGroup path based on version
      set_fact:
        cgroup_path: "{{ cgroup_v1_path if cgroup_v1_check.stat.exists else cgroup_v2_path }}"
        cgroup_version: "{{ 'v1' if cgroup_v1_check.stat.exists else 'v2' }}"
        
    - name: Display detected CGroup version
      debug:
        msg: "Node {{ inventory_hostname }} uses CGroup {{ cgroup_version }}, path: {{ cgroup_path }}"

    - name: Create CGroup directory
      file:
        path: "{{ cgroup_path }}"
        state: directory
        mode: '0770'
        owner: "{{ doris_user }}"
        group: "{{ doris_group }}"
      register: cgroup_dir_result
      
    - name: Configure CGroup v2 specific settings
      block:
        - name: Enable write permission on cgroup.procs
          file:
            path: /sys/fs/cgroup/cgroup.procs
            mode: 'a+w'
            
        - name: Enable CPU controller for CGroup v2
          shell: |
            if ! grep -q '+cpu' /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null; then
              echo '+cpu' > /sys/fs/cgroup/cgroup.subtree_control
            fi            
          args:
            executable: /bin/bash
          ignore_errors: yes
          
        - name: Verify CPU controller is enabled
          stat:
            path: "{{ cgroup_path }}/cpu.max"
          register: cpu_controller_check
          
        - name: Display CPU controller status
          debug:
            msg: "CPU controller is {{ 'enabled' if cpu_controller_check.stat.exists else 'NOT enabled' }}"
      when: cgroup_version == 'v2'

    - name: Verify CGroup directory structure
      shell: |
        echo "=== CGroup Version: {{ cgroup_version }} ==="
        echo "CGroup Path: {{ cgroup_path }}"
        ls -la {{ cgroup_path }} 2>/dev/null || echo "Directory verified"
        echo ""
        if [ "{{ cgroup_version }}" = "v2" ]; then
          echo "=== CPU Controller Check ==="
          cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null
          test -f {{ cgroup_path }}/cpu.max && echo "cpu.max: EXISTS" || echo "cpu.max: NOT FOUND"
        fi        
      args:
        executable: /bin/bash
      register: verify_result
      
    - name: Display verification results
      debug:
        var: verify_result.stdout_lines

    - name: Create systemd service for CGroup persistence (optional)
      copy:
        dest: /etc/systemd/system/doris-cgroup.service
        content: |
          [Unit]
          Description=Doris CGroup Setup
          Before=doris-be.service

          [Service]
          Type=oneshot
          ExecStart=/bin/bash -c 'mkdir -p {{ cgroup_path }} && chmod 770 {{ cgroup_path }} && chown -R {{ doris_user }}:{{ doris_group }} {{ cgroup_path }}'
          {% if cgroup_version == 'v2' %}
          ExecStartPost=/bin/bash -c 'chmod a+w /sys/fs/cgroup/cgroup.procs'
          ExecStartPost=/bin/bash -c 'echo "+cpu" > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true'
          {% endif %}

          [Install]
          WantedBy=multi-user.target          
        mode: '0644'
      when: cgroup_dir_result is changed
      notify: Enable systemd service

  handlers:   
    - name: Enable systemd service
      systemd:
        name: doris-cgroup
        enabled: yes
        daemon_reload: yes

1
2
3
4
5
6
7
8

# 1. Syntax check
ansible-playbook -i hosts setup_doris_cgroup.yaml --syntax-check

# 2. List all tasks to be executed
ansible-playbook -i hosts setup_doris_cgroup.yaml --list-tasks

# 3. Dry run (not actually executing, only showing changes to be made)
ansible-playbook -i hosts setup_doris_cgroup.yaml --check --diff

1
2
3
4
5
6
7
8

# Method 1: Execute on all BE nodes
ansible-playbook -i hosts setup_doris_cgroup.yaml

# Method 2: Execute only on specific nodes (e.g., test on single node first)
ansible-playbook -i hosts setup_doris_cgroup.yaml --limit 10.18.7.101

# Method 3: Execute step by step, each task requires confirmation
ansible-playbook -i hosts setup_doris_cgroup.yaml --step

1
2
3
4
5

# 1. Check if CGroup directory is created
ansible doris_dabe_nodes -i hosts -m shell -a "ls -la /sys/fs/cgroup/doris 2>/dev/null || ls -la /sys/fs/cgroup/cpu/doris 2>/dev/null"

# 2. Check CGroup initialization in BE logs (adjust according to actual BE log path)
ansible doris_dabe_nodes -i hosts -m shell -a "tail -100 /opt/doris/doris/be/log/be.INFO | grep -i 'cgroup\|add thread' | tail -5"

1
2
3
4

CREATE WORKLOAD GROUP IF NOT EXISTS g1
PROPERTIES (
    "cpu_share" = "1024"
);

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

CREATE WORKLOAD GROUP IF NOT EXISTS etl_group
PROPERTIES (
    -- CPU configuration (soft limit mode)
    "cpu_share" = "2048",
    "cpu_hard_limit" = "40%",
    
    -- Memory configuration (hard limit mode)
    "memory_limit" = "40%",
    "enable_memory_overcommit" = "false",
    
    -- Concurrency control
    "max_concurrency" = "1",
    "max_queue_size" = "100",
    "queue_timeout" = "10000",
    
    -- IO speed limit
    "read_bytes_per_second" = "104857600",        -- 100MB/s
    "remote_read_bytes_per_second" = "52428800"   -- 50MB/s
);