IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

自动配置DorisDB集群的脚本

wen 实用脚本 24

我来提供一个自动配置DorisDB（StarRocks）集群的脚本,这个脚本支持自动化部署和配置DorisDB集群。

自动配置DorisDB集群的脚本

DorisDB集群自动化部署脚本

主部署脚本 (deploy_doris_cluster.sh)

#!/bin/bash
set -e
# DorisDB集群自动部署脚本
# 支持FE和BE节点的自动配置
# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# 日志函数
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}
# 默认配置
DORIS_HOME="/opt/doris"
DORIS_VERSION="2.0.3"
DOWNLOAD_BASE_URL="https://apache-doris-releases.oss-accelerate.aliyuncs.com"
FE_HTTP_PORT="8030"
FE_QUERY_PORT="9030"
BE_HEARTBEAT_PORT="9050"
BE_PORT="9060"
CLUSTER_NAME="doris_cluster"
# 初始化配置
init_config() {
    cat > /tmp/doris_config.json <<EOF
{
    "cluster_name": "${CLUSTER_NAME}",
    "fe_nodes": [
        {
            "host": "192.168.1.10",
            "priority": 1
        },
        {
            "host": "192.168.1.11",
            "priority": 2
        },
        {
            "host": "192.168.1.12",
            "priority": 3
        }
    ],
    "be_nodes": [
        {
            "host": "192.168.1.20",
            "storage_path": "/data1/doris/data,/data2/doris/data"
        },
        {
            "host": "192.168.1.21",
            "storage_path": "/data1/doris/data,/data2/doris/data"
        },
        {
            "host": "192.168.1.22",
            "storage_path": "/data1/doris/data,/data2/doris/data"
        }
    ],
    "settings": {
        "max_memory": "64G",
        "log_retention_days": 7,
        "query_timeout": 300,
        "max_connections": 1000
    }
}
EOF
    log_info "配置文件已生成: /tmp/doris_config.json"
}
# 检查系统需求
check_requirements() {
    log_info "检查系统需求..."
    # 检查Java
    if ! command -v java &> /dev/null; then
        log_error "Java未安装，请先安装JDK 8或11"
        exit 1
    fi
    JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
    log_info "Java版本: $JAVA_VERSION"
    # 检查Python
    if ! command -v python3 &> /dev/null; then
        log_warn "Python3未安装，部分功能可能受限"
    fi
    # 检查磁盘空间
    available_space=$(df / | awk 'NR==2 {print $4}')
    if [ "$available_space" -lt 10485760 ]; then
        log_error "磁盘空间不足10GB，请清理磁盘"
        exit 1
    fi
    # 检查内存
    total_mem=$(free -g | awk '/^Mem:/{print $2}')
    if [ "$total_mem" -lt 8 ]; then
        log_warn "内存小于8GB，DorisDB可能需要更多内存"
    fi
    log_info "系统需求检查完成"
}
# 安装依赖
install_dependencies() {
    log_info "安装系统依赖..."
    if [ -f /etc/debian_version ]; then
        apt-get update
        apt-get install -y wget tar gzip supervisor
    elif [ -f /etc/redhat-release ]; then
        yum install -y wget tar gzip supervisor
    else
        log_error "不支持的操作系统"
        exit 1
    fi
    log_info "依赖安装完成"
}
# 下载DorisDB
download_doris() {
    local node_type=$1
    log_info "下载DorisDB ${DORIS_VERSION} ${node_type}..."
    mkdir -p "${DORIS_HOME}"
    cd "${DORIS_HOME}"
    if [ "$node_type" = "fe" ]; then
        wget -q "${DOWNLOAD_BASE_URL}/apache-doris-fe-${DORIS_VERSION}-bin-x86_64.tar.gz"
        tar xzf "apache-doris-fe-${DORIS_VERSION}-bin-x86_64.tar.gz"
        rm -f "apache-doris-fe-${DORIS_VERSION}-bin-x86_64.tar.gz"
        mv "apache-doris-fe-${DORIS_VERSION}-bin-x86_64" fe
    else
        wget -q "${DOWNLOAD_BASE_URL}/apache-doris-be-${DORIS_VERSION}-bin-x86_64.tar.gz"
        tar xzf "apache-doris-be-${DORIS_VERSION}-bin-x86_64.tar.gz"
        rm -f "apache-doris-be-${DORIS_VERSION}-bin-x86_64.tar.gz"
        mv "apache-doris-be-${DORIS_VERSION}-bin-x86_64" be
    fi
    log_info "DorisDB下载完成"
}
# 配置FE节点
configure_fe() {
    local host=$1
    local priority=$2
    local fe_conf="${DORIS_HOME}/fe/conf/fe.conf"
    log_info "配置FE节点: ${host}"
    cat > "${fe_conf}" <<EOF
# FE配置文件
priority_networks = ${host}/24
http_port = ${FE_HTTP_PORT}
rpc_port = 9020
query_port = ${FE_QUERY_PORT}
edit_log_port = 9010
mysql_service_nthreads = 4
# 元数据配置
meta_dir = ${DORIS_HOME}/fe/doris-meta
log_roll_size_mb = 1024
sys_log_dir = ${DORIS_HOME}/fe/log
audit_log_dir = ${DORIS_HOME}/fe/log
audit_log_modules = query
audit_log_roll_num = 90
# 性能配置
max_memory = $(free -g | awk '/^Mem:/{print $2}')G
max_connections = 1000
query_timeout = 300
max_query_retry_time = 2
enable_spilling = true
# 集群配置
cluster_id = $(date +%s)
http_max_header_size = 4096
EOF
    # 创建必要目录
    mkdir -p "${DORIS_HOME}/fe/doris-meta"
    mkdir -p "${DORIS_HOME}/fe/log"
    log_info "FE节点配置完成"
}
# 配置BE节点
configure_be() {
    local host=$1
    local storage_paths=$2
    local be_conf="${DORIS_HOME}/be/conf/be.conf"
    log_info "配置BE节点: ${host}"
    # 处理存储路径
    IFS=',' read -ra paths <<< "$storage_paths"
    cat > "${be_conf}" <<EOF
# BE配置文件
priority_networks = ${host}/24
be_port = ${BE_PORT}
webserver_port = 8040
heartbeat_service_port = ${BE_HEARTBEAT_PORT}
brpc_port = 8060
# 存储配置
storage_root_path = ${storage_paths}
default_rowset_type = beta
# 性能配置
max_memory = $(free -g | awk '/^Mem:/{print $2}')G
fragment_pthread_count = 512
flush_thread_num_per_store = 2
publish_version_thread_count = 8
consume_thread_num_per_store = 2
# 日志配置
sys_log_dir = ${DORIS_HOME}/be/log
sys_log_roll_num = 10
sys_log_verbose_modules = *
# 缓存配置
file_cache_enable = true
file_cache_max_capacity = 10G
file_cache_path = ${DORIS_HOME}/be/cache
EOF
    # 创建必要目录
    mkdir -p "${DORIS_HOME}/be/log"
    mkdir -p "${DORIS_HOME}/be/cache"
    for path in "${paths[@]}"; do
        mkdir -p "$(dirname $path)"
    done
    log_info "BE节点配置完成"
}
# 启动FE节点
start_fe() {
    log_info "启动FE节点..."
    cd "${DORIS_HOME}/fe"
    # 首次启动需要初始化
    if [ ! -f "${DORIS_HOME}/fe/doris-meta/image/ROLE" ]; then
        log_info "首次启动FE，需要进行初始化..."
        # 主FE节点
        ./bin/start_fe.sh --daemon
        # 等待FE启动
        sleep 5
        # 检查FE是否启动成功
        if netstat -tlnp 2>/dev/null | grep -q ":${FE_QUERY_PORT}"; then
            log_info "FE节点启动成功"
        else
            log_error "FE节点启动失败，检查日志"
            tail -20 "${DORIS_HOME}/fe/log/fe.log"
            exit 1
        fi
    else
        ./bin/start_fe.sh --daemon
        log_info "FE节点已启动"
    fi
}
# 启动BE节点
start_be() {
    log_info "启动BE节点..."
    cd "${DORIS_HOME}/be"
    # 检查是否已初始化
    if [ ! -f "${DORIS_HOME}/be/conf/be.conf" ]; then
        log_error "BE配置文件不存在"
        exit 1
    fi
    ./bin/start_be.sh --daemon
    # 等待BE启动
    sleep 3
    # 检查BE是否启动成功
    if netstat -tlnp 2>/dev/null | grep -q ":${BE_PORT}"; then
        log_info "BE节点启动成功"
    else
        log_error "BE节点启动失败，检查日志"
        tail -20 "${DORIS_HOME}/be/log/be.INFO"
        exit 1
    fi
}
# 添加BE节点到集群
add_be_to_cluster() {
    local be_host=$1
    local fe_host=$(jq -r '.fe_nodes[0].host' /tmp/doris_config.json)
    log_info "添加BE节点 ${be_host} 到集群..."
    # 使用MySQL客户端添加BE节点
    mysql -h "${fe_host}" -P "${FE_QUERY_PORT}" -u root -e \
        "ALTER SYSTEM ADD BACKEND '${be_host}:${BE_HEARTBEAT_PORT}';"
    if [ $? -eq 0 ]; then
        log_info "BE节点 ${be_host} 添加成功"
    else
        log_error "BE节点 ${be_host} 添加失败"
    fi
}
# Supervisor配置
configure_supervisor() {
    log_info "配置Supervisor管理服务..."
    cat > /etc/supervisor/conf.d/doris.conf <<EOF
[program:doris-fe]
command=${DORIS_HOME}/fe/bin/start_fe.sh
directory=${DORIS_HOME}/fe
user=root
autostart=true
autorestart=true
startretries=3
stderr_logfile=${DORIS_HOME}/fe/log/supervisor_fe.err.log
stdout_logfile=${DORIS_HOME}/fe/log/supervisor_fe.out.log
[program:doris-be]
command=${DORIS_HOME}/be/bin/start_be.sh
directory=${DORIS_HOME}/be
user=root
autostart=true
autorestart=true
startretries=3
stderr_logfile=${DORIS_HOME}/be/log/supervisor_be.err.log
stdout_logfile=${DORIS_HOME}/be/log/supervisor_be.out.log
EOF
    supervisorctl reread
    supervisorctl update
    log_info "Supervisor配置完成"
}
# 验证集群状态
verify_cluster() {
    local fe_host=$(jq -r '.fe_nodes[0].host' /tmp/doris_config.json)
    log_info "验证集群状态..."
    # 检查FE状态
    echo "FE节点状态:"
    curl -s "http://${fe_host}:${FE_HTTP_PORT}/api/bootstrap" | python3 -m json.tool
    # 检查BE状态
    echo "BE节点状态:"
    mysql -h "${fe_host}" -P "${FE_QUERY_PORT}" -u root -e "SHOW PROC '/backends';"
    # 检查集群健康状态
    echo "集群健康状态:"
    mysql -h "${fe_host}" -P "${FE_QUERY_PORT}" -u root -e "SHOW PROC '/cluster_health';"
}
# 主函数
main() {
    echo "=========================================="
    echo "    DorisDB集群自动部署脚本"
    echo "=========================================="
    # 检查root权限
    if [ "$EUID" -ne 0 ]; then
        log_error "请使用root权限运行此脚本"
        exit 1
    fi
    # 初始化配置
    init_config
    # 检查系统需求
    check_requirements
    # 安装依赖
    install_dependencies
    # 读取配置
    local fe_nodes=$(jq -c '.fe_nodes[]' /tmp/doris_config.json)
    local be_nodes=$(jq -c '.be_nodes[]' /tmp/doris_config.json)
    # 部署FE节点
    log_info "开始部署FE节点..."
    for fe_node in $fe_nodes; do
        local host=$(echo $fe_node | jq -r '.host')
        local priority=$(echo $fe_node | jq -r '.priority')
        download_doris "fe"
        configure_fe "$host" "$priority"
        start_fe
    done
    # 部署BE节点
    log_info "开始部署BE节点..."
    for be_node in $be_nodes; do
        local host=$(echo $be_node | jq -r '.host')
        local storage_path=$(echo $be_node | jq -r '.storage_path')
        download_doris "be"
        configure_be "$host" "$storage_path"
        start_be
        add_be_to_cluster "$host"
    done
    # 配置Supervisor
    configure_supervisor
    # 验证集群
    verify_cluster
    log_info "DorisDB集群部署完成！"
    log_info "FE地址: $(jq -r '.fe_nodes[0].host' /tmp/doris_config.json):${FE_QUERY_PORT}"
}
# 执行主函数
main "$@"

快速部署脚本 (quick_deploy.sh)

#!/bin/bash
set -e
# DorisDB快速部署脚本
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
# 配置参数
FE_HOSTS="192.168.1.10,192.168.1.11,192.168.1.12"
BE_HOSTS="192.168.1.20,192.168.1.21,192.168.1.22"
SSH_USER="root"
SSH_PORT="22"
# 批量部署
batch_deploy() {
    log_info "开始批量部署到所有节点..."
    # 部署FE节点
    IFS=',' read -ra fe_array <<< "$FE_HOSTS"
    for host in "${fe_array[@]}"; do
        log_info "部署FE节点: ${host}"
        scp deploy_doris_cluster.sh ${SSH_USER}@${host}:/tmp/
        ssh ${SSH_USER}@${host} "bash /tmp/deploy_doris_cluster.sh"
    done
    # 部署BE节点
    IFS=',' read -ra be_array <<< "$BE_HOSTS"
    for host in "${be_array[@]}"; do
        log_info "部署BE节点: ${host}"
        scp deploy_doris_cluster.sh ${SSH_USER}@${host}:/tmp/
        ssh ${SSH_USER}@${host} "bash /tmp/deploy_doris_cluster.sh"
    done
    log_info "批量部署完成"
}
# 节点状态检查
check_status() {
    log_info "检查所有节点状态..."
    # 检查FE节点
    for host in $(echo $FE_HOSTS | tr ',' ' '); do
        echo -n "FE节点 ${host}: "
        if nc -z ${host} 9030 2>/dev/null; then
            echo -e "${GREEN}[运行中]${NC}"
        else
            echo -e "${RED}[离线]${NC}"
        fi
    done
    # 检查BE节点
    for host in $(echo $BE_HOSTS | tr ',' ' '); do
        echo -n "BE节点 ${host}: "
        if nc -z ${host} 9060 2>/dev/null; then
            echo -e "${GREEN}[运行中]${NC}"
        else
            echo -e "${RED}[离线]${NC}"
        fi
    done
}
# 执行
batch_deploy
check_status

使用说明

环境准备：

# 1. 确保所有节点时间同步
yum install -y ntpdate
ntpdate pool.ntp.org
# 2. 关闭防火墙（生产环境请配置安全规则）
systemctl stop firewalld
systemctl disable firewalld
# 3. 配置SSH免密登录
ssh-keygen -t rsa
ssh-copy-id root@FE_HOST
ssh-copy-id root@BE_HOST

部署步骤：

# 1. 下载脚本
wget https://your-server/deploy_doris_cluster.sh
chmod +x deploy_doris_cluster.sh
# 2. 编辑配置文件（可选）
vim /tmp/doris_config.json
# 3. 运行脚本
./deploy_doris_cluster.sh
# 4. 批量部署（可选）
./quick_deploy.sh

使用示例：

# 连接DorisDB
mysql -h 192.168.1.10 -P 9030 -u root
# 创建数据库
CREATE DATABASE test_db;
USE test_db;
# 创建表
CREATE TABLE test_table (
    id INT,
    name VARCHAR(50),
    age INT
) DISTRIBUTED BY HASH(id) BUCKETS 10;
# 插入数据
INSERT INTO test_table VALUES (1, 'Alice', 25);

这是一个功能完整的DorisDB集群自动化部署脚本，支持批量部署、故障恢复和自动管理,根据实际需求调整配置参数即可。

上一篇脚本如何检测近场通信是否正常

下一篇Shell脚本如何实现金丝雀发布策略

相关文章

抱歉，评论功能暂时关闭!