IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

自动配置ClickHouse集群的脚本

wen 实用脚本 20

我来为您提供一个自动配置ClickHouse集群的脚本,这个脚本可以自动完成ClickHouse集群的安装和配置。

自动配置ClickHouse集群的脚本

主配置文件

cluster-config.yaml

# ClickHouse集群配置
cluster:
  name: clickhouse_cluster
  shards: 2
  replicas: 2
nodes:
  - host: 192.168.1.10
    hostname: node1
    role: "leader"
    shard: 1
    replica: 1
    user: root
  - host: 192.168.1.11
    hostname: node2
    role: "follower"
    shard: 1
    replica: 2
    user: root
  - host: 192.168.1.12
    hostname: node3
    role: "follower"
    shard: 2
    replica: 1
    user: root
  - host: 192.168.1.13
    hostname: node4
    role: "follower"
    shard: 2
    replica: 2
    user: root
clickhouse:
  version: "22.8.15"
  port: 9000
  http_port: 8123
  data_dir: /var/lib/clickhouse
  log_dir: /var/log/clickhouse-server
zookeeper:
  hosts:
    - 192.168.1.10:2181
    - 192.168.1.11:2181
    - 192.168.1.12:2181

主安装脚本

install-clickhouse-cluster.sh

#!/bin/bash
set -e
# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# 日志函数
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}
# 检查root权限
check_root() {
    if [[ $EUID -ne 0 ]]; then
        log_error "此脚本必须以root用户运行"
        exit 1
    fi
}
# 安装依赖
install_dependencies() {
    log_info "安装依赖包..."
    if command -v yum &> /dev/null; then
        yum install -y curl wget tar python3-pip sshpass
    elif command -v apt-get &> /dev/null; then
        apt-get update
        apt-get install -y curl wget tar python3-pip sshpass
    fi
    pip3 install pyyaml
}
# 配置SSH免密登录
setup_ssh() {
    log_info "配置SSH免密登录..."
    if [ ! -f ~/.ssh/id_rsa ]; then
        ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
    fi
    # 读取配置
    local config_file=$1
    # 获取所有节点信息（除自身）
    python3 <<-EOF
import yaml
with open('$config_file', 'r') as f:
    config = yaml.safe_load(f)
local_ip = "$(hostname -I | awk '{print $1}')"
for node in config['nodes']:
    if node['host'] != local_ip:
        print(f"正在配置免密登录到 {node['host']}...")
        import os
        os.system(f"sshpass -p '' ssh-copy-id -o StrictHostKeyChecking=no root@{node['host']}")
        print(f"完成配置 {node['host']}")
EOF
}
# 安装ClickHouse
install_clickhouse() {
    local version=$1
    log_info "安装ClickHouse ${version}..."
    # 下载RPM包
    local base_url="https://packages.clickhouse.com/rpm/stable"
    local packages=(
        "clickhouse-common-static-${version}.x86_64.rpm"
        "clickhouse-server-${version}.x86_64.rpm"
        "clickhouse-client-${version}.x86_64.rpm"
    )
    for pkg in "${packages[@]}"; do
        if [ ! -f "/tmp/${pkg}" ]; then
            wget -q "${base_url}/${pkg}" -O "/tmp/${pkg}"
        fi
        rpm -ivh "/tmp/${pkg}" 2>/dev/null || true
    done
}
# 生成集群配置
generate_cluster_config() {
    local config_file=$1
    log_info "生成集群配置..."
    python3 <<-EOF
import yaml
import os
with open('$config_file', 'r') as f:
    config = yaml.safe_load(f)
cluster_name = config['cluster']['name']
zookeeper_hosts = config['zookeeper']['hosts']
# 生成metrika.xml
metrika_config = f"""<yandex>
    <clickhouse_remote_servers>
        <{cluster_name}>
"""
for shard in range(1, config['cluster']['shards'] + 1):
    metrika_config += f"""            <shard>
                <internal_replication>true</internal_replication>
"""
    for node in config['nodes']:
        if node['shard'] == shard:
            metrika_config += f"""                <replica>
                    <host>{node['host']}</host>
                    <port>{config['clickhouse']['port']}</port>
                </replica>
"""
    metrika_config += """            </shard>
"""
metrika_config += f"""        </{cluster_name}>
    </clickhouse_remote_servers>
    <zookeeper>
"""
for zk_host in zookeeper_hosts:
    host, port = zk_host.split(':')
    metrika_config += f"""        <node>
            <host>{host}</host>
            <port>{port}</port>
        </node>
"""
metrika_config += """    </zookeeper>
    <macros>
        <shard>{shard}</shard>
        <replica>{replica}</replica>
    </macros>
    <networks>
        <ip>::/0</ip>
    </networks>
    <clickhouse_compression>
        <case>
            <min_part_size>10000000000</min_part_size>
            <min_part_size_ratio>0.01</min_part_size_ratio>
            <method>lz4</method>
        </case>
    </clickhouse_compression>
</yandex>"""
# 写入配置文件
with open('/etc/clickhouse-server/config.d/metrika.xml', 'w') as f:
    f.write(metrika_config.format(shard=1, replica=1))
print("集群配置已生成")
EOF
}
# 创建分布式表
create_distributed_tables() {
    local config_file=$1
    log_info "创建分布式表..."
    python3 <<-EOF
import yaml
import os
with open('$config_file', 'r') as f:
    config = yaml.safe_load(f)
cluster_name = config['cluster']['name']
distributed_sql_commands = [
    f"""
    CREATE DATABASE IF NOT EXISTS cluster_db ON CLUSTER {cluster_name};
    """,
    f"""
    CREATE TABLE IF NOT EXISTS cluster_db.local_table ON CLUSTER {cluster_name}
    (
        id UInt32,
        name String,
        timestamp DateTime
    ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/local_table', '{{replica}}')
    ORDER BY id
    PARTITION BY toYYYYMM(timestamp);
    """,
    f"""
    CREATE TABLE IF NOT EXISTS cluster_db.distributed_table ON CLUSTER {cluster_name}
    AS cluster_db.local_table
    ENGINE = Distributed({cluster_name}, cluster_db, local_table, rand());
    """
]
# 保存SQL脚本
with open('/tmp/distributed_tables.sql', 'w') as f:
    for cmd in distributed_sql_commands:
        f.write(cmd + "\n")
print("分布式表创建脚本已生成")
EOF
}
# 配置防火墙
configure_firewall() {
    log_info "配置防火墙规则..."
    local ports=(9000 9009 8123 8443 2181)
    if command -v firewall-cmd &> /dev/null; then
        for port in "${ports[@]}"; do
            firewall-cmd --permanent --add-port=${port}/tcp 2>/dev/null || true
        done
        firewall-cmd --reload
    elif command -v ufw &> /dev/null; then
        for port in "${ports[@]}"; do
            ufw allow ${port}/tcp
        done
        ufw reload
    fi
    log_info "防火墙配置完成"
}
# 启动服务
start_services() {
    log_info "启动ClickHouse服务..."
    systemctl daemon-reload
    systemctl enable clickhouse-server
    systemctl start clickhouse-server
    # 等待服务启动
    sleep 5
    if systemctl is-active --quiet clickhouse-server; then
        log_info "ClickHouse服务启动成功"
    else
        log_error "ClickHouse服务启动失败"
        tail -50 /var/log/clickhouse-server/clickhouse-server.err.log
        exit 1
    fi
}
# 验证集群
verify_cluster() {
    log_info "验证集群状态..."
    # 检查集群节点
    clickhouse-client -q "SELECT * FROM system.clusters WHERE cluster = '$(grep -oP 'cluster: \K.*' cluster-config.yaml)'" FORMAT PrettyCompact
    # 检查复制表
    clickhouse-client -q "SELECT * FROM system.replicas" FORMAT PrettyCompact
    log_info "集群验证完成"
}
# 主函数
main() {
    check_root
    local config_file=$1
    if [ ! -f "$config_file" ]; then
        log_error "配置文件 $config_file 不存在"
        exit 1
    fi
    log_info "开始部署ClickHouse集群..."
    # 安装依赖
    install_dependencies
    # 读取配置中的版本号
    local version=$(python3 -c "import yaml; print(yaml.safe_load(open('$config_file'))['clickhouse']['version'])")
    # 配置SSH免密登录
    setup_ssh "$config_file"
    # 安装ClickHouse
    install_clickhouse "$version"
    # 生成集群配置
    generate_cluster_config "$config_file"
    # 创建分布式表
    create_distributed_tables "$config_file"
    # 配置防火墙
    configure_firewall
    # 启动服务
    start_services
    # 创建分布式表
    clickhouse-client --multiquery < /tmp/distributed_tables.sql
    # 验证集群
    verify_cluster
    log_info "ClickHouse集群部署完成！"
    log_info "以下节点已配置完成："
    python3 -c "
import yaml
with open('$config_file', 'r') as f:
    config = yaml.safe_load(f)
for node in config['nodes']:
    print(f\"  - {node['hostname']} ({node['host']}): Shard {node['shard']}, Replica {node['replica']}\")
"
}
# 执行主函数
main "$@"

一键部署脚本

deploy-cluster.sh

#!/bin/bash
set -e
echo "=== ClickHouse集群快速部署脚本 ==="
echo ""
# 检查参数
if [ $# -lt 1 ]; then
    echo "用法: $0 <配置文件路径>"
    echo "示例: $0 cluster-config.yaml"
    exit 1
fi
CONFIG_FILE=$1
# 配置检查
echo "1. 检查配置文件..."
python3 -c "
import yaml
try:
    with open('$CONFIG_FILE', 'r') as f:
        config = yaml.safe_load(f)
        print('配置文件格式正确')
        print(f'集群名称: {config[\"cluster\"][\"name\"]}')
        print(f'分片数: {config[\"cluster\"][\"shards\"]}')
        print(f'副本数: {config[\"cluster\"][\"replicas\"]}')
        print(f'节点数: {len(config[\"nodes\"])}')
        print(f'ClickHouse版本: {config[\"clickhouse\"][\"version\"]}')
except Exception as e:
    print(f'配置文件错误: {e}')
    exit(1)
"
# 确认部署
echo ""
echo "2. 确认部署信息"
read -p "是否继续部署? (y/n): " confirm
if [[ $confirm != "y" && $confirm != "Y" ]]; then
    echo "部署已取消"
    exit 0
fi
# 获取本地IP
LOCAL_IP=$(hostname -I | awk '{print $1}')
python3 -c "
import yaml
with open('$CONFIG_FILE', 'r') as f:
    config = yaml.safe_load(f)
for node in config['nodes']:
    if node['host'] == '$LOCAL_IP':
        print(f'本机节点: {node[\"hostname\"]} (Shard {node[\"shard\"]}, Replica {node[\"replica\"]})')
        break
"
# 开始部署
echo ""
echo "3. 开始部署..."
bash install-clickhouse-cluster.sh "$CONFIG_FILE"
# 检查部署结果
if [ $? -eq 0 ]; then
    echo ""
    echo "=== 部署完成! ==="
    echo ""
    echo "管理命令:"
    echo "  查看集群状态: clickhouse-client -q \"SELECT * FROM system.clusters FORMAT PrettyCompact\""
    echo "  查看复制表: clickhouse-client -q \"SELECT * FROM system.replicas FORMAT PrettyCompact\""
    echo "  查询数据: clickhouse-client -q \"SELECT * FROM cluster_db.distributed_table\""
    echo ""
    echo "Web界面: http://$LOCAL_IP:8123/"
    echo "用户名: default"
    echo "密码: (空)"
else
    echo "部署失败，请检查日志"
    exit 1
fi

使用说明

配置使用步骤：

  1. 生成配置文件：

    # 根据实际环境修改 cluster-config.yaml
# 修改IP地址、主机名、分片配置等

  2. 在第一个节点执行：

    # 上传脚本到第一个节点
scp install-clickhouse-cluster.sh deploy-cluster.sh cluster-config.yaml root@192.168.1.10:/root/

登录到第一个节点执行

ssh root@192.168.1.10 chmod +x install-clickhouse-cluster.sh deploy-cluster.sh ./deploy-cluster.sh cluster-config.yaml


3. **在其他节点执行**：
```bash
# 在其他节点重复相同的操作
scp install-clickhouse-cluster.sh deploy-cluster.sh cluster-config.yaml root@192.168.1.11:/root/
ssh root@192.168.1.11
chmod +x install-clickhouse-cluster.sh deploy-cluster.sh
./deploy-cluster.sh cluster-config.yaml

常见问题排查：

# 检查服务状态
systemctl status clickhouse-server
# 查看日志
tail -f /var/log/clickhouse-server/clickhouse-server.log
# 测试连接
clickhouse-client -h localhost -q "SELECT 1"
# 检查集群配置
clickhouse-client -q "SELECT * FROM system.clusters"
# 检查ZooKeeper连接
clickhouse-client -q "SELECT * FROM system.zookeeper WHERE path = '/'"

这个脚本会自动完成ClickHouse集群的安装、配置、表创建和验证，根据实际环境修改配置文件中的IP地址和配置参数后即可使用。

上一篇脚本如何检测卫星通信是否正常

下一篇自动化脚本如何清理未使用标签

相关文章

抱歉，评论功能暂时关闭!