IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

自动配置HDFS集群的脚本

wen 实用脚本 7

我来提供一个自动配置HDFS集群的Shell脚本，这个脚本可以自动配置和部署Hadoop HDFS集群。

自动配置HDFS集群的脚本

HDFS集群自动配置脚本

主配置脚本：auto_config_hdfs.sh

#!/bin/bash
# HDFS集群自动配置脚本
# 支持CentOS/RHEL 7+ 和 Ubuntu 18.04+
set -e
# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# 日志函数
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}
log_step() {
    echo -e "${BLUE}[STEP]${NC} $1"
}
# 检查是否为root用户
check_root() {
    if [[ $EUID -ne 0 ]]; then
        log_error "请以root用户运行此脚本"
        exit 1
    fi
}
# 检测操作系统
detect_os() {
    if [[ -f /etc/os-release ]]; then
        . /etc/os-release
        OS=$ID
        OS_VERSION=$VERSION_ID
    else
        log_error "无法检测操作系统类型"
        exit 1
    fi
    log_info "检测到操作系统: $OS $OS_VERSION"
}
# 安装Java
install_java() {
    log_step "安装Java..."
    if command -v java &> /dev/null; then
        log_info "Java已安装，版本：$(java -version 2>&1 | head -n 1)"
        return 0
    fi
    case $OS in
        centos|rhel|fedora)
            yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel
            ;;
        ubuntu|debian)
            apt-get update
            apt-get install -y openjdk-8-jdk
            ;;
        *)
            log_error "不支持的操作系统类型"
            exit 1
            ;;
    esac
    # 设置Java环境变量
    JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
    echo "export JAVA_HOME=$JAVA_HOME" >> /etc/profile
    echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> /etc/profile
    log_info "Java安装完成"
}
# 下载和解压Hadoop
download_hadoop() {
    local HADOOP_VERSION=${1:-"3.3.6"}
    local HADOOP_HOME=${2:-"/opt/hadoop"}
    log_step "下载Hadoop ${HADOOP_VERSION}..."
    if [[ -d "$HADOOP_HOME" ]]; then
        log_warn "Hadoop目录已存在，跳过下载"
        return 0
    fi
    # 创建临时目录
    mkdir -p /tmp/hadoop_install
    cd /tmp/hadoop_install
    # 下载Hadoop
    local HADOOP_URL="https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz"
    if [[ ! -f "hadoop-${HADOOP_VERSION}.tar.gz" ]]; then
        wget -q --show-progress "$HADOOP_URL" || {
            log_error "下载Hadoop失败，请检查网络连接"
            exit 1
        }
    fi
    # 解压到指定目录
    tar -xzf "hadoop-${HADOOP_VERSION}.tar.gz" -C /opt/
    ln -sfn "/opt/hadoop-${HADOOP_VERSION}" "$HADOOP_HOME"
    log_info "Hadoop下载和解压完成"
}
# 创建Hadoop用户和组
create_hadoop_users() {
    log_step "创建Hadoop用户和组..."
    # 创建hadoop组
    if ! getent group hadoop > /dev/null; then
        groupadd hadoop
    fi
    # 创建hadoop用户
    if ! id -u hadoop > /dev/null 2>&1; then
        useradd -m -g hadoop -s /bin/bash hadoop
        echo "hadoop:hadoop" | chpasswd
    fi
    # 创建各组件用户
    local users=("hdfs" "yarn" "mapred")
    for user in "${users[@]}"; do
        if ! id -u $user > /dev/null 2>&1; then
            useradd -m -g hadoop -s /bin/bash $user
            echo "$user:$user" | chpasswd
        fi
    done
    log_info "Hadoop用户创建完成"
}
# 配置SSH免密登录
configure_ssh() {
    local HADOOP_HOME=${1:-"/opt/hadoop"}
    local HADOOP_USER=${2:-"hadoop"}
    log_step "配置SSH免密登录..."
    # 安装SSH
    case $OS in
        centos|rhel|fedora)
            yum install -y openssh-server openssh-clients
            ;;
        ubuntu|debian)
            apt-get install -y openssh-server openssh-client
            ;;
    esac
    # 生成SSH密钥
    su - $HADOOP_USER -c "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa"
    su - $HADOOP_USER -c "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"
    su - $HADOOP_USER -c "chmod 600 ~/.ssh/authorized_keys"
    # 配置SSH
    sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config
    sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
    # 重启SSH服务
    systemctl restart sshd
    log_info "SSH免密登录配置完成"
}
# 配置环境变量
configure_env() {
    local HADOOP_HOME=${1:-"/opt/hadoop"}
    log_step "配置环境变量..."
    # 创建hadoop环境配置文件
    cat > /etc/profile.d/hadoop.sh << 'EOF'
# Hadoop Environment Variables
export HADOOP_HOME=/opt/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_YARN_HOME=$HADOOP_HOME
export HADOOP_LOG_DIR=$HADOOP_HOME/logs
export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
EOF
    chmod +x /etc/profile.d/hadoop.sh
    source /etc/profile.d/hadoop.sh
    log_info "环境变量配置完成"
}
# 配置core-site.xml
configure_core_site() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    local NAMENODE_HOST=${2:-"localhost"}
    log_step "配置core-site.xml..."
    cat > "$HADOOP_CONF_DIR/core-site.xml" << EOF
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://${NAMENODE_HOST}:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/data/hadoop/tmp</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
    </property>
</configuration>
EOF
    log_info "core-site.xml配置完成"
}
# 配置hdfs-site.xml
configure_hdfs_site() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    local REPLICATION=${2:-"3"}
    log_step "配置hdfs-site.xml..."
    cat > "$HADOOP_CONF_DIR/hdfs-site.xml" << EOF
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/data/hadoop/namenode</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/data/hadoop/datanode</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>${REPLICATION}</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>0.0.0.0:50090</value>
    </property>
    <property>
        <name>dfs.namenode.http-address</name>
        <value>0.0.0.0:9870</value>
    </property>
    <property>
        <name>dfs.datanode.http.address</name>
        <value>0.0.0.0:9864</value>
    </property>
    <property>
        <name>dfs.permissions.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>dfs.name.dir</name>
        <value>/data/hadoop/namenode</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/data/hadoop/datanode</value>
    </property>
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>
</configuration>
EOF
    log_info "hdfs-site.xml配置完成"
}
# 配置workers/slaves文件
configure_workers() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    log_step "配置workers文件..."
    # 创建workers文件
    cat > "$HADOOP_CONF_DIR/workers" << 'EOF'
# Hadoop Worker Nodes
localhost
EOF
    # Hadoop 3.x使用workers，2.x使用slaves
    if [[ -f "$HADOOP_CONF_DIR/slaves" ]]; then
        cp "$HADOOP_CONF_DIR/workers" "$HADOOP_CONF_DIR/slaves"
    fi
    log_info "workers文件配置完成"
}
# 创建数据目录
create_data_dirs() {
    local HADOOP_USER=${1:-"hadoop"}
    log_step "创建HDFS数据目录..."
    # 创建数据目录
    mkdir -p /data/hadoop/{namenode,datanode,tmp}
    # 设置权限
    chown -R $HADOOP_USER:hadoop /data/hadoop
    chmod -R 755 /data/hadoop
    log_info "数据目录创建完成"
}
# 配置hadoop-env.sh
configure_hadoop_env() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    log_step "配置hadoop-env.sh..."
    # 获取Java路径
    JAVA_HOME_PATH=$(dirname $(dirname $(readlink -f $(which java))))
    # 更新hadoop-env.sh
    sed -i "s|# export JAVA_HOME=|export JAVA_HOME=${JAVA_HOME_PATH}|" "$HADOOP_CONF_DIR/hadoop-env.sh"
    # 添加额外配置
    cat >> "$HADOOP_CONF_DIR/hadoop-env.sh" << 'EOF'
# HDFS Configuration
export HDFS_NAMENODE_USER=hdfs
export HDFS_DATANODE_USER=hdfs
export HDFS_SECONDARYNAMENODE_USER=hdfs
# YARN Configuration
export YARN_RESOURCEMANAGER_USER=yarn
export YARN_NODEMANAGER_USER=yarn
# MapReduce Configuration
export MAPRED_HISTORYSERVER_USER=mapred
export HADOOP_LOG_DIR=/opt/hadoop/logs
export HADOOP_PID_DIR=/opt/hadoop/pids
EOF
    log_info "hadoop-env.sh配置完成"
}
# 初始化NameNode
initialize_namenode() {
    local HADOOP_HOME=${1:-"/opt/hadoop"}
    local HADOOP_USER=${2:-"hadoop"}
    log_step "初始化NameNode..."
    # 切换到hadoop用户执行初始化
    su - $HADOOP_USER -c "$HADOOP_HOME/bin/hdfs namenode -format -force"
    log_info "NameNode初始化完成"
}
# 启动HDFS服务
start_hdfs() {
    local HADOOP_HOME=${1:-"/opt/hadoop"}
    local HADOOP_USER=${2:-"hadoop"}
    log_step "启动HDFS服务..."
    # 启动NameNode
    su - $HADOOP_USER -c "$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode"
    # 启动DataNode
    su - $HADOOP_USER -c "$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode"
    # 启动SecondaryNameNode
    su - $HADOOP_USER -c "$HADOOP_HOME/sbin/hadoop-daemon.sh start secondarynamenode"
    log_info "HDFS服务启动完成"
}
# 验证HDFS集群
verify_cluster() {
    local HADOOP_HOME=${1:-"/opt/hadoop"}
    log_step "验证HDFS集群状态..."
    # 检查进程
    echo -e "${YELLOW}HDFS进程状态：${NC}"
    jps | grep -E "NameNode|DataNode|SecondaryNameNode"
    # 检查HDFS状态
    echo -e "\n${YELLOW}HDFS报告：${NC}"
    $HADOOP_HOME/bin/hdfs dfsadmin -report
    # 创建测试目录
    $HADOOP_HOME/bin/hdfs dfs -mkdir -p /user/hadoop
    $HADOOP_HOME/bin/hdfs dfs -ls /
    log_info "HDFS集群验证完成"
}
# 配置防火墙
configure_firewall() {
    log_step "配置防火墙..."
    case $OS in
        centos|rhel|fedora)
            systemctl stop firewalld
            systemctl disable firewalld
            ;;
        ubuntu|debian)
            ufw disable
            ;;
    esac
    log_info "防火墙配置完成"
}
# 主函数
main() {
    # 默认配置
    HADOOP_VERSION=${HADOOP_VERSION:-"3.3.6"}
    HADOOP_HOME=${HADOOP_HOME:-"/opt/hadoop"}
    HADOOP_USER=${HADOOP_USER:-"hadoop"}
    NAMENODE_HOST=${NAMENODE_HOST:-"localhost"}
    REPLICATION_FACTOR=${REPLICATION_FACTOR:-"3"}
    echo -e "${GREEN}========================================${NC}"
    echo -e "${GREEN}    HDFS集群自动配置脚本 v1.0${NC}"
    echo -e "${GREEN}========================================${NC}"
    echo ""
    # 执行配置步骤
    check_root
    detect_os
    install_java
    create_hadoop_users
    configure_ssh "$HADOOP_HOME" "$HADOOP_USER"
    download_hadoop "$HADOOP_VERSION" "$HADOOP_HOME"
    configure_env "$HADOOP_HOME"
    configure_core_site "$HADOOP_HOME/etc/hadoop" "$NAMENODE_HOST"
    configure_hdfs_site "$HADOOP_HOME/etc/hadoop" "$REPLICATION_FACTOR"
    configure_workers "$HADOOP_HOME/etc/hadoop"
    configure_hadoop_env "$HADOOP_HOME/etc/hadoop"
    create_data_dirs "$HADOOP_USER"
    configure_firewall
    # 询问是否初始化并启动HDFS
    read -p "$(echo -e ${YELLOW}"是否要初始化NameNode并启动HDFS? (y/n): "${NC})" START_HDFS
    if [[ $START_HDFS =~ ^[Yy]$ ]]; then
        initialize_namenode "$HADOOP_HOME" "$HADOOP_USER"
        start_hdfs "$HADOOP_HOME" "$HADOOP_USER"
        verify_cluster "$HADOOP_HOME"
    else
        log_info "跳过HDFS初始化，手动初始化命令："
        echo "  su - $HADOOP_USER -c '$HADOOP_HOME/bin/hdfs namenode -format'"
        echo "  su - $HADOOP_USER -c '$HADOOP_HOME/sbin/start-dfs.sh'"
    fi
    # 显示配置信息
    echo -e "\n${GREEN}========================================${NC}"
    echo -e "${GREEN}    HDFS集群配置完成${NC}"
    echo -e "${GREEN}========================================${NC}"
    echo -e "Hadoop安装目录: ${HADOOP_HOME}"
    echo -e "配置文件目录: ${HADOOP_HOME}/etc/hadoop"
    echo -e "数据目录: /data/hadoop"
    echo -e "NameNode Web UI: http://${NAMENODE_HOST}:9870"
    echo -e "HDFS默认端口: 9000"
    echo ""
    echo -e "常用命令:"
    echo -e "  ${YELLOW}hdfs dfs -ls /${NC} - 查看HDFS根目录"
    echo -e "  ${YELLOW}hdfs dfsadmin -report${NC} - 查看HDFS集群状态"
    echo -e "  ${YELLOW}hadoop-daemon.sh start namenode${NC} - 启动NameNode"
    echo -e "  ${YELLOW}hadoop-daemon.sh start datanode${NC} - 启动DataNode"
}
# 执行主函数
main "$@"

使用方法

# 1. 下载脚本
wget https://your-server/auto_config_hdfs.sh
# 2. 添加执行权限
chmod +x auto_config_hdfs.sh
# 3. 以root用户运行
sudo ./auto_config_hdfs.sh
# 4. 使用自定义参数运行
HADOOP_VERSION=3.3.6 \
HADOOP_HOME=/opt/hadoop \
HADOOP_USER=hadoop \
NAMENODE_HOST=master-node \
REPLICATION_FACTOR=3 \
./auto_config_hdfs.sh

多节点集群配置（扩展脚本）

#!/bin/bash
# 多节点HDFS集群自动配置脚本
# 需要在所有节点上运行
CONFIG_NODES=(
    "master-node:192.168.1.10"
    "worker1:192.168.1.11"
    "worker2:192.168.1.12"
    "worker3:192.168.1.13"
)
configure_hosts_file() {
    log_step "配置hosts文件..."
    # 备份原hosts文件
    cp /etc/hosts /etc/hosts.backup
    # 添加节点信息
    for node in "${CONFIG_NODES[@]}"; do
        local hostname=$(echo $node | cut -d: -f1)
        local ip=$(echo $node | cut -d: -f2)
        if ! grep -q "$hostname" /etc/hosts; then
            echo "$ip    $hostname" >> /etc/hosts
        fi
    done
    log_info "hosts文件配置完成"
}
# 分发SSH密钥到所有节点
distribute_ssh_keys() {
    local HADOOP_USER=${1:-"hadoop"}
    log_step "分发SSH密钥到所有节点..."
    for node in "${CONFIG_NODES[@]}"; do
        local hostname=$(echo $node | cut -d: -f1)
        local ip=$(echo $node | cut -d: -f2)
        if [[ "$(hostname)" != "$hostname" ]]; then
            su - $HADOOP_USER -c "ssh-copy-id -i ~/.ssh/id_rsa.pub $HADOOP_USER@$hostname"
        fi
    done
    log_info "SSH密钥分发完成"
}
# 更新workers文件
update_workers_file() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    log_step "更新workers文件..."
    # 清空workers文件
    > "$HADOOP_CONF_DIR/workers"
    # 添加worker节点
    for node in "${CONFIG_NODES[@]}"; do
        local hostname=$(echo $node | cut -d: -f1)
        if [[ "$hostname" != "master-node" ]]; then
            echo "$hostname" >> "$HADOOP_CONF_DIR/workers"
        fi
    done
    log_info "workers文件更新完成"
}
# 同步配置到所有节点
sync_config_to_nodes() {
    local HADOOP_CONF_DIR=${1:-"/opt/hadoop/etc/hadoop"}
    local HADOOP_USER=${2:-"hadoop"}
    log_step "同步配置到所有节点..."
    for node in "${CONFIG_NODES[@]}"; do
        local hostname=$(echo $node | cut -d: -f1)
        if [[ "$(hostname)" != "$hostname" ]]; then
            # 同步Hadoop配置
            su - $HADOOP_USER -c "scp -r $HADOOP_CONF_DIR/* $HADOOP_USER@$hostname:$HADOOP_CONF_DIR/"
            # 同步环境变量
            scp /etc/profile.d/hadoop.sh $hostname:/etc/profile.d/
        fi
    done
    log_info "配置同步完成"
}

快速配置示例

#!/bin/bash
# 一键配置单节点HDFS集群
curl -sSL https://raw.githubusercontent.com/your-repo/hadoop-scripts/main/auto_config_hdfs.sh | bash
# 或者使用本地脚本
./auto_config_hdfs.sh <<< "y"
# 静默安装（不交互）
HADOOP_VERSION=3.3.6 \
HADOOP_HOME=/opt/hadoop \
HADOOP_USER=hadoop \
NAMENODE_HOST=localhost \
REPLICATION_FACTOR=1 \
START_HDFS=y \
./auto_config_hdfs.sh

这个脚本会自动完成以下配置：

  1. 环境准备：安装Java、配置SSH免密登录
  2. Hadoop安装：下载并解压指定版本的Hadoop
  3. 用户创建：创建hadoop、hdfs、yarn等系统用户
  4. 配置文件：自动生成core-site.xml、hdfs-site.xml等配置文件
  5. 目录创建：创建NameNode和DataNode数据目录
  6. 服务启动：初始化NameNode并启动HDFS服务
  7. 验证测试：检查集群状态并创建测试目录

使用前请确保网络连接正常，并有足够的磁盘空间（建议至少10GB）。

上一篇Python脚本如何操作数据库分区表

下一篇自动化脚本如何清理未使用角色

相关文章

抱歉，评论功能暂时关闭!