Hadoop
docker
docker run -it --name=hadoop3 centos
준비
yum update -y
yum install wget -y
yum install vim -y
yum install openssh-server openssh-clients openssh-askpass -y
java설치
wget https://corretto.aws/downloads/latest/amazon-corretto-8-x64-linux-jdk.tar.gz
tar xvzf amazon-corretto-8-x64-linux-jdk.tar.gz
mv amazon-corretto-8.265.01.1-linux-x64 /java
chown -R root:root /java
hadoop설치
wget http://mirror.apache-kr.org/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz
tar xvzf hadoop-3.2.1.tar.gz
mv hadoop-3.2.1 /hadoop
chown -R root:root /hadoop
ssh key
ssh-keygen -t rsa -P '' -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
# ssh
cd /etc/ssh
ssh-keygen -t rsa -P '' -f ssh_host_rsa_key
ssh-keygen -t rsa -P '' -f ssh_host_ecdsa_key
ssh-keygen -t rsa -P '' -f ssh_host_ed25519_key
mkdir /var/run/sshd
변수설정
vim ~/.bashrc
# java
export JAVA_HOME=/java
export PATH=$PATH:$JAVA_HOME/bin
# hadoop
export HADOOP_HOME=/hadoop
export HADOOP_CONFIG_HOME=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
# hadoop user
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
# sshd
/usr/sbin/sshd
source ~/.bashrc
java -version
javac -version
node home 생성
mkdir /hadoop/temp
mkdir /hadoop/namenode_home
mkdir /hadoop/secondary_home
mkdir /hadoop/datanode_home
hadoop 설정
cd $HADOOP_CONFIG_HOME
vim hadoop-env.sh
export JAVA_HOME=$JAVA_HOME
export HADOOP_HOME=$HADOOP_HOME
export HADOOP_CONF_DIR=$HADOOP_CONFIG_HOME
vim core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/hadoop/temp</value>
</property>vim hdfs-site.xml
<property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/namenode_home</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/hadoop/secondary_home</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/datanode_home</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>slave1:50090</value>
</property>
<property>
<name>dfs.namenode.secondary.https-address</name>
<value>slave1:50091</value>
</property>vim mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</propertyvim yarn-site.xml
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<proerty>
<name>yarn.nodemanager.auc-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</proerty>container 생성
exit
docker commit hadoop3 hadoop3
docker run -it -h master --name master -p 9870:9870 -p 8088:8088 -p 8081:8080 - p8082:8081 hadoop3
exit
docker start master
docker run -it -h slave1 --name slave1 --link master:master hadoop3
exit
docker start slave1
docker run -it -h slave2 --name slave2 --link master:master hadoop3
exit
docker start slave2
docker run -it -h slave2 --name slave3 --link master:master hadoop3
exit
docker start slave3
docker container ip 확인
# window
docker inspect slave1 | find "IPAddress"
docker inspect slave2 | find "IPAddress"
docker inspect slave3 | find "IPAddress"
# linux / mac
docker inspect slave1 | grep IPAddress
docker inspect slave2 | grep IPAddress
docker inspect slave3 | grep IPAddress
node 연결
master에서 작업
vim /etc/hosts
172.17.0.2 master
172.17.0.3 slave1
172.17.0.4 slave2
172.17.0.5 slave3
source /etc/hosts
ssh 연결 확인
ssh slave1
yes
exit
ssh slave2
yes
exit
ssh slave3
yes
exit
cd $HADOOP_CONFIG_HOME
vim master
master
vim workers
slave1 s
lave2
slave3
시작
hadoop namenode -format
hadoop datanode -format
start-dfs.sh
start-yarn.sh
hdfs dfsadmin -report
localhost:9870
localhost:8088
stop-dfs.sh
stop-yarn.sh
업로드
docker cp 파일경로 master:/
# docker cp /Users/nonen/Documents/goodnews.txt master:/
docker exec -it master bash
hdfs dfs -mkdir /upload
hdfs dfs -put /Genesis.txt /upload
hdfs dfs -ls /upload