Ubuntu Hadoop分布式部署
(1) vim /usr/local/hadoop-3.2.2/etc/hadoop/hadoop-env.sh
(/usr/local/hadoop-3.2.2为Hadoop安装目录)
export JAVA_HOME=/usr/lib/jvm/jdk-15.0.2
(jdk安装目录)
(2) vim /usr/local/hadoop-3.2.2/etc/hadoop/core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/hadoop-3.2.2/tmp</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
(3) vim /usr/local/hadoop-3.2.2/etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop-3.2.2/hadoop_data/hdfs/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop-3.2.2/hadoop_data/hdfs/datanode</value>
</property>
</configuration>
(4) vim /usr/local/hadoop-3.2.2/etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.application.classpath</name> <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
</configuration>
(5) vim /usr/local/hadoop-3.2.2/etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,
HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,
HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
</configuration>
(6) vim .bashrc
#Java Enviromemt
export JAVA_HOME=/usr/lib/jvm/jdk-15.0.2
export PATH=${JAVA_HOME}/bin:$PATH
#Hadoop Enviroment
export HADOOP_HOME=/usr/local/hadoop-3.2.2
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath):$CLASSPATH
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
(7) hadoop classpath (将命令执行后出现的内容复制保存到文档)
(8) vim /usr/local/hadoop-3.2.2/etc/hadoop/yarn-site.xml
(</configuration>前面加入)
<property>
<name>yarn.application.classpath</name>
<value>步骤(6)复制的内容</value>
</property>
(9) source .bashrc
(10) hdfs namenode -format (格式化,只能执行一次)
(11) start-dfs.sh
(12) jps (查看进程是否启动)
出现三个进程
NameNode
DataNode
SecondaryNameNode
(13) start-yarn.sh
(14) jps
ResourceManager
NodeManager
(15) cd /usr/local/hadoop-3.2.2/share/hadoop/mapreduce
(查看hadoop-mapreduce-examples-3.2.2.jar)
(16) cd
(17) 跑一个程序:计算圆周率
hadoop jar /usr/local/hadoop-3.2.2/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.2.2.jar pi 5 10
(执行有问题再执行一次)