详解搭建ubuntu版hadoop集群

127.0.0.1  localhost192.168.1.100  s100192.168.1.101  s101192.168.1.102  s102192.168.1.103  s103192.168.1.104  s104192.168.1.105  s105

#interfaces(5) file used by ifup(8) and ifdown(8)#The loopback network interfaceauto loiface lo inet loopback#iface eth0 inet staticiface eth0 inet staticaddress 192.168.1.105netmask 255.255.255.0gateway 192.168.1.2dns-nameservers 192.168.1.2auto eth0

127.0.0.1    localhost192.168.1.100 s100192.168.1.101 s101192.168.1.102 s102192.168.1.103 s103192.168.1.104 s104192.168.1.105 s105

$>cd /etc/apt/$>gedit sources.list

deb http://mirrors.163.com/ubuntu/ trusty main restricted universe multiversedeb http://mirrors.163.com/ubuntu/ trusty-security main restricted universe multiversedeb http://mirrors.163.com/ubuntu/ trusty-updates main restricted universe multiversedeb http://mirrors.163.com/ubuntu/ trusty-proposed main restricted universe multiversedeb http://mirrors.163.com/ubuntu/ trusty-backports main restricted universe multiversedeb-src http://mirrors.163.com/ubuntu/ trusty main restricted universe multiversedeb-src http://mirrors.163.com/ubuntu/ trusty-security main restricted universe multiversedeb-src http://mirrors.163.com/ubuntu/ trusty-updates main restricted universe multiversedeb-src http://mirrors.163.com/ubuntu/ trusty-proposed main restricted universe multiversedeb-src http://mirrors.163.com/ubuntu/ trusty-backports main restricted universe multiverse

$>apt-get update

$> sudo cp hadoop-2.7.2.tar.gz jdk-8u65-linux-x64.tar.gz ~/Downloads/

$> tar -zxvf hadoop-2.7.2.tar.gz$>tar -zxvf jdk-8u65-linux-x64.tar.gz$>cp -r hadoop-2.7.2 /soft$>cp -r jdk1.8.0_65/ /soft

$>ln -s hadoop-2.7.2/ hadoop$>ln -s jdk1.8.0_65/ jdk$>ls -ll

$>vim /etc/environment

JAVA_HOME=/soft/jdkHADOOP_HOME=/soft/hadoopPATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/soft/jdk/bin:/soft/hadoop/bin:/soft/hadoop/sbin"

$>source environment

$>java –version

$>hadoop version

<configuration>  <property>    <name>fs.defaultFS</name>    <value>hdfs://s100/</value>  </property>  <property>     <name>hadoop.tmp.dir</name>     <value>/home/enmoedu/hadoop</value>  </property></configuration>

<configuration>  <property>    <name>dfs.replication</name>    <value>3</value>  </property>  <property>     <name>dfs.namenode.secondary.http-address</name>      <value>s104:50090</value>   <description>    The secondary namenode http server address and port.   </description></property></configuration>

<configuration>  <property>    <name>mapreduce.framework.name</name>    <value>yarn</value>  </property></configuration>

<configuration>  <property>    <name>yarn.resourcemanager.hostname</name>    <value>s100</value>  </property>  <property>    <name>yarn.nodemanager.aux-services</name>    <value>mapreduce_shuffle</value>  </property></configuration>

$>sudo apt-get install ssh

$>ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$>ssh localhost

s101s102s103s105

$>hadoop namenode –format

start-all.sh

#!/bin/bashpcount=$#if (( pcount<1 ));then  echo no args;  exit;fip1=$1;fname=`basename $p1`#echo $fname=$fname;pdir=`cd -P $(dirname $p1) ; pwd`#echo pdir=$pdircuser=`whoami`for (( host=101;host<106;host=host+1 )); do  echo ------------s$host----------------  rsync -rvl $pdir/$fname $cuser@s$host:$pdirdone

#!/bin/bashpcount=$#if (( pcount<1 ));then  echo no args;  exit;fiecho -----------localhost----------------$@for (( host=101;host<106;host=host+1 )); do  echo ------------s$host-------------  ssh s$host $@done

touch a.txtgedit a.txthadoop fs -mkdir -p /user/enmoedu/datahadoop fs -put a.txt /user/enmoedu/datahadoop fs -lsr /