Install Hadoop eco-system singlemode

install openjdk debian

sudo apt-get install openjdk-7-jdk ssh rsync
 
sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser
 
su - hduser
ssh-keygen -t rsa -P ""
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys

Edit ~/.bashrc

export HADOOP_HOME=/usr/local/hadoop/jre
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export ZOOKEEPER_PREFIX=/usr/local/zookeeper
 
unalias hfs &> /dev/null
alias hfs="hadoop fs"
unalias hls &> /dev/null
alias hls="fs -ls"
 
lzohead () {
    hadoop fs -cat $1 | lzop -dc | head -1000 | less
}
 
export PATH=$PATH:$HADOOP_HOME/bin

Install Hadoop

cd /tmp
wget http://apache.mivzakim.net/hadoop/common/stable/hadoop-2.5.0.tar.gz
tar xzf hadoop-2.5.0.tar.gz
sudo mv hadoop-2.5.0 /usr/local/
sudo ln -s /usr/local/hadoop-* /usr/local/hadoop
sudo chown -R hduser:hadoop /usr/local/hadoop*

Edit /usr/local/hadoop/etc/hadoop/mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
  <name>mapred.job.tracker</name>
  <value>localhost:54311</value>
  <description>The host and port that the MapReduce job tracker runs
  at.  If "local", then jobs are run in-process as a single map
  and reduce task.
  </description>
</property>
</configuration>

Edit /usr/local/hadoop/etc/conf/core-site.xml

<configuration>
<property>
  <name>hadoop.tmp.dir</name>
  <value>/home/hduser/hadoop.tmp</value>
  <description>A base for other temporary directories.</description>
</property>
 
<property>
  <name>fs.default.name</name>
  <value>hdfs://localhost:54310</value>
  <description>The name of the default file system.  A URI whose
  scheme and authority determine the FileSystem implementation.  The
  uri's scheme determines the config property (fs.SCHEME.impl) naming
  the FileSystem implementation class.  The uri's authority is used to
  determine the host, port, etc. for a filesystem.</description>
</property>
</configuration>

Format

cd /usr/local/hadoop/
bin/hadoop namenode -format

Zookeeper

wget http://apache.spd.co.il/zookeeper/stable/zookeeper-3.4.6.tar.gz
tar -zxvf zookeeper*
sudo mv zookeeper-3.4.6 /usr/local/
sudo ln -s /usr/local/zookeeper-* /usr/local/zookeeper
sudo chown -R hduser:hadoop /usr/local/zookeeper*
vi /usr/local/zookeeper/conf/zoo.cfg
tickTime=2000
dataDir=/var/zookeeper
clientPort=2181
sudo mkdir /var/zookeeper
sudo chown -R hduser:hadoop /var/zookeeper

HBase

wget http://apache.mivzakim.net/hbase/stable/hbase-0.98.5-hadoop2-bin.tar.gz
tar -zxvf hbase*
sudo mv hbase-0.98.5-hadoop2 /usr/local/
sudo ln -s /usr/local/hbase-* /usr/local/hbase
sudo chown -R hduser:hadoop /usr/local/hbase*

Edit /usr/local/hbase/conf/hbase-env.sh

export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64

Edit /usr/local/hbase/conf/hbase-site.xml

<configuration>
  <property>
    <name>hbase.rootdir</name>
    <value>file:///home/hduser/hbase</value>
  </property>
  <property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value>/home/hduser/zookeeper</value>
  </property>
</configuration>   

Start ALL

cd /usr/local/hadoop
sbin/start-all.sh
 
cd /usr/local/zookeeper
bin/zkServer.sh start
 
cd /usr/local/hbase
bin/start-hbase.sh
 
#start hbase shell
./bin/hbase shell
learn/bigdata/install_hadoop_eco_system_single_mode.txt · Last modified: 2014/08/14 19:55 by yehuda
Back to top
Driven by DokuWiki Recent changes RSS feed Valid CSS Valid XHTML 1.0