Apache storm installation on AWS Ubuntu

Storm installation on single node

1.sudo apt-get update
2.sudo apt-get install default-jre

$ sudo apt-get install git -y

$ sudo apt-get install libtool -y

$ sudo apt-get install automake -y

$ sudo apt-get install uuid-dev

$ sudo apt-get install g++ -y

$ sudo apt-get install gcc-multilib -y

*****************************************************************
1.ZooKEEPER install

https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/

$wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz


ubuntu@ip:~$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
ubuntu@ip:~$ tar -xvf zookeeper-3.4.6.tar.gz
ubuntu@ip:~$ mv zookeeper-3.4.6 zookeeper


 Optionally : a)  Add ZOOKEEPER_HOME under .bashrc b)  Add ZOOKEEPER_HOME/bin to the PATH


========================
ubuntu@ip-:~$ ls /usr/lib/jvm/
default-java  java-1.8.0-openjdk-amd64  java-8-openjdk-amd64
ubuntu@ip-:~$
ubuntu@ip-:~$ ls
zookeeper  zookeeper-3.4.6.tar.gz
ubuntu@ip-:~$ pwd
/home/ubuntu
ubuntu@ip-:~$

=========================

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
export ZOOKEEPER_HOME=/home/ubuntu/zookeeper
export PATH=$PATH:$JAVA_HOME/bin:$ZOOKEEPER_HOME/bin

create data directory
mkdir zookeeper-data

create config file

$ZOOKEEPER_HOME/conf/zoo.cfg
tickTme=2000
dataDir=/home/ubuntu/zookeeper-data
clientPort=2181


Restart the session

start zookeeper

$zkServer.sh start


https://zookeeper.apache.org/doc/current/zookeeperStarted.html#sc_InstallingSingleMode

*******************************
2.ZEROMQ

ubuntu@ip-:~$ wget http://download.zeromq.org/zeromq-2.1.7.tar.gz

ubuntu@ip-:~$ tar -xvf zeromq-2.1.7.tar.gz

ubuntu@ip-:~$ cd zeromq-2.1.7/
ubuntu@ip-:~$ ./configure
ubuntu@ip-:~$ sudo apt-get install make
ubuntu@ip-:~$ make
ubuntu@ip-:~$ sudo make install


***********************
3.java bindings for Zeromq (jzmq)
ubuntu@ip-:~$ git clone https://github.com/nathanmarz/jzmq.git
ubuntu@ip-:~$cd jzmq


 Configurating jzmq: 1) $ cd jzmq
2)  $ sed -i ‘s/classdist_noinst.stamp/classnoinst.stamp/g’ src/Makefile.am
3) $ ./autogen.sh

ubuntu@ip:~/jzmq$ ./autogen.sh
autogen.sh: error: could not find pkg-config.  pkg-config is required to run autogen.sh.


https://stackoverflow.com/questions/12115160/compiling-jzmq-on-ubuntu

ubuntu@ip-:~/jzmq/src$ CLASSPATH=.:./.:$CLASSPATH javac -d . org/zeromq/ZMQ.java org/zeromq/ZMQException.java org/zeromq/ZMQQueue.java org/zeromq/ZMQForwarder.java org/zeromq/ZMQStreamer.java
ubuntu@ip-:~/jzmq/src$

ubuntu@ip-:~/jzmq$ sudo apt-get install pkg-config

autogen.sh: error: could not find libtool. libtool is required to run autogen.sh
https://github.com/zeromq/libzmq/issues/1385
sudo ln -s /usr/bin/libtoolize /usr/bin/libtool


ubuntu@ip-:~/jzmq$./configure
ubuntu@ip-:~/jzmq$ make
ubuntu@ip-:~/jzmq$ sudo make install

**********************************************
4.configure storm

ubuntu@ip-:~$ wget http://www.apache.org/dyn/closer.lua/storm/apache-storm-0.9.1-incubating/apache-storm-0.9.1-incubating.tar.gz

wget https://github.com/downloads/nathanmarz/storm/storm-0.8.1.zip
unzip storm-0.8.1.zip


export STORM_HOME=/home/ubuntu/storm
export PATH=$PATH:$STORM_HOME/bin

Restart the session

cd $STORM_HOME
mkdir data

Edit storm.yml which is in $STORM_HOME/conf/storm.yaml

**************************************

storm.zookeeper.servers:
     - "10.0.0.194"
     - "10.0.0.195"
     - "10.0.0.196"

nimbus.host: "10.0.0.182"

storm.local.dir: "/var/storm"

supervisor.slots.ports:
    - 6700
    - 6701
    - 6702


OR


storm.zookeeper.servers:
- "localhost"
storm.zookeeper.port: 2181
nimbus.host: "localhost"
nimbus.thrift.port: 6627
ui.port: 8772
storm.local.dir: "/home/ubuntu/storm/data"
java.library.path: "/usr/lib/jvm/java-1.8.0-openjdk-amd64"
supervisor.slots.ports:
- 6700
- 6701
- 6702



*****************************************

$storm nimbus
$storm supervisor
$storm UI





ubuntu@ip-:~/storm$ storm nimbus
Running: java -server
-Dstorm.options= -Dstorm.home=/home/ubuntu/storm
                 -Djava.library.path=/usr/lib/jvm/java-1.8.0-openjdk-amd64
-cp /home/ubuntu/storm/storm-0.8.1.jar
    :/home/ubuntu/storm/lib/clout-0.4.1.jar
:/home/ubuntu/storm/lib/tools.logging-0.2.3.jar
:/home/ubuntu/storm/lib/ring-jetty-adapter-0.3.11.jar
:/home/ubuntu/storm/lib/math.numeric-tower-0.0.1.jar
:/home/ubuntu/storm/lib/commons-fileupload-1.2.1.jar
:/home/ubuntu/storm/lib/carbonite-1.5.0.jar
:/home/ubuntu/storm/lib/jetty-6.1.26.jar
:/home/ubuntu/storm/lib/json-simple-1.1.jar
:/home/ubuntu/storm/lib/curator-client-1.0.1.jar
:/home/ubuntu/storm/lib/snakeyaml-1.9.jar
:/home/ubuntu/storm/lib/reflectasm-1.07-shaded.jar
:/home/ubuntu/storm/lib/commons-lang-2.5.jar
:/home/ubuntu/storm/lib/httpclient-4.1.1.jar
:/home/ubuntu/storm/lib/jetty-util-6.1.26.jar
:/home/ubuntu/storm/lib/commons-logging-1.1.1.jar
:/home/ubuntu/storm/lib/commons-exec-1.1.jar
:/home/ubuntu/storm/lib/asm-4.0.jar
:/home/ubuntu/storm/lib/slf4j-api-1.5.8.jar
:/home/ubuntu/storm/lib/clj-time-0.4.1.jar
:/home/ubuntu/storm/lib/httpcore-4.1.jar
:/home/ubuntu/storm/lib/joda-time-2.0.jar
:/home/ubuntu/storm/lib/guava-13.0.jar
:/home/ubuntu/storm/lib/junit-3.8.1.jar
:/home/ubuntu/storm/lib/ring-core-0.3.10.jar
:/home/ubuntu/storm/lib/kryo-2.17.jar
:/home/ubuntu/storm/lib/commons-codec-1.4.jar
:/home/ubuntu/storm/lib/tools.macro-0.1.0.jar
:/home/ubuntu/storm/lib/jline-0.9.94.jar
:/home/ubuntu/storm/lib/commons-io-1.4.jar
:/home/ubuntu/storm/lib/slf4j-log4j12-1.5.8.jar
:/home/ubuntu/storm/lib/jzmq-2.1.0.jar
:/home/ubuntu/storm/lib/curator-framework-1.0.1.jar
:/home/ubuntu/storm/lib/jgrapht-0.8.3.jar
:/home/ubuntu/storm/lib/log4j-1.2.16.jar
:/home/ubuntu/storm/lib/servlet-api-2.5-20081211.jar
:/home/ubuntu/storm/lib/minlog-1.2.jar
:/home/ubuntu/storm/lib/disruptor-2.10.1.jar
:/home/ubuntu/storm/lib/libthrift7-0.7.0.jar
:/home/ubuntu/storm/lib/core.incubator-0.1.0.jar
:/home/ubuntu/storm/lib/servlet-api-2.5.jar
:/home/ubuntu/storm/lib/objenesis-1.2.jar
:/home/ubuntu/storm/lib/compojure-0.6.4.jar
:/home/ubuntu/storm/lib/clojure-1.4.0.jar
:/home/ubuntu/storm/lib/hiccup-0.3.6.jar
:/home/ubuntu/storm/lib/ring-servlet-0.3.11.jar
:/home/ubuntu/storm/lib/tools.cli-0.2.2.jar
:/home/ubuntu/storm/lib/zookeeper-3.3.3.jar
:/home/ubuntu/storm/log4j
:/home/ubuntu/storm/conf

-Xmx1024m
-Dlogfile.name=nimbus.log
-Dlog4j.configuration=storm.log.properties backtype.storm.daemon.nimbus




ubuntu@ip-:~/storm/bin$ storm supervisor
Running: java -server
-Dstorm.options= -Dstorm.home=/home/ubuntu/storm
-Djava.library.path=/usr/lib/jvm/java-1.8.0-openjdk-amd64
-cp /home/ubuntu/storm/storm-0.8.1.jar
     :/home/ubuntu/storm/lib/clout-0.4.1.jar
:/home/ubuntu/storm/lib/tools.logging-0.2.3.jar
:/home/ubuntu/storm/lib/ring-jetty-adapter-0.3.11.jar
:/home/ubuntu/storm/lib/math.numeric-tower-0.0.1.jar
:/home/ubuntu/storm/lib/commons-fileupload-1.2.1.jar
:/home/ubuntu/storm/lib/carbonite-1.5.0.jar
:/home/ubuntu/storm/lib/jetty-6.1.26.jar
:/home/ubuntu/storm/lib/json-simple-1.1.jar
:/home/ubuntu/storm/lib/curator-client-1.0.1.jar
:/home/ubuntu/storm/lib/snakeyaml-1.9.jar
:/home/ubuntu/storm/lib/reflectasm-1.07-shaded.jar
:/home/ubuntu/storm/lib/commons-lang-2.5.jar
:/home/ubuntu/storm/lib/httpclient-4.1.1.jar
:/home/ubuntu/storm/lib/jetty-util-6.1.26.jar
:/home/ubuntu/storm/lib/commons-logging-1.1.1.jar
:/home/ubuntu/storm/lib/commons-exec-1.1.jar
:/home/ubuntu/storm/lib/asm-4.0.jar
:/home/ubuntu/storm/lib/slf4j-api-1.5.8.jar
:/home/ubuntu/storm/lib/clj-time-0.4.1.jar
:/home/ubuntu/storm/lib/httpcore-4.1.jar
:/home/ubuntu/storm/lib/joda-time-2.0.jar
:/home/ubuntu/storm/lib/guava-13.0.jar
:/home/ubuntu/storm/lib/junit-3.8.1.jar
:/home/ubuntu/storm/lib/ring-core-0.3.10.jar
:/home/ubuntu/storm/lib/kryo-2.17.jar
:/home/ubuntu/storm/lib/commons-codec-1.4.jar
:/home/ubuntu/storm/lib/tools.macro-0.1.0.jar
:/home/ubuntu/storm/lib/jline-0.9.94.jar
:/home/ubuntu/storm/lib/commons-io-1.4.jar
:/home/ubuntu/storm/lib/slf4j-log4j12-1.5.8.jar
:/home/ubuntu/storm/lib/jzmq-2.1.0.jar
:/home/ubuntu/storm/lib/curator-framework-1.0.1.jar
:/home/ubuntu/storm/lib/jgrapht-0.8.3.jar
:/home/ubuntu/storm/lib/log4j-1.2.16.jar
:/home/ubuntu/storm/lib/servlet-api-2.5-20081211.jar
:/home/ubuntu/storm/lib/minlog-1.2.jar
:/home/ubuntu/storm/lib/disruptor-2.10.1.jar
:/home/ubuntu/storm/lib/libthrift7-0.7.0.jar
:/home/ubuntu/storm/lib/core.incubator-0.1.0.jar
:/home/ubuntu/storm/lib/servlet-api-2.5.jar
:/home/ubuntu/storm/lib/objenesis-1.2.jar
:/home/ubuntu/storm/lib/compojure-0.6.4.jar
:/home/ubuntu/storm/lib/clojure-1.4.0.jar
:/home/ubuntu/storm/lib/hiccup-0.3.6.jar
:/home/ubuntu/storm/lib/ring-servlet-0.3.11.jar
:/home/ubuntu/storm/lib/tools.cli-0.2.2.jar
:/home/ubuntu/storm/lib/zookeeper-3.3.3.jar
:/home/ubuntu/storm/log4j
:/home/ubuntu/storm/conf

-Xmx1024m
-Dlogfile.name=supervisor.log
-Dlog4j.configuration=storm.log.properties backtype.storm.daemon.supervisor



ubuntu@ip-:~/storm/bin$ storm ui
Running: java -server -Dstorm.options= -Dstorm.home=/home/ubuntu/storm -Djava.library.path=/usr/lib/jvm/java-1.8.0-openjdk-amd64 -cp /home/ubuntu/storm/storm-0.8.1.jar:/home/ubuntu/storm/lib/clout-0.4.1.jar:/home/ubuntu/storm/lib/tools.logging-0.2.3.jar:/home/ubuntu/storm/lib/ring-jetty-adapter-0.3.11.jar:/home/ubuntu/storm/lib/math.numeric-tower-0.0.1.jar:/home/ubuntu/storm/lib/commons-fileupload-1.2.1.jar:/home/ubuntu/storm/lib/carbonite-1.5.0.jar:/home/ubuntu/storm/lib/jetty-6.1.26.jar:/home/ubuntu/storm/lib/json-simple-1.1.jar:/home/ubuntu/storm/lib/curator-client-1.0.1.jar:/home/ubuntu/storm/lib/snakeyaml-1.9.jar:/home/ubuntu/storm/lib/reflectasm-1.07-shaded.jar:/home/ubuntu/storm/lib/commons-lang-2.5.jar:/home/ubuntu/storm/lib/httpclient-4.1.1.jar:/home/ubuntu/storm/lib/jetty-util-6.1.26.jar:/home/ubuntu/storm/lib/commons-logging-1.1.1.jar:/home/ubuntu/storm/lib/commons-exec-1.1.jar:/home/ubuntu/storm/lib/asm-4.0.jar:/home/ubuntu/storm/lib/slf4j-api-1.5.8.jar:/home/ubuntu/storm/lib/clj-time-0.4.1.jar:/home/ubuntu/storm/lib/httpcore-4.1.jar:/home/ubuntu/storm/lib/joda-time-2.0.jar:/home/ubuntu/storm/lib/guava-13.0.jar:/home/ubuntu/storm/lib/junit-3.8.1.jar:/home/ubuntu/storm/lib/ring-core-0.3.10.jar:/home/ubuntu/storm/lib/kryo-2.17.jar:/home/ubuntu/storm/lib/commons-codec-1.4.jar:/home/ubuntu/storm/lib/tools.macro-0.1.0.jar:/home/ubuntu/storm/lib/jline-0.9.94.jar:/home/ubuntu/storm/lib/commons-io-1.4.jar:/home/ubuntu/storm/lib/slf4j-log4j12-1.5.8.jar:/home/ubuntu/storm/lib/jzmq-2.1.0.jar:/home/ubuntu/storm/lib/curator-framework-1.0.1.jar:/home/ubuntu/storm/lib/jgrapht-0.8.3.jar:/home/ubuntu/storm/lib/log4j-1.2.16.jar:/home/ubuntu/storm/lib/servlet-api-2.5-20081211.jar:/home/ubuntu/storm/lib/minlog-1.2.jar:/home/ubuntu/storm/lib/disruptor-2.10.1.jar:/home/ubuntu/storm/lib/libthrift7-0.7.0.jar:/home/ubuntu/storm/lib/core.incubator-0.1.0.jar:/home/ubuntu/storm/lib/servlet-api-2.5.jar:/home/ubuntu/storm/lib/objenesis-1.2.jar:/home/ubuntu/storm/lib/compojure-0.6.4.jar:/home/ubuntu/storm/lib/clojure-1.4.0.jar:/home/ubuntu/storm/lib/hiccup-0.3.6.jar:/home/ubuntu/storm/lib/ring-servlet-0.3.11.jar:/home/ubuntu/storm/lib/tools.cli-0.2.2.jar:/home/ubuntu/storm/lib/zookeeper-3.3.3.jar:/home/ubuntu/storm/log4j:/home/ubuntu/storm:/home/ubuntu/storm/conf -Xmx768m -Dlogfile.name=ui.log -Dlog4j.configuration=storm.log.properties backtype.storm.ui.core