Merge pull request #141 from DICL/integrating-with-hadoop

Hadoop integration + Logical blocks feature

.gitignore

@@ -2,6 +2,7 @@
 *.slo
 *.lo
 *.o
+*.class
 
 # Compiled Dynamic libraries
 *.so

Makefile.am

@@ -1,11 +1,12 @@
 include tests/Makefile.am
 
+# Global options, users will use CPPFLAGS and CXXFLAGS
 AM_CPPFLAGS      = -I@srcdir@/src/common -I@srcdir@/src -include ./config.h $(BOOST_CPPFLAGS)
 AM_CXXFLAGS      = $(CXXFLAGS) -Wall -Werror
 
+# Binaries that we will produce
 bin_PROGRAMS     = eclipse_node veloxdfs velox_get_config
-dist_bin_SCRIPTS = src/stats/read_io_stats.sh
-bin_SCRIPTS      = src/targets/veloxd
+dist_bin_SCRIPTS = src/stats/read_io_stats.sh src/stats/scheduler.py src/targets/veloxd src/targets/veloxdfs-rm
 
 messages_files =   src/messages/message.cc         \
                    src/messages/boost_impl.cc      \
@@ -26,7 +27,7 @@ messages_files =   src/messages/message.cc         \
 
 
 # libs -----
-lib_LTLIBRARIES     = libvdfs.la
+lib_LTLIBRARIES     = libvdfs.la 
 libvdfs_la_SOURCES  = $(messages_files) \
                       src/client/dfs.cc \
                       src/client/vdfs.cc \
@@ -38,7 +39,7 @@ libvdfs_la_SOURCES  = $(messages_files) \
 libvdfs_la_LDFLAGS  = $(BOOST_LDFLAGS) -version-info 0:0:0
 libvdfs_la_LIBADD   = -lboost_system -lboost_serialization -lboost_coroutine -lboost_thread -lboost_context -ldl
 
-# Binaries ----
+# BOOST_STATIC only for environments where we cannot install all the dependencies in every node
 if BOOST_STATIC
 AM_LDFLAGS          = -static $(BOOST_LDFLAGS) -Wl,--start-group -Wl,-Bstatic,-lboost_system,-lboost_serialization,-lboost_coroutine,-lboost_thread,-lboost_context,-Bdynamic
 LDADD               = libvdfs.la -lrt
@@ -47,6 +48,7 @@ AM_LDFLAGS          = $(BOOST_LDFLAGS)
 LDADD               = libvdfs.la -lboost_system -lboost_serialization -lboost_coroutine -lboost_thread -lboost_context
 endif
 
+# Main binary
 eclipse_node_main    = src/targets/node_main.cc
 eclipse_node_SOURCES = $(eclipse_node_main) \
                        src/network/asyncchannel.cc \
@@ -64,19 +66,7 @@ eclipse_node_SOURCES = $(eclipse_node_main) \
                        src/fileleader/file_leader.cc \
                        src/fileleader/file_leader_router.cc
 
-eclipse_node_LDADD   =  $(LDADD)
-
-if LOGICAL_BLOCKS_FEATURE 
-
-eclipse_node_SOURCES+= src/stats/io_monitor_invoker.cc \
-                       src/stats/zk_listener.cc \
-                       src/stats/scheduler_factory.cc \
-                       src/stats/scheduler_score_based.cc \
-                       src/stats/scheduler_simple.cc
-
-eclipse_node_LDADD  +=  -lzookeeper_mt
-
-endif
+eclipse_node_LDADD   = $(LDADD)
 
 veloxdfs_SOURCES     = src/targets/client.cc \
                        src/client/cli_driver.cc
@@ -99,3 +89,49 @@ dist_doc_DATA       = README.md LICENSE doc/eclipse.json
 
 EXTRA_DIST          = doc .autotools_aux $(shell find @srcdir@/src -name "*.hh" -o -name "*.in") \
                       $(sh_scripts) doc/eclipse.json
+
+if LOGICAL_BLOCKS_FEATURE
+
+# Add eclipse_node code==============================================
+eclipse_node_SOURCES+= src/stats/io_monitor_invoker.cc \
+                       src/stats/zk_listener.cc \
+                       src/stats/scheduler_factory.cc \
+                       src/stats/scheduler_score_based.cc \
+                       src/stats/scheduler_simple.cc \
+                       src/stats/scheduler_python.cc \
+                       src/stats/scheduler_vlmb.cc \
+                       src/stats/scheduler_multiwave.cc
+
+eclipse_node_LDADD  +=  -lzookeeper_mt
+
+# Compile JAVA classes ==============================================
+AM_JAVACFLAGS = -cp $(top_srcdir)/lib/java/json-simple-1.1.jar
+javadir = $(datadir)/java
+dist_java_JAVA = src/java/velox/Configuration.java \
+		             src/java/velox/VeloxDFS.java \
+		             src/java/velox/model/BlockMetadata.java \
+		             src/java/velox/model/Metadata.java
+
+
+# CREATE JAR file      ==============================================
+all-local: veloxdfs.jar
+
+jar_DATA = veloxdfs.jar $(top_srcdir)/lib/java/json-simple-1.1.jar
+
+jardir = $(libdir)/java
+veloxdfs.jar: $(dist_java_JAVA) classjava.stamp
+	javah -jni -d $(top_srcdir)/java/jni com.dicl.velox.VeloxDFS
+	jar cf veloxdfs.jar com
+
+# Create shared lib    ==============================================
+# In lib_LTLIBRARIES we wrote libveloxdfs-jni.la now we change it to underscore.
+# Reason found at: https://lists.gnu.org/archive/html/automake/2003-08/msg00096.html
+lib_LTLIBRARIES    += libveloxdfs-jni.la
+libveloxdfs_jni_la_SOURCES = src/java/jni/velox_VeloxDFS.h \
+								             src/java/jni/com_dicl_velox_VeloxDFS.h \
+								             src/java/jni/VeloxDFS.cc
+
+libveloxdfs_jni_la_LDFLAGS  = $(BOOST_LDFLAGS) -version-info 0:0:0
+libveloxdfs_jni_la_LIBADD   = libvdfs.la
+
+endif

README.md

@@ -3,6 +3,7 @@ VeloxDFS {#mainpage}
 <!-- @cond Remove those links for Doxygen-->
 --- 
 [![Build Status](https://travis-ci.org/DICL/VeloxDFS.svg?branch=master)](https://travis-ci.org/DICL/VeloxDFS)
+[![Release](https://img.shields.io/github/release/DICL/VeloxDFS.svg)](https://img.shields.io/github/release/DICL/VeloxDFS.svg)
 [![Slack room](https://img.shields.io/badge/slack-join-pink.svg)](https://dicl.slack.com/messages/general/)
 ---
 <!-- @endcond -->
@@ -14,21 +15,34 @@ however, it can be used independently of the other components such as _VeloxMR_
 
 Key features of current VeloxDFS implementation includes:
  - No central directory service such as in _HDFS_ NameNode.
- - Zero Hop lookup file-system.
- - Instead for each file there is a file leader node which holds its metadata.
- - Highly secure and stable, total replication of the system's state.
+ - Logical block representation where the size adapts to the current workload.
+ - HADOOP API to be used instead of HDFS.
+
+VELOX ECOSYSTEM
+===============
+
+Related projects
+
+| Project                               | Description                  |URL                          |
+| -- | -- | -- |
+| VeloxMR           |  experimental MapReduce engine based on VeloxDFS | https://github.com/DICL/VeloxMR |
+| eclipsed          | deployment/debugging helper script writen in RUBY| https://github.com/DICL/eclipsed|
+| velox-hadoop      | VeloxDFS JAVA library for Hadoop                 | https://github.com/DICL/velox-hadoop |
+| velox-deploy-ansible | ansible playbook to automatize velox/hadoop installation | https://github.com/DICL/velox-experiments-utils |
+| hadoop-etc        | Hadoop configuration files to use VeloxDFS       | https://github.com/vicentebolea/hadoop-etc |
+| velox-report      | Project to benchmark and log VeloxDFS performance | https://github.com/vicentebolea/velox-report |
 
 USAGE
 =====
-VeloxDFS default [launcher] [eclipsed] is not included in this repository since its a temporary hack and 
-we want to let the user to choose any launcher or service managers such as:
- - systemd/init.d
- - puppet/chef/salt
-
-Once the system is up and running, you can interact with VeloxDFS with the following commands:
+To deploy the system please refer to `velox` command and its following options:
+
+    $ veloxd up|down|restart|status
+
+The command line API has the following options:
 
-    $ dfs put|get|cat|ls|rm|format|show
+    $ veloxdfs put|get|cat|ls|rm|format|show
 
+The C++ API can be found at `vdfs.hh` and `DFS.hh` files, as for the JAVA API, it can be found at `src/java` directory.
 
 COMPILING & INSTALLING
 ======================
@@ -39,7 +53,7 @@ Further information can be found it in: [Installation](https://github.com/DICL/V
 Compiling requirements
 ----------------------
  - C++14 support, this is: GCC >= 4.9, Clang >= 3.4, or ICC >= 16.0.
- - Boost library >= 1.53.
+ - Boost library >= 1.56.
  - Sqlite3 library.
  - GNU Autotools (Autoconf, Automake, Libtool).
  - Unittest++ [optional].

configure.ac

@@ -11,7 +11,7 @@ AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([Makefile ])
 
 AM_CONDITIONAL([DEFAULT_CXXFLAGS], [test -z "$CXXFLAGS"])
-AM_COND_IF([DEFAULT_CXXFLAGS], [CXXFLAGS='-g -O0 -march=native'])
+AM_COND_IF([DEFAULT_CXXFLAGS], [CXXFLAGS='-g3 -O0 -march=native'])
 
 AC_PROG_CXX
 AC_LANG([C++])
@@ -42,13 +42,22 @@ AC_TYPE_UINT8_T
 AC_HEADER_STDC
 AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h sys/socket.h sys/time.h unistd.h])
 AC_SEARCH_LIBS([pthread_create], [pthread])
-AC_SEARCH_LIBS([sqlite3_exec], [sqlite3])
+AC_SEARCH_LIBS([sqlite3_exec], [sqlite3], [], [AC_MSG_ERROR([sqlite3 not found, please install it.])])
 
 AX_BOOST_BASE([1.53.0], ,
               [AC_MSG_ERROR([Needs BOOST])])
 have_unittest=no
 PKG_CHECK_EXISTS([UnitTest++], [have_unittest=yes], [have_unittest=no])
 
+#}}}
+# JNI and javac {{{
+if test "x$enable_lblocks" = xyes; then
+  AX_JNI_INCLUDE_DIR
+  for include in $JNI_INCLUDE_DIRS; do
+    CPPFLAGS="$CPPFLAGS -I$include"
+  done
+fi
+
 #}}}
 # PATHS {{{
 FULLY_EVAL(["$sysconfdir"], [sysconfdirfull])
@@ -63,9 +72,15 @@ AC_SUBST(libdirfull)
 AH_TEMPLATE([ECLIPSE_CONF_PATH],[Configuration files (ETC) for Eclipse])
 AC_DEFINE_UNQUOTED([ECLIPSE_CONF_PATH], ["$sysconfdirfull"])
 
+AH_TEMPLATE([ECLIPSE_HOME_CONF_PATH],[Home configuration files (ETC) for Eclipse])
+AC_DEFINE_UNQUOTED([ECLIPSE_HOME_CONF_PATH], ["$HOME"])
+
 AH_TEMPLATE([IO_STAT_SCRIPT],[IOSTATS path])
 AC_DEFINE_UNQUOTED([IO_STAT_SCRIPT], ["${bindirfull}/read_io_stats.sh"])
 
+AH_TEMPLATE([PY_SCHEDULER_SCRIPT],[PY scheduler path])
+AC_DEFINE_UNQUOTED([PY_SCHEDULER_SCRIPT], ["${bindirfull}/scheduler.py"])
+
 AS_IF([test "x$enable_lblocks" = xyes], [
       AH_TEMPLATE([LOGICAL_BLOCKS_FEATURE],["Enables logical blocks feature" ]) 
       AC_DEFINE_UNQUOTED([LOGICAL_BLOCKS_FEATURE], [1])