apache · kosii · Sep 23, 2016 · Sep 23, 2016 · Sep 23, 2016 · Sep 23, 2016
diff --git a/.gitignore b/.gitignore
@@ -2,9 +2,7 @@
 /classes
 /lib
 deploy/lib
-deploy/logs
 .emacs-project
-*.jar
 bin/jzmq
 .DS_Store
 deploy/classes
@@ -21,20 +19,41 @@ _release
 *.zip
 *.tar.gz
 .lein-deps-sum
-*.iml
 target
 /.project/
 /.lein-plugins/
-*.ipr
-*.iws
-.idea
 #.*
 !/.travis.yml
 !/.gitignore
 _site
 dependency-reduced-pom.xml
-derby.log
 metastore_db
+build
+/docs/javadocs
+*.class
+
+# logs
+logs
+*.log
+
+# Eclipse
 .settings/
 .project
 .classpath
+
+# Intellij
+*.iml
+*.ipr
+*.iws
+.idea
+
+# Package Files
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+# ignore vagrant files
+/integration-test/config/.vagrant/
diff --git a/.travis.yml b/.travis.yml
@@ -9,22 +9,26 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+addons:
+  hosts:
+    - node1
 
 env:
   - MODULES=storm-core
   - MODULES='!storm-core'
+  - MODULES='INTEGRATION-TEST'
 
 language: java
 jdk:
-  - oraclejdk7
   - oraclejdk8
 before_install:
   - rvm use 2.1.5 --install
   - nvm install 0.12.2
   - nvm use 0.12.2
 install: /bin/bash ./dev-tools/travis/travis-install.sh `pwd`
-script: /bin/bash ./dev-tools/travis/travis-script.sh `pwd` $MODULES
-sudo: false
+script:
+  - /bin/bash ./dev-tools/travis/travis-script.sh `pwd` $MODULES
+sudo: true
 cache:
   directories:
     - "$HOME/.m2/repository"

diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -21,6 +21,7 @@ Table of Contents
     * <a href="#packaging">Create a Storm distribution (packaging)</a>
 * <a href="#best-practices">Best practices</a>
     * <a href="#best-practices-testing">Testing</a>
+    * <a href="#best-practices-versions">Version Changes</a>
 * <a href="#tools">Tools</a>
     * <a href="#code-repositories">Source code repositories (git)</a>
     * <a href="#issue-tracking">Issue tracking (JIRA)</a>
@@ -179,7 +180,7 @@ your fork up to date with the latest changes of the upstream (official) `storm`
 
 ### Approve a pull request
 
-[BYLAWS](http://storm.apache.org/documentation/BYLAWS.html) describes the condition of approval for code / non-code change. 
+[BYLAWS](http://storm.apache.org/contribute/BYLAWS.html) describes the condition of approval for code / non-code change. 
 
 Please refer Approvals -> Actions section for more details.
 
@@ -366,6 +367,19 @@ Storm using tracked topologies.  Here, the key is the `tracked-wait` function: i
 tuples have been emitted by the spouts _and_ the topology is idle (i.e. no tuples have been emitted nor will be emitted
 without further input).  Note that you should not use tracked topologies for topologies that have tick tuples.
 
+<a name="best-practices-versions"></a>
+
+## Version Changes
+
+An easy way to change versions across all pom files, for example from `1.0.0-SNAPSHOT` to `1.0.0`, is with the maven
+versions plugin.
+
+```
+mvn versions:set #This prompts for a new version
+mvn versions:commit
+```
+
+[Plugin Documentation] (http://www.mojohaus.org/versions-maven-plugin/)
 
 <a name="tools"></a>
 

diff --git a/README.markdown b/README.markdown
@@ -1,4 +1,6 @@
-Master Branch: [![Travis CI](https://travis-ci.org/apache/storm.svg?branch=master)](https://travis-ci.org/apache/storm)
+Master Branch:  
+[![Travis CI](https://travis-ci.org/apache/storm.svg?branch=master)](https://travis-ci.org/apache/storm)
+[![Maven Version](https://maven-badges.herokuapp.com/maven-central/org.apache.storm/storm-core/badge.svg)](http://search.maven.org/#search|gav|1|g:"org.apache.storm"%20AND%20a:"storm-core")
 
 Storm is a distributed realtime computation system. Similar to how Hadoop provides a set of general primitives for doing batch processing, Storm provides a set of general primitives for doing realtime computation. Storm is simple, can be used with any programming language, [is used by many companies](http://storm.apache.org/documentation/Powered-By.html), and is a lot of fun to use!
 
@@ -91,6 +93,11 @@ under the License.
 * Boyang Jerry Peng ([@jerrypeng](https://github.com/jerrypeng))
 * Zhuo Liu ([@zhuoliu](https://github.com/zhuoliu))
 * Haohui Mai ([@haohui](https://github.com/haohui))
+* Sanket Chintapalli ([@redsanket](https://github.com/redsanket))
+* Longda Feng ([@longda](https://github.com/longdafeng))
+* John Fang ([@hustfxj](https://github.com/hustfxj))
+* Abhishek Agarwal ([@abhishekagarwal87](https://github.com/abhishekagarwal87))
+* Satish Duggana ([@satishd](https://github.com/satishd))
 
 ## Contributors
 
@@ -228,7 +235,6 @@ under the License.
 * Renkai Ge ([@Renkai](https://github.com/Renkai))
 * Aaron Coburn ([@acoburn](https://github.com/acoburn))
 * Rick Kellogg ([@rmkellogg](https://github.com/rmkellogg))
-* Abhishek Agarwal ([@abhishekagarwal87](https://github.com/abhishekagarwal87))
 * chenyuzhao ([@danny0405](https://github.com/danny0405))
 * Michael Schonfeld ([@schonfeld](https://github.com/schonfeld))
 * Erik Weathers ([@erikdw](https://github.com/erikdw))
@@ -249,10 +255,25 @@ under the License.
 * Satish Duggana ([@satishd](https://github.com/satishd))
 * Seth Ammons ([@sethgrid](https://github.com/sethgrid))
 * Aaron Dixon ([@atdixon](https://github.com/atdixon))
-* Roshan Naik([@roshannaik](https://github.com/roshannaik))
+* Roshan Naik ([@roshannaik](https://github.com/roshannaik))
+* Dan Bahir ([#dbahir](https://github.com/dbahir))
+* Alessandro Bellina ([#abellina](https://github.com/abellina))
+* Basti Liu ([@basti](https://github.com/bastiliu))
+* Jark Wu ([@jark](https://github.com/wuchong))
+* Cody Wang ([@unsleepy22](https://github.com/unsleepy22))
+* Karthick Duraisamy Soundararaj ([@dsKarthick](https://github.com/dsKarthick))
+* Olivier Mallassi ([@omallassi](https://github.com/omallassi))
+* Balázs Kossovics ([@kosii](https://github.com/kosii))
+* csivaguru ([@csivaguru](https://github.com/csivaguru))
+* sakanaou ([@sakanaou](https://github.com/sakanaou))
+* wangperry ([@wangperry](https://github.com/wangperry))
+* Slava Andreyev ([@slava92](https://github.com/slava92))
+* Pavel Grochal ([@Darkless012](https://github.com/Darkless012))
+* Manikumar Reddy O ([@omkreddy](https://github.com/omkreddy))
+* Dohyun Lee ([@leedohyun](https://github.com/leedohyun))
+* Arthur Maciejewicz ([@Zintinio](https://github.com/Zintinio]))
+* Juan Paulo Gutierrez ([@juanpaulo](https://github.com/juanpaulo)) 
 
 ## Acknowledgements
 
 YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).
-
-
diff --git a/SECURITY.md b/SECURITY.md
@@ -133,7 +133,31 @@ To set up 2-way authentication:
 1. `drpc.https.want.client.auth` (If this set to true, server requests for client certificate authentication, but keeps the connection even if no authentication is provided)
 2. `drpc.https.need.client.auth` (If this set to true, server requires the client to provide authentication)
 
+#### GENERATE CERTIFICATES FOR LOCAL TESTING SSL SETUP
 
+Run the following script and fill in the values and passwords when prompted. The `keyalg` must be set to `RSA`
+
+```bash
+#!/bin/bash
+
+DIR=/Users/user/certs/dir/
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -validity 365 -keyalg RSA -genkey
+
+openssl req -new -x509 -keyout $DIR/ca-key -out $DIR/ca-cert -days 365
+
+keytool -keystore $DIR/server.truststore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/client.truststore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -certreq -file $DIR/cert-file
+
+openssl x509 -req -CA $DIR/ca-cert -CAkey $DIR/ca-key -in $DIR/cert-file -out $DIR/cert-signed -days 365 -CAcreateserial -passin pass:test12
+
+keytool -keystore $DIR/server.keystore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -import -file $DIR/cert-signed
+```
 
 ## Authentication (Kerberos)
 
@@ -469,7 +493,6 @@ nimbus.groups:
    - "storm"
 ```
 
-
 ### DRPC
 Hopefully more on this soon
 

diff --git a/bin/flight.bash b/bin/flight.bash
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-JDKPATH="/home/y/share/yjava_jdk/java"
+JDKPATH=$JAVA_HOME
 BINPATH="/usr/bin"
 USER=`whoami`
 
@@ -59,7 +59,7 @@ function dump_record {
 
 function jstack_record {
     FILENAME=jstack-$1-${NOW}.txt
-    $BINPATH/jstack $1 > "$2/${FILENAME}"
+    $BINPATH/jstack $1 > "$2/${FILENAME}" 2>&1
 }
 
 function jmap_record {

diff --git a/bin/storm b/bin/storm
@@ -30,16 +30,8 @@ while [ -h "${PRG}" ]; do
   fi
 done
 
-# find python >= 2.6
-if [ -a /usr/bin/python2.6 ]; then
-  PYTHON=/usr/bin/python2.6
-fi
-
-if [ -z "$PYTHON" ]; then
-  PYTHON=/usr/bin/python
-fi
-
 # check for version
+PYTHON="/usr/bin/env python"
 majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1`
 minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2`
 numversion=$(( 10 * $majversion + $minversion))
@@ -51,12 +43,15 @@ fi
 STORM_BIN_DIR=`dirname ${PRG}`
 export STORM_BASE_DIR=`cd ${STORM_BIN_DIR}/..;pwd`
 
-#check to see if the conf dir is given as an optional argument
+#check to see if the conf dir or file is given as an optional argument
 if [ $# -gt 1 ]; then
   if [ "--config" = "$1" ]; then
     conf_file=$2
+    if [ -d "$conf_file" ]; then
+      conf_file=$conf_file/storm.yaml
+    fi
     if [ ! -f "$conf_file" ]; then
-      echo "Error: Cannot find configuration directory: $conf_file"
+      echo "Error: Cannot find configuration file: $conf_file"
       exit 1
     fi
     STORM_CONF_FILE=$conf_file
@@ -71,4 +66,4 @@ if [ -f "${STORM_CONF_DIR}/storm-env.sh" ]; then
   . "${STORM_CONF_DIR}/storm-env.sh"
 fi
 
-exec "$PYTHON" "${STORM_BIN_DIR}/storm.py" "$@"
+exec "${STORM_BIN_DIR}/storm.py" "$@"
diff --git a/bin/storm-config.cmd b/bin/storm-config.cmd
@@ -86,7 +86,11 @@ if not defined STORM_LOG_DIR (
 @rem retrieve storm.log4j2.conf.dir from conf file
 @rem
 
-"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.config_value storm.log4j2.conf.dir > %CMD_TEMP_FILE%
+if not defined CMD_TEMP_FILE (
+  set CMD_TEMP_FILE=tmpfile
+)
+
+"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.ConfigValue storm.log4j2.conf.dir > %CMD_TEMP_FILE%
 
 FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
 	FOR /F "tokens=1,* delims= " %%a in ("%%i") do (
@@ -113,7 +117,7 @@ if not defined STORM_LOG4J2_CONFIGURATION_FILE (
   set STORM_LOG4J2_CONFIGURATION_FILE="file:///%STORM_HOME%\log4j2\cluster.xml"
 )
 
-"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.config_value java.library.path > %CMD_TEMP_FILE%
+"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.ConfigValue java.library.path > %CMD_TEMP_FILE%
 
 FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
     FOR /F "tokens=1,* delims= " %%a in ("%%i") do (
@@ -125,7 +129,11 @@ FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
 
 
 :storm_opts
- set STORM_OPTS=-Dstorm.options= -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
+ if "%set_storm_options%"=="true" (
+  set STORM_OPTS=-Dstorm.options=
+ )
+
+ set STORM_OPTS=%STORM_OPTS% -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
  set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOG4J2_CONFIGURATION_FILE%
  set STORM_OPTS=%STORM_OPTS% -Dstorm.log.dir=%STORM_LOG_DIR%
  del /F %CMD_TEMP_FILE%

diff --git a/bin/storm-kafka-monitor b/bin/storm-kafka-monitor
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+  ls=`ls -ld "${PRG}"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "${PRG}"`/"$link"
+  fi
+done
+
+STORM_BIN_DIR=`dirname ${PRG}`
+export STORM_BASE_DIR=`cd ${STORM_BIN_DIR}/..;pwd`
+
+# Which java to use
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+exec "$JAVA" -cp "$STORM_BASE_DIR/toollib/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"