Skip to content

Introduce config params to use latest EH client, control request prefetch size, batch size#2322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
raviperi wants to merge 9 commits into from
Closed

Introduce config params to use latest EH client, control request prefetch size, batch size #2322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e9fc45d
Introduce config params to dictate EH request prefetch size, max number
raviperi Sep 11, 2017
3f8f37f
Remove unncessary file.
raviperi Sep 13, 2017
9a18f9a
Add license header, remove next tuple sleep time as it is redundant.
raviperi Sep 14, 2017
5a7eb2e
Remove unused nextTuple sleep interval field
raviperi Sep 14, 2017
6ba9d11
Add javadoc for EventHubConfig. Remove unused fields. Improve logging…
raviperi Sep 19, 2017
53569ac
Organize usings. Remove extra constructor and fields in EventHubSpout…
raviperi Sep 19, 2017
b1bde9b
Remove obselete Binary and EventDataSchemes, and add unit tests for s…
raviperi Sep 20, 2017
4516d01
Mark all fields final in EventHUbMessage
raviperi Sep 20, 2017
2063f45
Update batch size log message to debug level
raviperi Sep 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 76 additions & 60 deletions external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,12 +17,9 @@
*******************************************************************************/
package org.apache.storm.eventhubs.bolt;

import java.util.Map;
import java.util.concurrent.ExecutionException;

import com.microsoft.azure.eventhubs.EventData;
import com.microsoft.azure.eventhubs.EventHubClient;
import com.microsoft.azure.eventhubs.PartitionSender;
import com.microsoft.azure.servicebus.ServiceBusException;
import org.apache.storm.eventhubs.spout.EventHubException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
Expand All@@ -31,82 +28,102 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;
import java.util.concurrent.ExecutionException;

import java.util.Map;
import java.util.concurrent.ExecutionException;
import com.microsoft.azure.eventhubs.EventData;
import com.microsoft.azure.eventhubs.EventHubClient;
import com.microsoft.azure.eventhubs.PartitionSender;
import com.microsoft.azure.servicebus.ServiceBusException;

/**
* A bolt that writes event message to EventHub.
*
* <p>
* The implementation has two modes of operation:
* <ul>
* <li>partitionmode = true, One bolt for per partition write.</li>
* <li>partitionmode = false, use default partitioning key strategy to write to
* partition(s)</li>
* </ul>
* </p>
*/
public class EventHubBolt extends BaseRichBolt {
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory
.getLogger(EventHubBolt.class);
private static final Logger logger = LoggerFactory.getLogger(EventHubBolt.class);

protected OutputCollector collector;
protected PartitionSender sender;
protected EventHubClient ehClient;
protected PartitionSender sender;
protected EventHubBoltConfig boltConfig;

/**
* Constructs an instance that uses the specified connection string to connect
* to an EventHub and write to the specified entityPath
*
* @param connectionString
* EventHub connection String
* @param entityPath
* entity path to write to
*/
public EventHubBolt(String connectionString, String entityPath) {
boltConfig = new EventHubBoltConfig(connectionString, entityPath);
}

public EventHubBolt(String userName, String password, String namespace,
String entityPath, boolean partitionMode) {
boltConfig = new EventHubBoltConfig(userName, password, namespace,
entityPath, partitionMode);
/**
* Constructs an instance that connects to an EventHub using the specified
* connection credentials.
*
* @param userName
* UserName to connect as
* @param password
* Password to use
* @param namespace
* target namespace for the service bus
* @param entityPath
* Name of the event hub
* @param partitionMode
* number of partitions
*/
public EventHubBolt(String userName, String password, String namespace, String entityPath, boolean partitionMode) {
boltConfig = new EventHubBoltConfig(userName, password, namespace, entityPath, partitionMode);
}

/**
* Constructs an instance using the specified configuration
*
* @param config
* EventHub connection and partition configuration
*/
public EventHubBolt(EventHubBoltConfig config) {
boltConfig = config;
}

@Override
public void prepare(Map config, TopologyContext context,
OutputCollector collector) {
public void prepare(Map config, TopologyContext context, OutputCollector collector) {
this.collector = collector;
String myPartitionId = null;
if (boltConfig.getPartitionMode()) {
// We can use the task index (starting from 0) as the partition ID
myPartitionId = "" + context.getThisTaskIndex();
}
logger.info("creating sender: " + boltConfig.getConnectionString()
+ ", " + boltConfig.getEntityPath() + ", " + myPartitionId);
logger.info(String.format("Conn String: %s, PartitionMode: %s", boltConfig.getConnectionString(),
String.valueOf(boltConfig.getPartitionMode())));
try {
ehClient = EventHubClient.createFromConnectionStringSync(boltConfig.getConnectionString());
if (boltConfig.getPartitionMode()) {
sender = ehClient.createPartitionSenderSync(Integer.toString(context.getThisTaskIndex()));
String partitionId = String.valueOf(context.getThisTaskIndex());
logger.info("Writing to partition id: " + partitionId);
sender = ehClient.createPartitionSenderSync(partitionId);
}
} catch (Exception ex) {
collector.reportError(ex);
throw new RuntimeException(ex);
}

}

@Override
public void execute(Tuple tuple) {
try {
EventData sendEvent = new EventData(boltConfig.getEventDataFormat().serialize(tuple));
if (boltConfig.getPartitionMode() && sender!=null) {
sender.sendSync(sendEvent);
}
else if (boltConfig.getPartitionMode() && sender==null) {
throw new EventHubException("Sender is null");
}
else if (!boltConfig.getPartitionMode() && ehClient!=null) {
if (sender == null) {
ehClient.sendSync(sendEvent);
}
else if (!boltConfig.getPartitionMode() && ehClient==null) {
throw new EventHubException("ehclient is null");
} else {
sender.sendSync(sendEvent);
}
collector.ack(tuple);
} catch (EventHubException ex ) {
collector.reportError(ex);
collector.fail(tuple);
} catch (ServiceBusException e) {
collector.reportError(e);
collector.fail(tuple);
Expand All@@ -115,32 +132,31 @@ else if (!boltConfig.getPartitionMode() && ehClient==null) {

@Override
public void cleanup() {
if(sender != null) {
try {
sender.close().whenComplete((voidargs,error)->{
try{
if(error!=null){
logger.error("Exception during sender cleanup phase"+error.toString());
}
ehClient.closeSync();
}catch (Exception e){
logger.error("Exception during ehclient cleanup phase"+e.toString());
logger.debug("EventHubBolt cleanup");
try {
ehClient.close().whenComplete((voidargs, error) -> {
try {
if (error != null) {
logger.error("Exception during EventHubBolt cleanup phase" + error.toString());
}
}).get();
} catch (InterruptedException e) {
logger.error("Exception occured during cleanup phase"+e.toString());
} catch (ExecutionException e) {
logger.error("Exception occured during cleanup phase"+e.toString());
}
logger.info("Eventhub Bolt cleaned up");
if (sender != null) {
sender.closeSync();
}
} catch (Exception e) {
logger.error("Exception during EventHubBolt cleanup phase" + e.toString());
}
}).get();

} catch (InterruptedException | ExecutionException e) {
logger.error("Exception occured during cleanup phase" + e.toString());
} finally {
sender = null;
ehClient = null;
ehClient = null;
}
}

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {

}

}
Loading