Kafka Docker Wrapper

Running

• run npm install
• run docker-compose up

You can now see the producer tab making new messages and the consumer tab reading those messages. You can start more producers (while still running 1 consumer) and see more messages coming in.

Topic Errors

If you are getting an error about the topic not existing, try running docker-compose rm to delete the containers. Then re-run docker-compose up again.

Use

Install $ npm install kafka-node-wrapper --save

constKafka=require('kafka-node-wrapper');// Node < 9.x// import * as Kafka from 'kafka-node-wrapper' // Node > 9.xconstconsumer=newKafka.Consumer();constproducer=newKafka.Producer();// Connect with producer and consumer in parallelfunctionconnect(){returnPromise.all[consumer.connect(),producer.connect()];}functionevents(){// Consumer Eventsconsumer.onMessage().subscribe((data)=>{console.log('Data',data);});consumer.onError().subscribe((err)=>{console.log('Consumer Error',err);});// Producer Eventsproducer.onReport().subscribe((report)=>{console.log('Producer Delivery Report',report);});producer.onError().subscribe((err)=>{console.log('Producer Error',err);});}// DEMO!connect().then(()=>{events();returnproducer.publish(JSON.stringify({foo: 1,bar: 2}));}).catch((err)=>{console.error('Error!',err.message);});

Configuration

All

Configurations custom to this wrapper

Field	Description	Type	Default
throttle	Throttle interval time (ms)	Number	500
topics	Topics to subscribe to	String[]	['kafka-test-topic']
autoInterval	Allow auto intervals for polling (producer) and consuming (consumer).	boolean	true

Consumer

Configurations custom to this wrapper's Consumer class.

Field	Description	Type	Default
consumeMax	Number of messages to consume for each interval.	Number	1

This rest of the configuration is described here.

Examples

Basic Usage

Sample code for sample pub sub.

constKafka=require('./src/index');constconsumer=newKafka.consumer();constproducer=newKafka.producer();/** * @param {Consumer} con * @return {Promise<any>} data */functionconsumerEvent(con){returnnewPromise((resolve,reject)=>{con.message().subscribe((data)=>{resolve(data);console.log('SAMPLE Consumer Data:',data);});con.error().subscribe((err)=>{reject(err);console.log('SAMPLE Consumer Error:',err);});console.log('SAMPLE Consumer Connected');con.log().subscribe((log)=>{console.log('SAMPLE log',log);});});}/** * @param pro * @return {Promise<any>} */functionproducerEvent(pro){returnnewPromise((resolve,reject)=>{pro.report().subscribe((report)=>{console.log('SAMPLE Producer Delivery Report:',report);returnresolve(report);});pro.error().subscribe((err)=>{console.log('SAMPLE Producer Error:',err);returnreject(err);});console.log('SAMPLE Producer Connected');constmessage={foo: 1,bar: 2};pro.publish(JSON.stringify(message));});}functionmain(){Promise.all([consumer.connect(),producer.connect()]).then(()=>{returnPromise.all([consumerEvent(consumer),producerEvent(producer)]);}).then((data)=>{console.log('DATA',data);returnPromise.all([consumer.disconnect(),producer.disconnect()]);}).then(()=>{console.log('Disconnected');}).catch((err)=>{console.error('Error',err);});}main();module.exports={consumer: consumer,producer: producer,consumerEvent: consumerEvent,producerEvent: producerEvent};

Overriding

Sample code for doing your own pulling or commit/consume

constKafkaWrapper=require('./src/index');/* ARBITRARY EXTERNAL CODE BEGIN *//** * This callback type is called `requestCallback` and is displayed as a global symbol. * @callback requestCallback * @param {{name: string, age: number}} responseCode *//** * Some stuff that has a callback (maybe write to a database or whatever) * @param data * @param {requestCallback} cb */functiondoStuffCB(data,cb){setTimeout(()=>{console.log('Did stuff to data');returncb({name: 'Mary',age: 42});},500);}/** * Promise Wrapper for our doStuffCB function * @param {Object} data - some data * @return {Promise<{name: string, age: number}>} */asyncfunctioncbWrapper(data){returnnewPromise((resolve,reject)=>{doStuffCB(data,(user)=>{returnresolve(user);});});}/** * Does stuff * @param {Object} data - some data * @return {Promise<{name: string, age: number}>} */asyncfunctiondoStuff(data){returnnewPromise((resolve,reject)=>{setTimeout(()=>{console.log('Did more stuff to data');returnresolve({name: 'Bob',age: 40});},500);});}/* ARBITRARY EXTERNAL CODE END */classSampleOverride{constructor(){this._interval=null;this.consumer=newKafkaWrapper.consumer({consumeMax: 10,autoInterval: false,topics: ['upload-user']});// we must consume/commit ourselvesthis.producer=newKafkaWrapper.producer({topics: ['upload-user']});}/** * Run Service * @param message * @return {Promise<void>} */asyncrun(message){// Connectawaitthis.consumer.connect();awaitthis.producer.connect();returnawaitthis._scenario();}async_scenario(){returnnewPromise(async(resolve,reject)=>{// Send message to publish somethingawaitthis.producer.publish(JSON.stringify({name: 'Dave',age: 38}));// Start consuming 1 message every 500 msthis._startConsuming(1);// Listen to Messagesthis.consumer.message().subscribe(async(message)=>{this._stopConsuming();// Got a message stop consuming (this will not cancel other consumed messages)awaitthis._process(message);// Process Messagethis._startConsuming(1);// Consume the next message});// Loggingthis.consumer.error().subscribe((err)=>{console.error(err);});// End scenario based on some conditionsetTimeout(async()=>{awaitthis.producer.disconnect();awaitthis.consumer.disconnect();returnresolve();},20000);// 20 seconds});}/** * Process Function * @param message * @return {Promise<DeliveryReport>} */async_process(message){constuser1=awaitcbWrapper(message);// Step 1constuser2=awaitdoStuff(message);// Step 2constoutput={user1: user1,user2: user2};returnawaitthis.producer.publish(JSON.stringify(output));// Step 3 - Send}_startConsuming(numMessages){this._interval=setInterval(()=>{this.consumer.consume(numMessages);},500);}_stopConsuming(){clearInterval(this._interval);}}constoverride=newSampleOverride();override.run({foo: 1,bar: 2}).then(()=>{console.log('Done');process.exit(0);}).catch((err)=>{console.error('Error',err);});