From 76913e23179228481c98fbba36a54ca32fe20aed Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sat, 22 Nov 2014 21:15:30 -0600 Subject: [PATCH 01/42] Batch oriented kafka rdd, WIP. todo: cluster metadata / finding leader --- external/kafka/pom.xml | 2 +- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 126 ++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index b3f44471cd326..bf86a6391c8c3 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -44,7 +44,7 @@ org.apache.kafka kafka_${scala.binary.version} - 0.8.0 + 0.8.1.1 com.sun.jmx diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala new file mode 100644 index 0000000000000..b76151a93a1f9 --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.rdd.kafka + +import scala.reflect.{classTag, ClassTag} + +import org.apache.spark.{Logging, Partition, SparkContext, TaskContext} +import org.apache.spark.rdd.RDD +import org.apache.spark.util.NextIterator + +import java.util.Properties +import kafka.api.FetchRequestBuilder +import kafka.common.{ErrorMapping, TopicAndPartition} +import kafka.consumer.{ConsumerConfig, SimpleConsumer} +import kafka.message.{MessageAndMetadata, MessageAndOffset} +import kafka.serializer.Decoder +import kafka.utils.VerifiableProperties + +private[spark] case class KafkaRDDPartition( + override val index: Int, + topic: String, + partition: Int, + fromOffset: Long, + untilOffset: Long +) extends Partition + +/** A batch-oriented interface to Kafka. + * Each given Kafka topic/partition corresponds to an RDD partition. + * Starting and ending offsets are specified in advance, so that you can control exactly-once semantics. + * For an easy interface to Kafka-managed offsets, see {@link org.apache.spark.rdd.kafka.KafkaCluster} + * @param kafkaParams Kafka configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch + * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch + * @param messageHandler function for translating each message into the desired type + */ +class KafkaRDD[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag]( + sc: SparkContext, + kafkaParams: Map[String, String], + fromOffsets: Map[TopicAndPartition, Long], + untilOffsets: Map[TopicAndPartition, Long], + messageHandler: MessageAndMetadata[K, V] => R + ) extends RDD[R](sc, Nil) with Logging { + + assert(fromOffsets.keys == untilOffsets.keys, + "Must provide both from and until offsets for each topic/partition") + + override def getPartitions: Array[Partition] = fromOffsets.zipWithIndex.map { kvi => + val ((tp, from), index) = kvi + new KafkaRDDPartition(index, tp.topic, tp.partition, from, untilOffsets(tp)) + }.toArray + + override def compute(thePart: Partition, context: TaskContext) = new NextIterator[R] { + context.addTaskCompletionListener{ context => closeIfNeeded() } + + val part = thePart.asInstanceOf[KafkaRDDPartition] + val props = new Properties() + kafkaParams.foreach(param => props.put(param._1, param._2)) + val fetchSize = Option(props.getProperty("fetch.message.max.bytes")).map(_.toInt).getOrElse(1024*1024) + val leaderBackoff = Option(props.getProperty("refresh.leader.backoff.ms")).map(_.toLong).getOrElse(200L) + val consumerConfig = new ConsumerConfig(props) + val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(consumerConfig.props) + .asInstanceOf[Decoder[K]] + val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(consumerConfig.props) + .asInstanceOf[Decoder[V]] + val consumer: SimpleConsumer = ??? + var requestOffset = part.fromOffset + var iter: Iterator[MessageAndOffset] = null + + override def getNext: R = { + if (iter == null || !iter.hasNext) { + val req = new FetchRequestBuilder(). + addFetch(part.topic, part.partition, requestOffset, fetchSize). + build() + val resp = consumer.fetch(req) + if (resp.hasError) { + val err = resp.errorCode(part.topic, part.partition) + if (err == ErrorMapping.LeaderNotAvailableCode || + err == ErrorMapping.NotLeaderForPartitionCode) { + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${leaderBackoff}ms") + Thread.sleep(leaderBackoff) + } + // Let normal rdd retry sort out reconnect attempts + throw ErrorMapping.exceptionFor(err) + } + iter = resp.messageSet(part.topic, part.partition) + .iterator + .filter(_.offset < requestOffset) + } + val item = iter.next + if (item.offset >= part.untilOffset) { + finished = true + null.asInstanceOf[R] + } else { + requestOffset = item.nextOffset + messageHandler(new MessageAndMetadata(part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) + } + } + + override def close() = consumer.close() + } + +} From 1d706257ac848d37caeaff0409bf60b080d66e48 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sun, 23 Nov 2014 00:10:56 -0600 Subject: [PATCH 02/42] WIP on kafka cluster --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 89 +++++++++++++++++++ .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 21 ++--- 2 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala new file mode 100644 index 0000000000000..26a897f1dc236 --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.rdd.kafka + +import scala.util.control.NonFatal +import java.util.Properties +import kafka.api.{TopicMetadataRequest, TopicMetadataResponse} +import kafka.consumer.{ConsumerConfig, SimpleConsumer} + +/** + * Convenience methods for interacting with a Kafka cluster. + * @param kafkaParams Kafka configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form + */ +class KafkaCluster(val kafkaParams: Map[String, String]) { + val brokers: Array[(String, Int)] = + kafkaParams.get("metadata.broker.list") + .orElse(kafkaParams.get("bootstrap.servers")) + .getOrElse(throw new Exception("Must specify metadata.broker.list or bootstrap.servers")) + .split(",").map { hp => + val hpa = hp.split(":") + (hpa(0), hpa(1).toInt) + } + + val config: ConsumerConfig = KafkaCluster.consumerConfig(kafkaParams) + + def connect(host: String, port: Int): SimpleConsumer = + new SimpleConsumer(host, port, config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId) + + def connect(hostAndPort: (String, Int)): SimpleConsumer = + connect(hostAndPort._1, hostAndPort._2) + + def connectLeader(topic: String, partition: Int): Option[SimpleConsumer] = + findLeader(topic, partition).map(connect) + + def findLeader(topic: String, partition: Int): Option[(String, Int)] = { + brokers.foreach { hp => + var consumer: SimpleConsumer = null + try { + consumer = connect(hp) + val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) + val resp: TopicMetadataResponse = consumer.send(req) + resp.topicsMetadata.find(_.topic == topic).flatMap { t => + t.partitionsMetadata.find(_.partitionId == partition) + }.foreach { partitionMeta => + partitionMeta.leader.foreach { leader => + return Some((leader.host, leader.port)) + } + } + } catch { + case NonFatal(e) => + } finally { + if (consumer != null) consumer.close() + } + } + None + } +} + +object KafkaCluster { + /** Make a consumer config without requiring group.id or zookeeper.connect, + * since communicating with brokers also needs common settings such as timeout + */ + def consumerConfig(kafkaParams: Map[String, String]): ConsumerConfig = { + val props = new Properties() + kafkaParams.foreach(param => props.put(param._1, param._2)) + Seq("zookeeper.connect", "group.id").foreach { s => + if (!props.contains(s)) + props.setProperty(s, "") + } + new ConsumerConfig(props) + } +} diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index b76151a93a1f9..121364785515f 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -39,7 +39,7 @@ private[spark] case class KafkaRDDPartition( untilOffset: Long ) extends Partition -/** A batch-oriented interface to Kafka. +/** A batch-oriented interface for consuming from Kafka. * Each given Kafka topic/partition corresponds to an RDD partition. * Starting and ending offsets are specified in advance, so that you can control exactly-once semantics. * For an easy interface to Kafka-managed offsets, see {@link org.apache.spark.rdd.kafka.KafkaCluster} @@ -74,34 +74,31 @@ class KafkaRDD[ override def compute(thePart: Partition, context: TaskContext) = new NextIterator[R] { context.addTaskCompletionListener{ context => closeIfNeeded() } + val kc = new KafkaCluster(kafkaParams) val part = thePart.asInstanceOf[KafkaRDDPartition] - val props = new Properties() - kafkaParams.foreach(param => props.put(param._1, param._2)) - val fetchSize = Option(props.getProperty("fetch.message.max.bytes")).map(_.toInt).getOrElse(1024*1024) - val leaderBackoff = Option(props.getProperty("refresh.leader.backoff.ms")).map(_.toLong).getOrElse(200L) - val consumerConfig = new ConsumerConfig(props) val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(consumerConfig.props) + .newInstance(kc.config.props) .asInstanceOf[Decoder[K]] val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(consumerConfig.props) + .newInstance(kc.config.props) .asInstanceOf[Decoder[V]] - val consumer: SimpleConsumer = ??? + val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition) + .getOrElse(throw new Exception(s"Couldn't connect to leader for topic ${part.topic} ${part.partition}")) var requestOffset = part.fromOffset var iter: Iterator[MessageAndOffset] = null override def getNext: R = { if (iter == null || !iter.hasNext) { val req = new FetchRequestBuilder(). - addFetch(part.topic, part.partition, requestOffset, fetchSize). + addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). build() val resp = consumer.fetch(req) if (resp.hasError) { val err = resp.errorCode(part.topic, part.partition) if (err == ErrorMapping.LeaderNotAvailableCode || err == ErrorMapping.NotLeaderForPartitionCode) { - log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${leaderBackoff}ms") - Thread.sleep(leaderBackoff) + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${kc.config.refreshLeaderBackoffMs}ms") + Thread.sleep(kc.config.refreshLeaderBackoffMs) } // Let normal rdd retry sort out reconnect attempts throw ErrorMapping.exceptionFor(err) From 0b94b3363cbc97c5d99e78c42f9be1c08a974fb1 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 24 Nov 2014 08:49:24 -0600 Subject: [PATCH 03/42] use dropWhile rather than filter to trim beginning of fetch response --- .../src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 121364785515f..639165279a537 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -105,7 +105,7 @@ class KafkaRDD[ } iter = resp.messageSet(part.topic, part.partition) .iterator - .filter(_.offset < requestOffset) + .dropWhile(_.offset < requestOffset) } val item = iter.next if (item.offset >= part.untilOffset) { From 4dafd1b0d58215cb27218e569cb5bea9d5146815 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 24 Nov 2014 10:45:40 -0600 Subject: [PATCH 04/42] method to get leader offsets, switch rdd bound to being exclusive start, inclusive end to match offsets typically returned from cluster --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 53 ++++++++++++++++++- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 27 +++++----- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 26a897f1dc236..18d6f30045243 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -19,7 +19,8 @@ package org.apache.spark.rdd.kafka import scala.util.control.NonFatal import java.util.Properties -import kafka.api.{TopicMetadataRequest, TopicMetadataResponse} +import kafka.api.{OffsetRequest, OffsetResponse, PartitionOffsetRequestInfo, TopicMetadataRequest, TopicMetadataResponse} +import kafka.common.{ErrorMapping, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} /** @@ -50,11 +51,11 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { findLeader(topic, partition).map(connect) def findLeader(topic: String, partition: Int): Option[(String, Int)] = { + val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) brokers.foreach { hp => var consumer: SimpleConsumer = null try { consumer = connect(hp) - val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) val resp: TopicMetadataResponse = consumer.send(req) resp.topicsMetadata.find(_.topic == topic).flatMap { t => t.partitionsMetadata.find(_.partitionId == partition) @@ -71,6 +72,54 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } None } + + def getLatestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = + getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) + + def getEarliestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = + getLeaderOffsets(topicAndPartitions, OffsetRequest.EarliestTime) + + def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long): Map[TopicAndPartition, Long] = + getLeaderOffsets(topicAndPartitions, before, 1).map { kv => + // mapValues isnt serializable, see SI-7005 + kv._1 -> kv._2.head + } + + def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long, maxNumOffsets: Int): Map[TopicAndPartition, Seq[Long]] = { + var result = Map[TopicAndPartition, Seq[Long]]() + val req = OffsetRequest( + topicAndPartitions.map(tp => tp -> PartitionOffsetRequestInfo(before, 1)).toMap + ) + brokers.foreach { hp => + var consumer: SimpleConsumer = null + try { + consumer = connect(hp) + val resp: OffsetResponse = consumer.getOffsetsBefore(req) + val respParts = resp.partitionErrorAndOffsets + val needed = topicAndPartitions.diff(result.keys.toSet) + needed.foreach { tp => + respParts.get(tp).foreach { errAndOffsets => + if (errAndOffsets.error == ErrorMapping.NoError) { + result += tp -> errAndOffsets.offsets + } + } + } + if (result.keys.size == topicAndPartitions.size) { + return result + } + } catch { + case NonFatal(e) => + } finally { + if (consumer != null) consumer.close() + } + } + val missing = topicAndPartitions.diff(result.keys.toSet) + throw new Exception(s"Couldn't find offsets for ${missing}") + } + + def getConsumerOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = ??? + + def setConsumerOffsets(offsets: Map[TopicAndPartition, Long]): Unit = ??? } object KafkaCluster { diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 639165279a537..32e6d297133e0 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -35,8 +35,8 @@ private[spark] case class KafkaRDDPartition( override val index: Int, topic: String, partition: Int, - fromOffset: Long, - untilOffset: Long + afterOffset: Long, + throughOffset: Long ) extends Partition /** A batch-oriented interface for consuming from Kafka. @@ -46,8 +46,8 @@ private[spark] case class KafkaRDDPartition( * @param kafkaParams Kafka configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch - * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch + * @param afterOffsets per-topic/partition Kafka offsets defining the (exclusive) starting point of the batch + * @param throughOffsets per-topic/partition Kafka offsets defining the (inclusive) ending point of the batch * @param messageHandler function for translating each message into the desired type */ class KafkaRDD[ @@ -58,17 +58,17 @@ class KafkaRDD[ R: ClassTag]( sc: SparkContext, kafkaParams: Map[String, String], - fromOffsets: Map[TopicAndPartition, Long], - untilOffsets: Map[TopicAndPartition, Long], + afterOffsets: Map[TopicAndPartition, Long], + throughOffsets: Map[TopicAndPartition, Long], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging { - assert(fromOffsets.keys == untilOffsets.keys, + assert(afterOffsets.keys == throughOffsets.keys, "Must provide both from and until offsets for each topic/partition") - override def getPartitions: Array[Partition] = fromOffsets.zipWithIndex.map { kvi => + override def getPartitions: Array[Partition] = afterOffsets.zipWithIndex.map { kvi => val ((tp, from), index) = kvi - new KafkaRDDPartition(index, tp.topic, tp.partition, from, untilOffsets(tp)) + new KafkaRDDPartition(index, tp.topic, tp.partition, from, throughOffsets(tp)) }.toArray override def compute(thePart: Partition, context: TaskContext) = new NextIterator[R] { @@ -84,7 +84,7 @@ class KafkaRDD[ .asInstanceOf[Decoder[V]] val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition) .getOrElse(throw new Exception(s"Couldn't connect to leader for topic ${part.topic} ${part.partition}")) - var requestOffset = part.fromOffset + var requestOffset = part.afterOffset + 1 var iter: Iterator[MessageAndOffset] = null override def getNext: R = { @@ -107,11 +107,14 @@ class KafkaRDD[ .iterator .dropWhile(_.offset < requestOffset) } - val item = iter.next - if (item.offset >= part.untilOffset) { + if (!iter.hasNext) { finished = true null.asInstanceOf[R] } else { + val item = iter.next + if (item.offset > part.throughOffset) { + finished = true + } requestOffset = item.nextOffset messageHandler(new MessageAndMetadata(part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } From ce91c591569b8ac4e91dd29d013961fe0ee5c316 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 24 Nov 2014 12:07:24 -0600 Subject: [PATCH 05/42] method to get consumer offsets, explicit error handling --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 129 +++++++++++------- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 6 +- 2 files changed, 87 insertions(+), 48 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 18d6f30045243..721d90264a1db 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -18,8 +18,9 @@ package org.apache.spark.rdd.kafka import scala.util.control.NonFatal +import scala.collection.mutable.ArrayBuffer import java.util.Properties -import kafka.api.{OffsetRequest, OffsetResponse, PartitionOffsetRequestInfo, TopicMetadataRequest, TopicMetadataResponse} +import kafka.api.{OffsetRequest, OffsetResponse, OffsetFetchRequest, OffsetFetchResponse, PartitionOffsetRequestInfo, TopicMetadataRequest, TopicMetadataResponse} import kafka.common.{ErrorMapping, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} @@ -30,6 +31,8 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ class KafkaCluster(val kafkaParams: Map[String, String]) { + type Err = ArrayBuffer[Throwable] + val brokers: Array[(String, Int)] = kafkaParams.get("metadata.broker.list") .orElse(kafkaParams.get("bootstrap.servers")) @@ -47,79 +50,113 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { def connect(hostAndPort: (String, Int)): SimpleConsumer = connect(hostAndPort._1, hostAndPort._2) - def connectLeader(topic: String, partition: Int): Option[SimpleConsumer] = - findLeader(topic, partition).map(connect) + def connectLeader(topic: String, partition: Int): Either[Err, SimpleConsumer] = + findLeader(topic, partition).right.map(connect) - def findLeader(topic: String, partition: Int): Option[(String, Int)] = { + def findLeader(topic: String, partition: Int): Either[Err, (String, Int)] = { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) - brokers.foreach { hp => - var consumer: SimpleConsumer = null - try { - consumer = connect(hp) - val resp: TopicMetadataResponse = consumer.send(req) - resp.topicsMetadata.find(_.topic == topic).flatMap { t => - t.partitionsMetadata.find(_.partitionId == partition) - }.foreach { partitionMeta => - partitionMeta.leader.foreach { leader => - return Some((leader.host, leader.port)) - } + val errs = new Err + withBrokers(errs) { consumer => + val resp: TopicMetadataResponse = consumer.send(req) + resp.topicsMetadata.find(_.topic == topic).flatMap { t => + t.partitionsMetadata.find(_.partitionId == partition) + }.foreach { partitionMeta => + partitionMeta.leader.foreach { leader => + return Right((leader.host, leader.port)) } - } catch { - case NonFatal(e) => - } finally { - if (consumer != null) consumer.close() } } - None + Left(errs) } - def getLatestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = + def getLatestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) - def getEarliestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = + def getEarliestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.EarliestTime) - def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long): Map[TopicAndPartition, Long] = - getLeaderOffsets(topicAndPartitions, before, 1).map { kv => - // mapValues isnt serializable, see SI-7005 - kv._1 -> kv._2.head + def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long): Either[Err, Map[TopicAndPartition, Long]] = + getLeaderOffsets(topicAndPartitions, before, 1).right.map { r => + r.map { kv => + // mapValues isnt serializable, see SI-7005 + kv._1 -> kv._2.head + } } - def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long, maxNumOffsets: Int): Map[TopicAndPartition, Seq[Long]] = { + def getLeaderOffsets( + topicAndPartitions: Set[TopicAndPartition], + before: Long, + maxNumOffsets: Int + ): Either[Err, Map[TopicAndPartition, Seq[Long]]] = { var result = Map[TopicAndPartition, Seq[Long]]() val req = OffsetRequest( topicAndPartitions.map(tp => tp -> PartitionOffsetRequestInfo(before, 1)).toMap ) + val errs = new Err + withBrokers(errs) { consumer => + val resp: OffsetResponse = consumer.getOffsetsBefore(req) + val respMap = resp.partitionErrorAndOffsets + val needed = topicAndPartitions.diff(result.keys.toSet) + needed.foreach { tp => + respMap.get(tp).foreach { errAndOffsets => + if (errAndOffsets.error == ErrorMapping.NoError) { + result += tp -> errAndOffsets.offsets + } else { + errs.append(ErrorMapping.exceptionFor(errAndOffsets.error)) + } + } + } + if (result.keys.size == topicAndPartitions.size) { + return Right(result) + } + } + val missing = topicAndPartitions.diff(result.keys.toSet) + errs.append(new Exception(s"Couldn't find offsets for ${missing}")) + Left(errs) + } + + def getConsumerOffsets(groupId: String, topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = { + var result = Map[TopicAndPartition, Long]() + val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) + val errs = new Err + withBrokers(errs) { consumer => + val resp: OffsetFetchResponse = consumer.fetchOffsets(req) + val respMap = resp.requestInfo + val needed = topicAndPartitions.diff(result.keys.toSet) + needed.foreach { tp => + respMap.get(tp).foreach { offsetMeta => + if (offsetMeta.error == ErrorMapping.NoError) { + result += tp -> offsetMeta.offset + } else { + errs.append(ErrorMapping.exceptionFor(offsetMeta.error)) + } + } + } + if (result.keys.size == topicAndPartitions.size) { + return Right(result) + } + } + val missing = topicAndPartitions.diff(result.keys.toSet) + errs.append(new Exception(s"Couldn't find offsets for ${missing}")) + Left(errs) + } + + def setConsumerOffsets(groupId: String, offsets: Map[TopicAndPartition, Long]): Unit = ??? + + private def withBrokers(errs: Err)(fn: SimpleConsumer => Any): Unit = { brokers.foreach { hp => var consumer: SimpleConsumer = null try { consumer = connect(hp) - val resp: OffsetResponse = consumer.getOffsetsBefore(req) - val respParts = resp.partitionErrorAndOffsets - val needed = topicAndPartitions.diff(result.keys.toSet) - needed.foreach { tp => - respParts.get(tp).foreach { errAndOffsets => - if (errAndOffsets.error == ErrorMapping.NoError) { - result += tp -> errAndOffsets.offsets - } - } - } - if (result.keys.size == topicAndPartitions.size) { - return result - } + fn(consumer) } catch { case NonFatal(e) => + errs.append(e) } finally { if (consumer != null) consumer.close() } } - val missing = topicAndPartitions.diff(result.keys.toSet) - throw new Exception(s"Couldn't find offsets for ${missing}") } - - def getConsumerOffsets(topicAndPartitions: Set[TopicAndPartition]): Map[TopicAndPartition, Long] = ??? - - def setConsumerOffsets(offsets: Map[TopicAndPartition, Long]): Unit = ??? } object KafkaCluster { diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 32e6d297133e0..d155f22f93931 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -82,8 +82,10 @@ class KafkaRDD[ val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) .asInstanceOf[Decoder[V]] - val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition) - .getOrElse(throw new Exception(s"Couldn't connect to leader for topic ${part.topic} ${part.partition}")) + val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( + errs => throw new Exception(s"""Couldn't connect to leader for topic ${part.topic} ${part.partition}: ${errs.mkString("\n")}"""), + consumer => consumer + ) var requestOffset = part.afterOffset + 1 var iter: Iterator[MessageAndOffset] = null From 7d050bcb0bcacfbd4a7b858cffae809fd2af8e9d Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 24 Nov 2014 16:11:24 -0600 Subject: [PATCH 06/42] methods to set consumer offsets and get topic metadata, switch back to inclusive start / exclusive end to match typical kafka consumer behavior --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 80 ++++++++++-- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 119 ++++++++++-------- 2 files changed, 137 insertions(+), 62 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 721d90264a1db..2c0b62f8b0fb8 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -20,8 +20,8 @@ package org.apache.spark.rdd.kafka import scala.util.control.NonFatal import scala.collection.mutable.ArrayBuffer import java.util.Properties -import kafka.api.{OffsetRequest, OffsetResponse, OffsetFetchRequest, OffsetFetchResponse, PartitionOffsetRequestInfo, TopicMetadataRequest, TopicMetadataResponse} -import kafka.common.{ErrorMapping, TopicAndPartition} +import kafka.api.{OffsetCommitRequest, OffsetRequest, OffsetFetchRequest, PartitionOffsetRequestInfo, TopicMetadata, TopicMetadataRequest, TopicMetadataResponse} +import kafka.common.{ErrorMapping, OffsetMetadataAndError, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} /** @@ -69,6 +69,27 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } + def getPartitions(topics: Set[String]): Either[Err, Set[TopicAndPartition]] = + getPartitionMetadata(topics).right.map { r => + r.flatMap { tm: TopicMetadata => + tm.partitionsMetadata.map { pm => + TopicAndPartition(tm.topic, pm.partitionId) + } + } + } + + def getPartitionMetadata(topics: Set[String]): Either[Err, Set[TopicMetadata]] = { + val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) + val errs = new Err + withBrokers(errs) { consumer => + val resp: TopicMetadataResponse = consumer.send(req) + // error codes here indicate missing / just created topic, + // repeating on a different broker wont be useful + return Right(resp.topicsMetadata.toSet) + } + Left(errs) + } + def getLatestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) @@ -94,7 +115,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { ) val errs = new Err withBrokers(errs) { consumer => - val resp: OffsetResponse = consumer.getOffsetsBefore(req) + val resp = consumer.getOffsetsBefore(req) val respMap = resp.partitionErrorAndOffsets val needed = topicAndPartitions.diff(result.keys.toSet) needed.foreach { tp => @@ -116,17 +137,28 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } def getConsumerOffsets(groupId: String, topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = { - var result = Map[TopicAndPartition, Long]() + getConsumerOffsetMetadata(groupId, topicAndPartitions).right.map { r => + r.map { kv => + kv._1 -> kv._2.offset + } + } + } + + def getConsumerOffsetMetadata( + groupId: String, + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, OffsetMetadataAndError]] = { + var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) val errs = new Err withBrokers(errs) { consumer => - val resp: OffsetFetchResponse = consumer.fetchOffsets(req) + val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keys.toSet) needed.foreach { tp => respMap.get(tp).foreach { offsetMeta => if (offsetMeta.error == ErrorMapping.NoError) { - result += tp -> offsetMeta.offset + result += tp -> offsetMeta } else { errs.append(ErrorMapping.exceptionFor(offsetMeta.error)) } @@ -141,7 +173,41 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } - def setConsumerOffsets(groupId: String, offsets: Map[TopicAndPartition, Long]): Unit = ??? + def setConsumerOffsets(groupId: String, offsets: Map[TopicAndPartition, Long]): Unit = { + setConsumerOffsetMetadata(groupId, offsets.map { kv => + kv._1 -> OffsetMetadataAndError(kv._2) + }) + } + + def setConsumerOffsetMetadata( + groupId: String, + metadata: Map[TopicAndPartition, OffsetMetadataAndError] + ): Either[Err, Map[TopicAndPartition, Short]] = { + var result = Map[TopicAndPartition, Short]() + val req = OffsetCommitRequest(groupId, metadata) + val errs = new Err + val topicAndPartitions = metadata.keys.toSet + withBrokers(errs) { consumer => + val resp = consumer.commitOffsets(req) + val respMap = resp.requestInfo + val needed = topicAndPartitions.diff(result.keys.toSet) + needed.foreach { tp => + respMap.get(tp).foreach { err => + if (err == ErrorMapping.NoError) { + result += tp -> err + } else { + errs.append(ErrorMapping.exceptionFor(err)) + } + } + } + if (result.keys.size == topicAndPartitions.size) { + return Right(result) + } + } + val missing = topicAndPartitions.diff(result.keys.toSet) + errs.append(new Exception(s"Couldn't set offsets for ${missing}")) + Left(errs) + } private def withBrokers(errs: Err)(fn: SimpleConsumer => Any): Unit = { brokers.foreach { hp => diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index d155f22f93931..b8ec122a086be 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -35,8 +35,8 @@ private[spark] case class KafkaRDDPartition( override val index: Int, topic: String, partition: Int, - afterOffset: Long, - throughOffset: Long + fromOffset: Long, + untilOffset: Long ) extends Partition /** A batch-oriented interface for consuming from Kafka. @@ -46,8 +46,8 @@ private[spark] case class KafkaRDDPartition( * @param kafkaParams Kafka configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param afterOffsets per-topic/partition Kafka offsets defining the (exclusive) starting point of the batch - * @param throughOffsets per-topic/partition Kafka offsets defining the (inclusive) ending point of the batch + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch + * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch * @param messageHandler function for translating each message into the desired type */ class KafkaRDD[ @@ -58,71 +58,80 @@ class KafkaRDD[ R: ClassTag]( sc: SparkContext, kafkaParams: Map[String, String], - afterOffsets: Map[TopicAndPartition, Long], - throughOffsets: Map[TopicAndPartition, Long], + fromOffsets: Map[TopicAndPartition, Long], + untilOffsets: Map[TopicAndPartition, Long], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging { - assert(afterOffsets.keys == throughOffsets.keys, + assert(fromOffsets.keys == untilOffsets.keys, "Must provide both from and until offsets for each topic/partition") - override def getPartitions: Array[Partition] = afterOffsets.zipWithIndex.map { kvi => + override def getPartitions: Array[Partition] = fromOffsets.zipWithIndex.map { kvi => val ((tp, from), index) = kvi - new KafkaRDDPartition(index, tp.topic, tp.partition, from, throughOffsets(tp)) + new KafkaRDDPartition(index, tp.topic, tp.partition, from, untilOffsets(tp)) }.toArray - override def compute(thePart: Partition, context: TaskContext) = new NextIterator[R] { - context.addTaskCompletionListener{ context => closeIfNeeded() } - - val kc = new KafkaCluster(kafkaParams) + override def compute(thePart: Partition, context: TaskContext) = { val part = thePart.asInstanceOf[KafkaRDDPartition] - val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(kc.config.props) - .asInstanceOf[Decoder[K]] - val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(kc.config.props) - .asInstanceOf[Decoder[V]] - val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( - errs => throw new Exception(s"""Couldn't connect to leader for topic ${part.topic} ${part.partition}: ${errs.mkString("\n")}"""), - consumer => consumer - ) - var requestOffset = part.afterOffset + 1 - var iter: Iterator[MessageAndOffset] = null + if (part.fromOffset >= part.untilOffset) { + log.warn(s"Beginning offset is same or after ending offset, skipping ${part.topic} ${part.partition}") + Iterator.empty + } else { + new NextIterator[R] { + context.addTaskCompletionListener{ context => closeIfNeeded() } + + val kc = new KafkaCluster(kafkaParams) + log.info(s"Computing partition ${part.topic} ${part.partition} ${part.fromOffset} -> ${part.untilOffset}") + val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(kc.config.props) + .asInstanceOf[Decoder[K]] + val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(kc.config.props) + .asInstanceOf[Decoder[V]] + val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( + errs => throw new Exception(s"""Couldn't connect to leader for topic ${part.topic} ${part.partition}: ${errs.mkString("\n")}"""), + consumer => consumer + ) + var requestOffset = part.fromOffset + var iter: Iterator[MessageAndOffset] = null - override def getNext: R = { - if (iter == null || !iter.hasNext) { - val req = new FetchRequestBuilder(). - addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). - build() - val resp = consumer.fetch(req) - if (resp.hasError) { - val err = resp.errorCode(part.topic, part.partition) - if (err == ErrorMapping.LeaderNotAvailableCode || - err == ErrorMapping.NotLeaderForPartitionCode) { - log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${kc.config.refreshLeaderBackoffMs}ms") - Thread.sleep(kc.config.refreshLeaderBackoffMs) + override def close() = consumer.close() + + override def getNext: R = { + if (iter == null || !iter.hasNext) { + log.info(s"Fetching ${part.topic}, ${part.partition}, ${requestOffset}") + val req = new FetchRequestBuilder(). + addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). + build() + val resp = consumer.fetch(req) + if (resp.hasError) { + val err = resp.errorCode(part.topic, part.partition) + if (err == ErrorMapping.LeaderNotAvailableCode || + err == ErrorMapping.NotLeaderForPartitionCode) { + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${kc.config.refreshLeaderBackoffMs}ms") + Thread.sleep(kc.config.refreshLeaderBackoffMs) + } + // Let normal rdd retry sort out reconnect attempts + throw ErrorMapping.exceptionFor(err) + } + iter = resp.messageSet(part.topic, part.partition) + .iterator + .dropWhile(_.offset < requestOffset) + } + if (!iter.hasNext) { + finished = true + null.asInstanceOf[R] + } else { + val item = iter.next + if (item.offset > part.untilOffset) { + finished = true + } + requestOffset = item.nextOffset + messageHandler(new MessageAndMetadata(part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } - // Let normal rdd retry sort out reconnect attempts - throw ErrorMapping.exceptionFor(err) - } - iter = resp.messageSet(part.topic, part.partition) - .iterator - .dropWhile(_.offset < requestOffset) - } - if (!iter.hasNext) { - finished = true - null.asInstanceOf[R] - } else { - val item = iter.next - if (item.offset > part.throughOffset) { - finished = true } - requestOffset = item.nextOffset - messageHandler(new MessageAndMetadata(part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } } - - override def close() = consumer.close() } } From 783b4775c89dbcbde9172d34653eab2718eee494 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 25 Nov 2014 08:29:20 -0600 Subject: [PATCH 07/42] update tests for kafka 8.1.1 --- .../org/apache/spark/streaming/kafka/KafkaStreamSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala index b19c053ebfc44..52454f4206d15 100644 --- a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala @@ -26,7 +26,7 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.Random -import kafka.admin.CreateTopicCommand +import kafka.admin.AdminUtils import kafka.common.{KafkaException, TopicAndPartition} import kafka.producer.{KeyedMessage, Producer, ProducerConfig} import kafka.serializer.{StringDecoder, StringEncoder} @@ -130,7 +130,7 @@ abstract class KafkaStreamSuiteBase extends FunSuite with Eventually with Loggin } def createTopic(topic: String) { - CreateTopicCommand.createTopic(zkClient, topic, 1, 1, "0") + AdminUtils.createTopic(zkClient, topic, 1, 1) logInfo("==================== 5 ====================") // wait until metadata is propagated waitUntilMetadataIsPropagated(topic, 0) @@ -166,7 +166,7 @@ abstract class KafkaStreamSuiteBase extends FunSuite with Eventually with Loggin private def waitUntilMetadataIsPropagated(topic: String, partition: Int) { eventually(timeout(1000 milliseconds), interval(100 milliseconds)) { assert( - server.apis.leaderCache.keySet.contains(TopicAndPartition(topic, partition)), + server.apis.metadataCache.containsTopicAndPartition(topic, partition), s"Partition [$topic, $partition] metadata not propagated after timeout" ) } From 29c6b430cc6bf5e2354b397289c4445f4993fc5b Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 25 Nov 2014 09:33:45 -0600 Subject: [PATCH 08/42] cleanup logging --- .../src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index b8ec122a086be..5b946d448f140 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -81,7 +81,7 @@ class KafkaRDD[ context.addTaskCompletionListener{ context => closeIfNeeded() } val kc = new KafkaCluster(kafkaParams) - log.info(s"Computing partition ${part.topic} ${part.partition} ${part.fromOffset} -> ${part.untilOffset}") + log.info(s"Computing topic ${part.topic}, partition ${part.partition}, offsets ${part.fromOffset} -> ${part.untilOffset}") val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) .asInstanceOf[Decoder[K]] @@ -99,7 +99,6 @@ class KafkaRDD[ override def getNext: R = { if (iter == null || !iter.hasNext) { - log.info(s"Fetching ${part.topic}, ${part.partition}, ${requestOffset}") val req = new FetchRequestBuilder(). addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). build() From 3c2a96af2322754e8c76000b083ec3630a03e8c8 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 25 Nov 2014 14:02:37 -0600 Subject: [PATCH 09/42] fix scalastyle errors --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 42 +++++++++++++------ .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 31 +++++++++----- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 2c0b62f8b0fb8..b27c2bcbf7464 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -20,13 +20,14 @@ package org.apache.spark.rdd.kafka import scala.util.control.NonFatal import scala.collection.mutable.ArrayBuffer import java.util.Properties -import kafka.api.{OffsetCommitRequest, OffsetRequest, OffsetFetchRequest, PartitionOffsetRequestInfo, TopicMetadata, TopicMetadataRequest, TopicMetadataResponse} +import kafka.api._ import kafka.common.{ErrorMapping, OffsetMetadataAndError, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} /** * Convenience methods for interacting with a Kafka cluster. - * @param kafkaParams Kafka configuration parameters. + * @param kafkaParams Kafka + * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ @@ -45,7 +46,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { val config: ConsumerConfig = KafkaCluster.consumerConfig(kafkaParams) def connect(host: String, port: Int): SimpleConsumer = - new SimpleConsumer(host, port, config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId) + new SimpleConsumer(host, port, config.socketTimeoutMs, + config.socketReceiveBufferBytes, config.clientId) def connect(hostAndPort: (String, Int)): SimpleConsumer = connect(hostAndPort._1, hostAndPort._2) @@ -54,7 +56,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { findLeader(topic, partition).right.map(connect) def findLeader(topic: String, partition: Int): Either[Err, (String, Int)] = { - val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) + val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, + 0, config.clientId, Seq(topic)) val errs = new Err withBrokers(errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) @@ -79,7 +82,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } def getPartitionMetadata(topics: Set[String]): Either[Err, Set[TopicMetadata]] = { - val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) + val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, + 0, config.clientId, topics.toSeq) val errs = new Err withBrokers(errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) @@ -90,13 +94,20 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } - def getLatestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = + def getLatestLeaderOffsets( + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) - def getEarliestLeaderOffsets(topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = + def getEarliestLeaderOffsets( + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.EarliestTime) - def getLeaderOffsets(topicAndPartitions: Set[TopicAndPartition], before: Long): Either[Err, Map[TopicAndPartition, Long]] = + def getLeaderOffsets( + topicAndPartitions: Set[TopicAndPartition], + before: Long + ): Either[Err, Map[TopicAndPartition, Long]] = getLeaderOffsets(topicAndPartitions, before, 1).right.map { r => r.map { kv => // mapValues isnt serializable, see SI-7005 @@ -136,7 +147,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } - def getConsumerOffsets(groupId: String, topicAndPartitions: Set[TopicAndPartition]): Either[Err, Map[TopicAndPartition, Long]] = { + def getConsumerOffsets( + groupId: String, + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, Long]] = { getConsumerOffsetMetadata(groupId, topicAndPartitions).right.map { r => r.map { kv => kv._1 -> kv._2.offset @@ -173,7 +187,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } - def setConsumerOffsets(groupId: String, offsets: Map[TopicAndPartition, Long]): Unit = { + def setConsumerOffsets( + groupId: String, + offsets: Map[TopicAndPartition, Long] + ): Either[Err, Map[TopicAndPartition, Short]] = { setConsumerOffsetMetadata(groupId, offsets.map { kv => kv._1 -> OffsetMetadataAndError(kv._2) }) @@ -233,8 +250,9 @@ object KafkaCluster { val props = new Properties() kafkaParams.foreach(param => props.put(param._1, param._2)) Seq("zookeeper.connect", "group.id").foreach { s => - if (!props.contains(s)) - props.setProperty(s, "") + if (!props.contains(s)) { + props.setProperty(s, "") + } } new ConsumerConfig(props) } diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 5b946d448f140..6d3d8e88218d1 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -41,13 +41,18 @@ private[spark] case class KafkaRDDPartition( /** A batch-oriented interface for consuming from Kafka. * Each given Kafka topic/partition corresponds to an RDD partition. - * Starting and ending offsets are specified in advance, so that you can control exactly-once semantics. - * For an easy interface to Kafka-managed offsets, see {@link org.apache.spark.rdd.kafka.KafkaCluster} - * @param kafkaParams Kafka configuration parameters. + * Starting and ending offsets are specified in advance, + * so that you can control exactly-once semantics. + * For an easy interface to Kafka-managed offsets, + * see {@link org.apache.spark.rdd.kafka.KafkaCluster} + * @param kafkaParams Kafka + * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch - * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the batch + * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) + * ending point of the batch * @param messageHandler function for translating each message into the desired type */ class KafkaRDD[ @@ -74,14 +79,16 @@ class KafkaRDD[ override def compute(thePart: Partition, context: TaskContext) = { val part = thePart.asInstanceOf[KafkaRDDPartition] if (part.fromOffset >= part.untilOffset) { - log.warn(s"Beginning offset is same or after ending offset, skipping ${part.topic} ${part.partition}") + log.warn("Beginning offset is same or after ending offset" + + s"skipping ${part.topic} ${part.partition}") Iterator.empty } else { new NextIterator[R] { context.addTaskCompletionListener{ context => closeIfNeeded() } val kc = new KafkaCluster(kafkaParams) - log.info(s"Computing topic ${part.topic}, partition ${part.partition}, offsets ${part.fromOffset} -> ${part.untilOffset}") + log.info(s"Computing topic ${part.topic}, partition ${part.partition}" + + s"offsets ${part.fromOffset} -> ${part.untilOffset}") val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) .asInstanceOf[Decoder[K]] @@ -89,7 +96,9 @@ class KafkaRDD[ .newInstance(kc.config.props) .asInstanceOf[Decoder[V]] val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( - errs => throw new Exception(s"""Couldn't connect to leader for topic ${part.topic} ${part.partition}: ${errs.mkString("\n")}"""), + errs => throw new Exception( + s"Couldn't connect to leader for topic ${part.topic} ${part.partition}:" + + errs.mkString("\n")), consumer => consumer ) var requestOffset = part.fromOffset @@ -107,7 +116,8 @@ class KafkaRDD[ val err = resp.errorCode(part.topic, part.partition) if (err == ErrorMapping.LeaderNotAvailableCode || err == ErrorMapping.NotLeaderForPartitionCode) { - log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, sleeping for ${kc.config.refreshLeaderBackoffMs}ms") + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, " + + s" sleeping for ${kc.config.refreshLeaderBackoffMs}ms") Thread.sleep(kc.config.refreshLeaderBackoffMs) } // Let normal rdd retry sort out reconnect attempts @@ -126,7 +136,8 @@ class KafkaRDD[ finished = true } requestOffset = item.nextOffset - messageHandler(new MessageAndMetadata(part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) + messageHandler(new MessageAndMetadata( + part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } } } From 4b078bf1e71745a6bc160c0836b54cc7b0d4171d Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 25 Nov 2014 14:48:32 -0600 Subject: [PATCH 10/42] differentiate between leader and consumer offsets in error message --- .../main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index b27c2bcbf7464..45b6f58623023 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -143,7 +143,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } } val missing = topicAndPartitions.diff(result.keys.toSet) - errs.append(new Exception(s"Couldn't find offsets for ${missing}")) + errs.append(new Exception(s"Couldn't find leader offsets for ${missing}")) Left(errs) } @@ -183,7 +183,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } } val missing = topicAndPartitions.diff(result.keys.toSet) - errs.append(new Exception(s"Couldn't find offsets for ${missing}")) + errs.append(new Exception(s"Couldn't find consumer offsets for ${missing}")) Left(errs) } From 8d7de4ab5a447a53f65be852702ca90512b2a639 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 25 Nov 2014 17:54:40 -0600 Subject: [PATCH 11/42] make sure leader offsets can be found even for leaders that arent in the seed brokers --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 93 +++++++++++++------ 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 45b6f58623023..8cfcc97788f55 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -34,7 +34,7 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} class KafkaCluster(val kafkaParams: Map[String, String]) { type Err = ArrayBuffer[Throwable] - val brokers: Array[(String, Int)] = + val seedBrokers: Array[(String, Int)] = kafkaParams.get("metadata.broker.list") .orElse(kafkaParams.get("bootstrap.servers")) .getOrElse(throw new Exception("Must specify metadata.broker.list or bootstrap.servers")) @@ -59,7 +59,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) val errs = new Err - withBrokers(errs) { consumer => + withBrokers(seedBrokers, errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) resp.topicsMetadata.find(_.topic == topic).flatMap { t => t.partitionsMetadata.find(_.partitionId == partition) @@ -72,6 +72,33 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } + def findLeaders( + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, (String, Int)]] = { + getPartitionMetadata(topicAndPartitions.map(_.topic)).right.flatMap { tms => + val result = tms.flatMap { tm: TopicMetadata => + tm.partitionsMetadata.flatMap { pm => + val tp = TopicAndPartition(tm.topic, pm.partitionId) + if (topicAndPartitions(tp)) { + pm.leader.map { l => + tp -> (l.host -> l.port) + } + } else { + None + } + } + }.toMap + if (result.keys.size == topicAndPartitions.size) { + Right(result) + } else { + val missing = topicAndPartitions.diff(result.keys.toSet) + val err = new Err + err.append(new Exception(s"Couldn't find leaders for ${missing}")) + Left(err) + } + } + } + def getPartitions(topics: Set[String]): Either[Err, Set[TopicAndPartition]] = getPartitionMetadata(topics).right.map { r => r.flatMap { tm: TopicMetadata => @@ -85,7 +112,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) val errs = new Err - withBrokers(errs) { consumer => + withBrokers(seedBrokers, errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) // error codes here indicate missing / just created topic, // repeating on a different broker wont be useful @@ -115,36 +142,47 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } } + private def flip[K, V](m: Map[K, V]): Map[V, Seq[K]] = + m.groupBy(_._2).map { kv => + kv._1 -> kv._2.keys.toSeq + } + def getLeaderOffsets( topicAndPartitions: Set[TopicAndPartition], before: Long, maxNumOffsets: Int ): Either[Err, Map[TopicAndPartition, Seq[Long]]] = { - var result = Map[TopicAndPartition, Seq[Long]]() - val req = OffsetRequest( - topicAndPartitions.map(tp => tp -> PartitionOffsetRequestInfo(before, 1)).toMap - ) - val errs = new Err - withBrokers(errs) { consumer => - val resp = consumer.getOffsetsBefore(req) - val respMap = resp.partitionErrorAndOffsets - val needed = topicAndPartitions.diff(result.keys.toSet) - needed.foreach { tp => - respMap.get(tp).foreach { errAndOffsets => - if (errAndOffsets.error == ErrorMapping.NoError) { - result += tp -> errAndOffsets.offsets - } else { - errs.append(ErrorMapping.exceptionFor(errAndOffsets.error)) + findLeaders(topicAndPartitions).right.flatMap { tpToLeader => + val leaderToTp: Map[(String, Int), Seq[TopicAndPartition]] = flip(tpToLeader) + val leaders = leaderToTp.keys + var result = Map[TopicAndPartition, Seq[Long]]() + val errs = new Err + withBrokers(leaders, errs) { consumer => + val needed: Seq[TopicAndPartition] = leaderToTp((consumer.host, consumer.port)) + val req = OffsetRequest( + needed.map { tp => + tp -> PartitionOffsetRequestInfo(before, maxNumOffsets) + }.toMap + ) + val resp = consumer.getOffsetsBefore(req) + val respMap = resp.partitionErrorAndOffsets + needed.foreach { tp => + respMap.get(tp).foreach { errAndOffsets => + if (errAndOffsets.error == ErrorMapping.NoError) { + result += tp -> errAndOffsets.offsets + } else { + errs.append(ErrorMapping.exceptionFor(errAndOffsets.error)) + } } } + if (result.keys.size == topicAndPartitions.size) { + return Right(result) + } } - if (result.keys.size == topicAndPartitions.size) { - return Right(result) - } + val missing = topicAndPartitions.diff(result.keys.toSet) + errs.append(new Exception(s"Couldn't find leader offsets for ${missing}")) + Left(errs) } - val missing = topicAndPartitions.diff(result.keys.toSet) - errs.append(new Exception(s"Couldn't find leader offsets for ${missing}")) - Left(errs) } def getConsumerOffsets( @@ -165,7 +203,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) val errs = new Err - withBrokers(errs) { consumer => + withBrokers(seedBrokers, errs) { consumer => val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keys.toSet) @@ -204,7 +242,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { val req = OffsetCommitRequest(groupId, metadata) val errs = new Err val topicAndPartitions = metadata.keys.toSet - withBrokers(errs) { consumer => + withBrokers(seedBrokers, errs) { consumer => val resp = consumer.commitOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keys.toSet) @@ -226,7 +264,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { Left(errs) } - private def withBrokers(errs: Err)(fn: SimpleConsumer => Any): Unit = { + private def withBrokers(brokers: Iterable[(String, Int)], errs: Err) + (fn: SimpleConsumer => Any): Unit = { brokers.foreach { hp => var consumer: SimpleConsumer = null try { From 979da25f4d48e5ffccf13ba1ff66eb2527ff01f9 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 26 Nov 2014 09:31:38 -0600 Subject: [PATCH 12/42] dont allow empty leader offsets to be returned --- .../scala/org/apache/spark/rdd/kafka/KafkaCluster.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 8cfcc97788f55..4c48639d939ff 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -169,7 +169,12 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { needed.foreach { tp => respMap.get(tp).foreach { errAndOffsets => if (errAndOffsets.error == ErrorMapping.NoError) { - result += tp -> errAndOffsets.offsets + if (errAndOffsets.offsets.nonEmpty) { + result += tp -> errAndOffsets.offsets + } else { + errs.append(new Exception( + s"Empty offsets for ${tp}, is ${before} before log beginning?")) + } } else { errs.append(ErrorMapping.exceptionFor(errAndOffsets.error)) } From 38bb727cf31744fa625248c86c2a666920e83c36 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 3 Dec 2014 15:42:25 -0600 Subject: [PATCH 13/42] give easy access to the parameters of a KafkaRDD --- .../main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 6d3d8e88218d1..c2a56b556fdd2 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -62,9 +62,9 @@ class KafkaRDD[ T <: Decoder[_]: ClassTag, R: ClassTag]( sc: SparkContext, - kafkaParams: Map[String, String], - fromOffsets: Map[TopicAndPartition, Long], - untilOffsets: Map[TopicAndPartition, Long], + val kafkaParams: Map[String, String], + val fromOffsets: Map[TopicAndPartition, Long], + val untilOffsets: Map[TopicAndPartition, Long], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging { From 326ff3cbda37066ebef7492241276754164d2879 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 16 Dec 2014 15:27:44 -0600 Subject: [PATCH 14/42] add some tests --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 4 +- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 6 +- .../spark/rdd/kafka/KafkaRDDSuite.scala | 79 +++++++++++++++++++ .../streaming/kafka/KafkaStreamSuite.scala | 2 +- 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 4c48639d939ff..afbe18ff1c3e0 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -32,7 +32,7 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ class KafkaCluster(val kafkaParams: Map[String, String]) { - type Err = ArrayBuffer[Throwable] + import KafkaCluster.Err val seedBrokers: Array[(String, Int)] = kafkaParams.get("metadata.broker.list") @@ -287,6 +287,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { } object KafkaCluster { + type Err = ArrayBuffer[Throwable] + /** Make a consumer config without requiring group.id or zookeeper.connect, * since communicating with brokers also needs common settings such as timeout */ diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index c2a56b556fdd2..5837dd405aee5 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -79,7 +79,7 @@ class KafkaRDD[ override def compute(thePart: Partition, context: TaskContext) = { val part = thePart.asInstanceOf[KafkaRDDPartition] if (part.fromOffset >= part.untilOffset) { - log.warn("Beginning offset is same or after ending offset" + + log.warn("Beginning offset is same or after ending offset " + s"skipping ${part.topic} ${part.partition}") Iterator.empty } else { @@ -87,7 +87,7 @@ class KafkaRDD[ context.addTaskCompletionListener{ context => closeIfNeeded() } val kc = new KafkaCluster(kafkaParams) - log.info(s"Computing topic ${part.topic}, partition ${part.partition}" + + log.info(s"Computing topic ${part.topic}, partition ${part.partition} " + s"offsets ${part.fromOffset} -> ${part.untilOffset}") val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) @@ -97,7 +97,7 @@ class KafkaRDD[ .asInstanceOf[Decoder[V]] val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( errs => throw new Exception( - s"Couldn't connect to leader for topic ${part.topic} ${part.partition}:" + + s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + errs.mkString("\n")), consumer => consumer ) diff --git a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala new file mode 100644 index 0000000000000..284c9d9dc996d --- /dev/null +++ b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.rdd.kafka + +import scala.util.Random + +import kafka.serializer.StringDecoder +import org.scalatest.BeforeAndAfter + +import org.apache.spark._ +import org.apache.spark.SparkContext._ +import org.apache.spark.streaming.kafka.KafkaStreamSuiteBase + +class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { + var sc: SparkContext = _ + before { + setupKafka() + } + + after { + if (sc != null) { + sc.stop + sc = null + } + tearDownKafka() + } + + test("Kafka RDD") { + val sparkConf = new SparkConf().setMaster("local[4]").setAppName(this.getClass.getSimpleName) + sc = new SparkContext(sparkConf) + val topic = "topic1" + val sent = Map("a" -> 5, "b" -> 3, "c" -> 10) + createTopic(topic) + produceAndSendMessage(topic, sent) + + val kafkaParams = Map("metadata.broker.list" -> s"localhost:$brokerPort", + "group.id" -> s"test-consumer-${Random.nextInt(10000)}") + + val kc = new KafkaCluster(kafkaParams) + + val rdd = getRdd(kc, Set(topic)) + assert(rdd.isDefined) + assert(rdd.get.countByValue.size === sent.size) + + kc.setConsumerOffsets(kafkaParams("group.id"), rdd.get.untilOffsets) + + val rdd2 = getRdd(kc, Set(topic)) + assert(rdd2.isDefined) + assert(rdd2.get.count === 0) + } + + private def getRdd(kc: KafkaCluster, topics: Set[String]) = { + val groupId = kc.kafkaParams("group.id") + for { + topicPartitions <- kc.getPartitions(topics).right.toOption + from <- kc.getConsumerOffsets(groupId, topicPartitions).right.toOption.orElse( + kc.getEarliestLeaderOffsets(topicPartitions).right.toOption) + until <- kc.getLatestLeaderOffsets(topicPartitions).right.toOption + } yield { + new KafkaRDD[String, String, StringDecoder, StringDecoder, String]( + sc, kc.kafkaParams, from, until, mmd => mmd.message) + } + } +} diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala index 52454f4206d15..629a758f6f1a3 100644 --- a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaStreamSuite.scala @@ -56,7 +56,7 @@ abstract class KafkaStreamSuiteBase extends FunSuite with Eventually with Loggin private val zkSessionTimeout = 6000 private var zookeeper: EmbeddedZookeeper = _ private var zkPort: Int = 0 - private var brokerPort = 9092 + protected var brokerPort = 9092 private var brokerConf: KafkaConfig = _ private var server: KafkaServer = _ private var producer: Producer[String, String] = _ From 6bf14f2850f9f40f53b4c1eec373214e1b6d3465 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 24 Dec 2014 11:38:52 -0600 Subject: [PATCH 15/42] first attempt at a Kafka dstream that allows for exactly-once semantics --- .../DeterministicKafkaInputDStream.scala | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala new file mode 100644 index 0000000000000..cd975acbd58c6 --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import scala.annotation.tailrec +import scala.reflect.{classTag, ClassTag} + +import kafka.common.TopicAndPartition +import kafka.message.MessageAndMetadata +import kafka.serializer.Decoder + +import org.apache.spark.Logging +import org.apache.spark.rdd.RDD +import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD} +import org.apache.spark.streaming.{StreamingContext, Time} +import org.apache.spark.streaming.dstream._ + +/** A stream of {@link org.apache.spark.rdd.kafka.KafkaRDD} where + * each given Kafka topic/partition corresponds to an RDD partition. + * Starting offsets are specified in advance, + * and this DStream is not responsible for committing offsets, + * so that you can control exactly-once semantics. + * For an easy interface to Kafka-managed offsets, + * see {@link org.apache.spark.rdd.kafka.KafkaCluster} + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the stream + * @param messageHandler function for translating each message into the desired type + * @param maxRetries maximum number of times in a row to retry getting leaders' offsets + */ +class DeterministicKafkaInputDStream[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag]( + @transient ssc_ : StreamingContext, + val kafkaParams: Map[String, String], + val fromOffsets: Map[TopicAndPartition, Long], + messageHandler: MessageAndMetadata[K, V] => R, + maxRetries: Int = 1 +) extends InputDStream[R](ssc_) with Logging { + + private val kc = new KafkaCluster(kafkaParams) + + // TODO based on the design of InputDStream's lastValidTime, it appears there isn't a + // thread safety concern with private mutable state, but is this certain? + private var currentOffsets = fromOffsets + + @tailrec + private def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, Long] = { + val o = kc.getLatestLeaderOffsets(currentOffsets.keys.toSet) + // Either.fold would confuse @tailrec, do it manually + if (o.isLeft) { + val err = o.left.get.toString + if (retries <= 0) { + throw new Exception(err) + } else { + log.error(err) + Thread.sleep(kc.config.refreshLeaderBackoffMs) + latestLeaderOffsets(retries - 1) + } + } else { + o.right.get + } + } + + override def compute(validTime: Time): Option[KafkaRDD[K, V, U, T, R]] = { + val untilOffsets = latestLeaderOffsets(maxRetries) + val rdd = new KafkaRDD[K, V, U, T, R]( + ssc_.sparkContext, kafkaParams, currentOffsets, untilOffsets, messageHandler) + + currentOffsets = untilOffsets + Some(rdd) + } + + override def start(): Unit = { + } + + def stop(): Unit = { + } +} From 37d305320e72de1ee6ffcd42f6a45d331a4d465d Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 24 Dec 2014 22:41:40 -0600 Subject: [PATCH 16/42] make KafkaRDDPartition available to users so offsets can be committed per partition --- .../src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 5837dd405aee5..5a9fd18c338c3 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -31,7 +31,7 @@ import kafka.message.{MessageAndMetadata, MessageAndOffset} import kafka.serializer.Decoder import kafka.utils.VerifiableProperties -private[spark] case class KafkaRDDPartition( +case class KafkaRDDPartition( override val index: Int, topic: String, partition: Int, From cac63eec4a0bee6b662c4577404622a08904f0cb Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Thu, 25 Dec 2014 01:11:58 -0600 Subject: [PATCH 17/42] additional testing, fix fencepost error --- .../scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 10 ++++++---- .../org/apache/spark/rdd/kafka/KafkaRDDSuite.scala | 12 ++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 5a9fd18c338c3..6ab698d8b61f1 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -132,12 +132,14 @@ class KafkaRDD[ null.asInstanceOf[R] } else { val item = iter.next - if (item.offset > part.untilOffset) { + if (item.offset >= part.untilOffset) { finished = true + null.asInstanceOf[R] + } else { + requestOffset = item.nextOffset + messageHandler(new MessageAndMetadata( + part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } - requestOffset = item.nextOffset - messageHandler(new MessageAndMetadata( - part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } } } diff --git a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala index 284c9d9dc996d..f8a603c9fa965 100644 --- a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala @@ -55,13 +55,21 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { val rdd = getRdd(kc, Set(topic)) assert(rdd.isDefined) - assert(rdd.get.countByValue.size === sent.size) + assert(rdd.get.count === sent.values.sum) kc.setConsumerOffsets(kafkaParams("group.id"), rdd.get.untilOffsets) val rdd2 = getRdd(kc, Set(topic)) + val sent2 = Map("d" -> 1) + produceAndSendMessage(topic, sent2) assert(rdd2.isDefined) assert(rdd2.get.count === 0) + + val rdd3 = getRdd(kc, Set(topic)) + produceAndSendMessage(topic, Map("extra" -> 22)) + assert(rdd3.isDefined) + assert(rdd3.get.count === sent2.values.sum) + } private def getRdd(kc: KafkaCluster, topics: Set[String]) = { @@ -73,7 +81,7 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { until <- kc.getLatestLeaderOffsets(topicPartitions).right.toOption } yield { new KafkaRDD[String, String, StringDecoder, StringDecoder, String]( - sc, kc.kafkaParams, from, until, mmd => mmd.message) + sc, kc.kafkaParams, from, until, mmd => s"${mmd.offset} ${mmd.message}") } } } From e09045b6088fb8df47cf12ce61f8d3b88c236acf Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 26 Dec 2014 16:07:38 -0600 Subject: [PATCH 18/42] [SPARK-4964] add foreachPartitionWithIndex, to avoid doing equivalent map + empty foreach boilerplate --- core/src/main/scala/org/apache/spark/rdd/RDD.scala | 14 ++++++++++++++ .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index f47c2d1fcdcc7..894fb2133a4f0 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -787,6 +787,20 @@ abstract class RDD[T: ClassTag]( sc.runJob(this, (iter: Iterator[T]) => cleanF(iter)) } + /** + * Applies a function to each partition of this RDD, while tracking the index + * of the original partition. + */ + def foreachPartitionWithIndex( + f: (Int, Iterator[T]) => Unit) { + val func = (index: Int, iter: Iterator[T]) => { + f(index, iter) + Iterator.empty + } + sc.runJob( + mapPartitionsWithIndex(func, true), (iter: Iterator[T]) => ()) + } + /** * Return an array that contains all of the elements in this RDD. */ diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 6ab698d8b61f1..6638525d36185 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -128,11 +128,19 @@ class KafkaRDD[ .dropWhile(_.offset < requestOffset) } if (!iter.hasNext) { + assert(requestOffset == part.untilOffset, + s"ran out of messages before reaching ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates that messages may have been lost") finished = true null.asInstanceOf[R] } else { val item = iter.next if (item.offset >= part.untilOffset) { + assert(item.offset == part.untilOffset, + s"got ${item.offset} > ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates a message may have been skipped") finished = true null.asInstanceOf[R] } else { From 8bfd6c053b5eca6ba5e66593902fce467f95f5f7 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 29 Dec 2014 23:30:23 -0600 Subject: [PATCH 19/42] [SPARK-4964] configure rate limiting via spark.streaming.receiver.maxRate --- .../DeterministicKafkaInputDStream.scala | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index cd975acbd58c6..0cfb0d55e8164 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -32,6 +32,8 @@ import org.apache.spark.streaming.dstream._ /** A stream of {@link org.apache.spark.rdd.kafka.KafkaRDD} where * each given Kafka topic/partition corresponds to an RDD partition. + * The spark configuration spark.streaming.receiver.maxRate gives the maximum number of messages + * per second that each '''partition''' will accept. * Starting offsets are specified in advance, * and this DStream is not responsible for committing offsets, * so that you can control exactly-once semantics. @@ -61,6 +63,16 @@ class DeterministicKafkaInputDStream[ private val kc = new KafkaCluster(kafkaParams) + private val maxMessagesPerPartition: Option[Long] = { + val ratePerSec = ssc.sparkContext.getConf.getInt("spark.streaming.receiver.maxRate", 0) + if (ratePerSec > 0) { + val secsPerBatch = ssc.graph.batchDuration.milliseconds.toDouble / 1000 + Some((secsPerBatch * ratePerSec).toLong) + } else { + None + } + } + // TODO based on the design of InputDStream's lastValidTime, it appears there isn't a // thread safety concern with private mutable state, but is this certain? private var currentOffsets = fromOffsets @@ -83,8 +95,19 @@ class DeterministicKafkaInputDStream[ } } + private def clamp(leaderOffsets: Map[TopicAndPartition, Long]): Map[TopicAndPartition, Long] = { + maxMessagesPerPartition.map { mmp => + leaderOffsets.map { kv => + val (k, v) = kv + val curr = currentOffsets(k) + val diff = v - curr + if (diff > mmp) (k, curr + mmp) else (k, v) + } + }.getOrElse(leaderOffsets) + } + override def compute(validTime: Time): Option[KafkaRDD[K, V, U, T, R]] = { - val untilOffsets = latestLeaderOffsets(maxRetries) + val untilOffsets = clamp(latestLeaderOffsets(maxRetries)) val rdd = new KafkaRDD[K, V, U, T, R]( ssc_.sparkContext, kafkaParams, currentOffsets, untilOffsets, messageHandler) From 1d507490c8300bdb0c423d538aeaa81b402b5926 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 30 Dec 2014 16:43:37 -0600 Subject: [PATCH 20/42] [SPARK-4964] code cleanup per tdas --- .../streaming/kafka/DeterministicKafkaInputDStream.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 0cfb0d55e8164..a7cd48ba7989f 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -73,8 +73,6 @@ class DeterministicKafkaInputDStream[ } } - // TODO based on the design of InputDStream's lastValidTime, it appears there isn't a - // thread safety concern with private mutable state, but is this certain? private var currentOffsets = fromOffsets @tailrec @@ -99,9 +97,7 @@ class DeterministicKafkaInputDStream[ maxMessagesPerPartition.map { mmp => leaderOffsets.map { kv => val (k, v) = kv - val curr = currentOffsets(k) - val diff = v - curr - if (diff > mmp) (k, curr + mmp) else (k, v) + k -> Math.min(currentOffsets(k) + mmp, v) } }.getOrElse(leaderOffsets) } From adf99a6d16d9c3cc86dbd327fba094f85c76b7ea Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sun, 4 Jan 2015 20:58:37 -0600 Subject: [PATCH 21/42] [SPARK-4964] fix serialization issues for checkpointing --- .../org/apache/spark/rdd/kafka/KafkaCluster.scala | 12 ++++++++++-- .../kafka/DeterministicKafkaInputDStream.scala | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index afbe18ff1c3e0..cbe61c0d5fc4a 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -31,7 +31,7 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ -class KafkaCluster(val kafkaParams: Map[String, String]) { +class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { import KafkaCluster.Err val seedBrokers: Array[(String, Int)] = @@ -43,7 +43,15 @@ class KafkaCluster(val kafkaParams: Map[String, String]) { (hpa(0), hpa(1).toInt) } - val config: ConsumerConfig = KafkaCluster.consumerConfig(kafkaParams) + // ConsumerConfig isn't serializable + @transient private var _config: ConsumerConfig = null + + def config: ConsumerConfig = this.synchronized { + if (_config == null) { + _config = KafkaCluster.consumerConfig(kafkaParams) + } + _config + } def connect(host: String, port: Int): SimpleConsumer = new SimpleConsumer(host, port, config.socketTimeoutMs, diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index a7cd48ba7989f..7a33eeb7931aa 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -64,9 +64,9 @@ class DeterministicKafkaInputDStream[ private val kc = new KafkaCluster(kafkaParams) private val maxMessagesPerPartition: Option[Long] = { - val ratePerSec = ssc.sparkContext.getConf.getInt("spark.streaming.receiver.maxRate", 0) + val ratePerSec = context.sparkContext.getConf.getInt("spark.streaming.receiver.maxRate", 0) if (ratePerSec > 0) { - val secsPerBatch = ssc.graph.batchDuration.milliseconds.toDouble / 1000 + val secsPerBatch = context.graph.batchDuration.milliseconds.toDouble / 1000 Some((secsPerBatch * ratePerSec).toLong) } else { None @@ -105,7 +105,7 @@ class DeterministicKafkaInputDStream[ override def compute(validTime: Time): Option[KafkaRDD[K, V, U, T, R]] = { val untilOffsets = clamp(latestLeaderOffsets(maxRetries)) val rdd = new KafkaRDD[K, V, U, T, R]( - ssc_.sparkContext, kafkaParams, currentOffsets, untilOffsets, messageHandler) + context.sparkContext, kafkaParams, currentOffsets, untilOffsets, messageHandler) currentOffsets = untilOffsets Some(rdd) From 356c7ccfb018ca057df06480e39d30cffeeb6922 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 9 Jan 2015 15:50:31 -0600 Subject: [PATCH 22/42] [SPARK-4964] code cleanup per helena --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 14 ++++----- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 29 +++++++++++-------- .../DeterministicKafkaInputDStream.scala | 5 ++-- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index cbe61c0d5fc4a..12e8d36eff78a 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -99,7 +99,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { if (result.keys.size == topicAndPartitions.size) { Right(result) } else { - val missing = topicAndPartitions.diff(result.keys.toSet) + val missing = topicAndPartitions.diff(result.keySet) val err = new Err err.append(new Exception(s"Couldn't find leaders for ${missing}")) Left(err) @@ -192,7 +192,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { return Right(result) } } - val missing = topicAndPartitions.diff(result.keys.toSet) + val missing = topicAndPartitions.diff(result.keySet) errs.append(new Exception(s"Couldn't find leader offsets for ${missing}")) Left(errs) } @@ -219,7 +219,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { withBrokers(seedBrokers, errs) { consumer => val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo - val needed = topicAndPartitions.diff(result.keys.toSet) + val needed = topicAndPartitions.diff(result.keySet) needed.foreach { tp => respMap.get(tp).foreach { offsetMeta => if (offsetMeta.error == ErrorMapping.NoError) { @@ -233,7 +233,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { return Right(result) } } - val missing = topicAndPartitions.diff(result.keys.toSet) + val missing = topicAndPartitions.diff(result.keySet) errs.append(new Exception(s"Couldn't find consumer offsets for ${missing}")) Left(errs) } @@ -254,11 +254,11 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { var result = Map[TopicAndPartition, Short]() val req = OffsetCommitRequest(groupId, metadata) val errs = new Err - val topicAndPartitions = metadata.keys.toSet + val topicAndPartitions = metadata.keySet withBrokers(seedBrokers, errs) { consumer => val resp = consumer.commitOffsets(req) val respMap = resp.requestInfo - val needed = topicAndPartitions.diff(result.keys.toSet) + val needed = topicAndPartitions.diff(result.keySet) needed.foreach { tp => respMap.get(tp).foreach { err => if (err == ErrorMapping.NoError) { @@ -272,7 +272,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { return Right(result) } } - val missing = topicAndPartitions.diff(result.keys.toSet) + val missing = topicAndPartitions.diff(result.keySet) errs.append(new Exception(s"Couldn't set offsets for ${missing}")) Left(errs) } diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 6638525d36185..50528690ae00f 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -24,7 +24,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.util.NextIterator import java.util.Properties -import kafka.api.FetchRequestBuilder +import kafka.api.{FetchRequestBuilder, FetchResponse} import kafka.common.{ErrorMapping, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} import kafka.message.{MessageAndMetadata, MessageAndOffset} @@ -104,6 +104,20 @@ class KafkaRDD[ var requestOffset = part.fromOffset var iter: Iterator[MessageAndOffset] = null + def handleErr(resp: FetchResponse) { + if (resp.hasError) { + val err = resp.errorCode(part.topic, part.partition) + if (err == ErrorMapping.LeaderNotAvailableCode || + err == ErrorMapping.NotLeaderForPartitionCode) { + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, " + + s" sleeping for ${kc.config.refreshLeaderBackoffMs}ms") + Thread.sleep(kc.config.refreshLeaderBackoffMs) + } + // Let normal rdd retry sort out reconnect attempts + throw ErrorMapping.exceptionFor(err) + } + } + override def close() = consumer.close() override def getNext: R = { @@ -112,17 +126,8 @@ class KafkaRDD[ addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). build() val resp = consumer.fetch(req) - if (resp.hasError) { - val err = resp.errorCode(part.topic, part.partition) - if (err == ErrorMapping.LeaderNotAvailableCode || - err == ErrorMapping.NotLeaderForPartitionCode) { - log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, " + - s" sleeping for ${kc.config.refreshLeaderBackoffMs}ms") - Thread.sleep(kc.config.refreshLeaderBackoffMs) - } - // Let normal rdd retry sort out reconnect attempts - throw ErrorMapping.exceptionFor(err) - } + handleErr(resp) + // kafka may return a batch that starts before the requested offset iter = resp.messageSet(part.topic, part.partition) .iterator .dropWhile(_.offset < requestOffset) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 7a33eeb7931aa..0234da1b4a2a6 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -77,7 +77,7 @@ class DeterministicKafkaInputDStream[ @tailrec private def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, Long] = { - val o = kc.getLatestLeaderOffsets(currentOffsets.keys.toSet) + val o = kc.getLatestLeaderOffsets(currentOffsets.keySet) // Either.fold would confuse @tailrec, do it manually if (o.isLeft) { val err = o.left.get.toString @@ -96,8 +96,7 @@ class DeterministicKafkaInputDStream[ private def clamp(leaderOffsets: Map[TopicAndPartition, Long]): Map[TopicAndPartition, Long] = { maxMessagesPerPartition.map { mmp => leaderOffsets.map { kv => - val (k, v) = kv - k -> Math.min(currentOffsets(k) + mmp, v) + kv._1 -> Math.min(currentOffsets(kv._1) + mmp, kv._2) } }.getOrElse(leaderOffsets) } From e93eb72c9d35812e6eef0bd4fd69f0da5cba6764 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 9 Jan 2015 17:29:28 -0600 Subject: [PATCH 23/42] [SPARK-4964] refactor to add preferredLocations. depends on SPARK-4014 --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 18 +-- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 113 ++++++++++++++---- .../DeterministicKafkaInputDStream.scala | 14 ++- .../spark/rdd/kafka/KafkaRDDSuite.scala | 7 +- 4 files changed, 114 insertions(+), 38 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 12e8d36eff78a..55b75e3e98721 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -32,7 +32,7 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { - import KafkaCluster.Err + import KafkaCluster.{Err, LeaderOffset} val seedBrokers: Array[(String, Int)] = kafkaParams.get("metadata.broker.list") @@ -131,18 +131,18 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def getLatestLeaderOffsets( topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, Long]] = + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) def getEarliestLeaderOffsets( topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, Long]] = + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.EarliestTime) def getLeaderOffsets( topicAndPartitions: Set[TopicAndPartition], before: Long - ): Either[Err, Map[TopicAndPartition, Long]] = + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = getLeaderOffsets(topicAndPartitions, before, 1).right.map { r => r.map { kv => // mapValues isnt serializable, see SI-7005 @@ -159,11 +159,11 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { topicAndPartitions: Set[TopicAndPartition], before: Long, maxNumOffsets: Int - ): Either[Err, Map[TopicAndPartition, Seq[Long]]] = { + ): Either[Err, Map[TopicAndPartition, Seq[LeaderOffset]]] = { findLeaders(topicAndPartitions).right.flatMap { tpToLeader => val leaderToTp: Map[(String, Int), Seq[TopicAndPartition]] = flip(tpToLeader) val leaders = leaderToTp.keys - var result = Map[TopicAndPartition, Seq[Long]]() + var result = Map[TopicAndPartition, Seq[LeaderOffset]]() val errs = new Err withBrokers(leaders, errs) { consumer => val needed: Seq[TopicAndPartition] = leaderToTp((consumer.host, consumer.port)) @@ -178,7 +178,9 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { respMap.get(tp).foreach { errAndOffsets => if (errAndOffsets.error == ErrorMapping.NoError) { if (errAndOffsets.offsets.nonEmpty) { - result += tp -> errAndOffsets.offsets + result += tp -> errAndOffsets.offsets.map { off => + LeaderOffset(consumer.host, consumer.port, off) + } } else { errs.append(new Exception( s"Empty offsets for ${tp}, is ${before} before log beginning?")) @@ -297,6 +299,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { object KafkaCluster { type Err = ArrayBuffer[Throwable] + case class LeaderOffset(host: String, port: Int, offset: Long) + /** Make a consumer config without requiring group.id or zookeeper.connect, * since communicating with brokers also needs common settings such as timeout */ diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 50528690ae00f..a23dc3e8fd7f4 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -31,16 +31,24 @@ import kafka.message.{MessageAndMetadata, MessageAndOffset} import kafka.serializer.Decoder import kafka.utils.VerifiableProperties + case class KafkaRDDPartition( override val index: Int, + /** kafka topic name */ topic: String, + /** kafka partition id */ partition: Int, + /** inclusive starting offset */ fromOffset: Long, - untilOffset: Long + /** exclusive ending offset */ + untilOffset: Long, + /** preferred kafka host, i.e. the leader at the time the rdd was created */ + host: String, + /** preferred kafka host's port */ + port: Int ) extends Partition /** A batch-oriented interface for consuming from Kafka. - * Each given Kafka topic/partition corresponds to an RDD partition. * Starting and ending offsets are specified in advance, * so that you can control exactly-once semantics. * For an easy interface to Kafka-managed offsets, @@ -49,10 +57,8 @@ case class KafkaRDDPartition( * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) - * starting point of the batch - * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) - * ending point of the batch + * @param rddPartitions Each RDD partition corresponds to a + * range of offsets for a given Kafka topic/partition * @param messageHandler function for translating each message into the desired type */ class KafkaRDD[ @@ -63,20 +69,31 @@ class KafkaRDD[ R: ClassTag]( sc: SparkContext, val kafkaParams: Map[String, String], - val fromOffsets: Map[TopicAndPartition, Long], - val untilOffsets: Map[TopicAndPartition, Long], + val rddPartitions: Traversable[KafkaRDDPartition], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging { - assert(fromOffsets.keys == untilOffsets.keys, - "Must provide both from and until offsets for each topic/partition") + /** per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch */ + def fromOffsets: Map[TopicAndPartition, Long] = + rddPartitions.map { kr => + TopicAndPartition(kr.topic, kr.partition) -> kr.fromOffset + }.toMap + + /** per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch */ + def untilOffsets: Map[TopicAndPartition, Long] = + rddPartitions.map { kr => + TopicAndPartition(kr.topic, kr.partition) -> kr.untilOffset + }.toMap + + override def getPartitions: Array[Partition] = rddPartitions.toArray - override def getPartitions: Array[Partition] = fromOffsets.zipWithIndex.map { kvi => - val ((tp, from), index) = kvi - new KafkaRDDPartition(index, tp.topic, tp.partition, from, untilOffsets(tp)) - }.toArray + override def getPreferredLocations(thePart: Partition): Seq[String] = { + val part = thePart.asInstanceOf[KafkaRDDPartition] + // TODO is additional hostname resolution necessary here + Seq(part.host) + } - override def compute(thePart: Partition, context: TaskContext) = { + override def compute(thePart: Partition, context: TaskContext): Iterator[R] = { val part = thePart.asInstanceOf[KafkaRDDPartition] if (part.fromOffset >= part.untilOffset) { log.warn("Beginning offset is same or after ending offset " + @@ -86,25 +103,37 @@ class KafkaRDD[ new NextIterator[R] { context.addTaskCompletionListener{ context => closeIfNeeded() } - val kc = new KafkaCluster(kafkaParams) log.info(s"Computing topic ${part.topic}, partition ${part.partition} " + s"offsets ${part.fromOffset} -> ${part.untilOffset}") + + val kc = new KafkaCluster(kafkaParams) val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) .asInstanceOf[Decoder[K]] val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) .newInstance(kc.config.props) .asInstanceOf[Decoder[V]] - val consumer: SimpleConsumer = kc.connectLeader(part.topic, part.partition).fold( - errs => throw new Exception( - s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + - errs.mkString("\n")), - consumer => consumer - ) + val consumer = connectLeader var requestOffset = part.fromOffset var iter: Iterator[MessageAndOffset] = null - def handleErr(resp: FetchResponse) { + // TODO broken until SPARK-4014 is resolved and attemptId / attemptNumber is meaningful. + // The idea is to use the provided preferred host, except on task retry atttempts, + // to minimize number of kafka metadata requests + private def connectLeader: SimpleConsumer = { + if (context.attemptId > 0) { + kc.connectLeader(part.topic, part.partition).fold( + errs => throw new Exception( + s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + + errs.mkString("\n")), + consumer => consumer + ) + } else { + kc.connect(part.host, part.port) + } + } + + private def handleErr(resp: FetchResponse) { if (resp.hasError) { val err = resp.errorCode(part.topic, part.partition) if (err == ErrorMapping.LeaderNotAvailableCode || @@ -160,3 +189,41 @@ class KafkaRDD[ } } + +object KafkaRDD { + import KafkaCluster.LeaderOffset + + /** + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the batch + * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) + * ending point of the batch + * @param messageHandler function for translating each message into the desired type + */ + def apply[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag]( + sc: SparkContext, + kafkaParams: Map[String, String], + fromOffsets: Map[TopicAndPartition, Long], + untilOffsets: Map[TopicAndPartition, LeaderOffset], + messageHandler: MessageAndMetadata[K, V] => R + ): KafkaRDD[K, V, U, T, R] = { + assert(fromOffsets.keys == untilOffsets.keys, + "Must provide both from and until offsets for each topic/partition") + + val partitions = fromOffsets.zipWithIndex.map { case ((tp, from), index) => + val lo = untilOffsets(tp) + new KafkaRDDPartition(index, tp.topic, tp.partition, from, lo.offset, lo.host, lo.port) + } + + new KafkaRDD[K, V, U, T, R](sc, kafkaParams, partitions, messageHandler) + } +} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 0234da1b4a2a6..00353ba065fa0 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -27,6 +27,7 @@ import kafka.serializer.Decoder import org.apache.spark.Logging import org.apache.spark.rdd.RDD import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD} +import org.apache.spark.rdd.kafka.KafkaCluster.LeaderOffset import org.apache.spark.streaming.{StreamingContext, Time} import org.apache.spark.streaming.dstream._ @@ -76,7 +77,7 @@ class DeterministicKafkaInputDStream[ private var currentOffsets = fromOffsets @tailrec - private def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, Long] = { + private def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, LeaderOffset] = { val o = kc.getLatestLeaderOffsets(currentOffsets.keySet) // Either.fold would confuse @tailrec, do it manually if (o.isLeft) { @@ -93,20 +94,21 @@ class DeterministicKafkaInputDStream[ } } - private def clamp(leaderOffsets: Map[TopicAndPartition, Long]): Map[TopicAndPartition, Long] = { + private def clamp( + leaderOffsets: Map[TopicAndPartition, LeaderOffset]): Map[TopicAndPartition, LeaderOffset] = { maxMessagesPerPartition.map { mmp => - leaderOffsets.map { kv => - kv._1 -> Math.min(currentOffsets(kv._1) + mmp, kv._2) + leaderOffsets.map { case (tp, lo) => + tp -> lo.copy(offset = Math.min(currentOffsets(tp) + mmp, lo.offset)) } }.getOrElse(leaderOffsets) } override def compute(validTime: Time): Option[KafkaRDD[K, V, U, T, R]] = { val untilOffsets = clamp(latestLeaderOffsets(maxRetries)) - val rdd = new KafkaRDD[K, V, U, T, R]( + val rdd = KafkaRDD[K, V, U, T, R]( context.sparkContext, kafkaParams, currentOffsets, untilOffsets, messageHandler) - currentOffsets = untilOffsets + currentOffsets = untilOffsets.map(kv => kv._1 -> kv._2.offset) Some(rdd) } diff --git a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala index f8a603c9fa965..955cbad603adf 100644 --- a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala @@ -77,10 +77,13 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { for { topicPartitions <- kc.getPartitions(topics).right.toOption from <- kc.getConsumerOffsets(groupId, topicPartitions).right.toOption.orElse( - kc.getEarliestLeaderOffsets(topicPartitions).right.toOption) + kc.getEarliestLeaderOffsets(topicPartitions).right.toOption.map { offs => + offs.map(kv => kv._1 -> kv._2.offset) + } + ) until <- kc.getLatestLeaderOffsets(topicPartitions).right.toOption } yield { - new KafkaRDD[String, String, StringDecoder, StringDecoder, String]( + KafkaRDD[String, String, StringDecoder, StringDecoder, String]( sc, kc.kafkaParams, from, until, mmd => s"${mmd.offset} ${mmd.message}") } } From e86317ba74aded99c496d9043f7e77150b854480 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 9 Jan 2015 22:42:27 -0600 Subject: [PATCH 24/42] [SPARK-4964] try seed brokers in random order to spread metadata requests --- .../scala/org/apache/spark/rdd/kafka/KafkaCluster.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 55b75e3e98721..9c5699e075552 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -18,6 +18,7 @@ package org.apache.spark.rdd.kafka import scala.util.control.NonFatal +import scala.util.Random import scala.collection.mutable.ArrayBuffer import java.util.Properties import kafka.api._ @@ -67,7 +68,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) val errs = new Err - withBrokers(seedBrokers, errs) { consumer => + withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) resp.topicsMetadata.find(_.topic == topic).flatMap { t => t.partitionsMetadata.find(_.partitionId == partition) @@ -120,7 +121,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) val errs = new Err - withBrokers(seedBrokers, errs) { consumer => + withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) // error codes here indicate missing / just created topic, // repeating on a different broker wont be useful @@ -218,7 +219,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) val errs = new Err - withBrokers(seedBrokers, errs) { consumer => + withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) @@ -257,7 +258,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = OffsetCommitRequest(groupId, metadata) val errs = new Err val topicAndPartitions = metadata.keySet - withBrokers(seedBrokers, errs) { consumer => + withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp = consumer.commitOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) From 0458e4ebdfa04e258a9333c4f8259a77db7f51d2 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Sat, 10 Jan 2015 00:48:11 -0600 Subject: [PATCH 25/42] [SPARK-4964] recovery of generated rdds from checkpoint --- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 37 +-------- .../spark/rdd/kafka/KafkaRDDPartition.scala | 80 +++++++++++++++++++ .../DeterministicKafkaInputDStream.scala | 31 ++++++- .../spark/rdd/kafka/KafkaRDDSuite.scala | 5 +- 4 files changed, 118 insertions(+), 35 deletions(-) create mode 100644 external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index a23dc3e8fd7f4..31601538435a6 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -31,23 +31,6 @@ import kafka.message.{MessageAndMetadata, MessageAndOffset} import kafka.serializer.Decoder import kafka.utils.VerifiableProperties - -case class KafkaRDDPartition( - override val index: Int, - /** kafka topic name */ - topic: String, - /** kafka partition id */ - partition: Int, - /** inclusive starting offset */ - fromOffset: Long, - /** exclusive ending offset */ - untilOffset: Long, - /** preferred kafka host, i.e. the leader at the time the rdd was created */ - host: String, - /** preferred kafka host's port */ - port: Int -) extends Partition - /** A batch-oriented interface for consuming from Kafka. * Starting and ending offsets are specified in advance, * so that you can control exactly-once semantics. @@ -57,7 +40,7 @@ case class KafkaRDDPartition( * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param rddPartitions Each RDD partition corresponds to a + * @param batch Each KafkaRDDPartition in the batch corresponds to a * range of offsets for a given Kafka topic/partition * @param messageHandler function for translating each message into the desired type */ @@ -69,23 +52,11 @@ class KafkaRDD[ R: ClassTag]( sc: SparkContext, val kafkaParams: Map[String, String], - val rddPartitions: Traversable[KafkaRDDPartition], + val batch: Array[KafkaRDDPartition], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging { - /** per-topic/partition Kafka offsets defining the (inclusive) starting point of the batch */ - def fromOffsets: Map[TopicAndPartition, Long] = - rddPartitions.map { kr => - TopicAndPartition(kr.topic, kr.partition) -> kr.fromOffset - }.toMap - - /** per-topic/partition Kafka offsets defining the (exclusive) ending point of the batch */ - def untilOffsets: Map[TopicAndPartition, Long] = - rddPartitions.map { kr => - TopicAndPartition(kr.topic, kr.partition) -> kr.untilOffset - }.toMap - - override def getPartitions: Array[Partition] = rddPartitions.toArray + override def getPartitions: Array[Partition] = batch.asInstanceOf[Array[Partition]] override def getPreferredLocations(thePart: Partition): Seq[String] = { val part = thePart.asInstanceOf[KafkaRDDPartition] @@ -222,7 +193,7 @@ object KafkaRDD { val partitions = fromOffsets.zipWithIndex.map { case ((tp, from), index) => val lo = untilOffsets(tp) new KafkaRDDPartition(index, tp.topic, tp.partition, from, lo.offset, lo.host, lo.port) - } + }.toArray new KafkaRDD[K, V, U, T, R](sc, kafkaParams, partitions, messageHandler) } diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala new file mode 100644 index 0000000000000..946ad5f4a405d --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.rdd.kafka + +import org.apache.spark.Partition + +/** @param topic kafka topic name + * @param partition kafka partition id + * @param fromOffset inclusive starting offset + * @param untilOffset exclusive ending offset + * @param host preferred kafka host, i.e. the leader at the time the rdd was created + * @param port preferred kafka host's port + */ +class KafkaRDDPartition( + override val index: Int, + val topic: String, + val partition: Int, + val fromOffset: Long, + val untilOffset: Long, + val host: String, + val port: Int +) extends Partition { + def toTuple: (Int, String, Int, Long, Long, String, Int) = ( + index, + topic, + partition, + fromOffset, + untilOffset, + host, + port + ) + +} + +object KafkaRDDPartition { + def apply( + index: Int, + topic: String, + partition: Int, + fromOffset: Long, + untilOffset: Long, + host: String, + port: Int + ): KafkaRDDPartition = new KafkaRDDPartition( + index, + topic, + partition, + fromOffset, + untilOffset, + host, + port + ) + + def apply(tuple: (Int, String, Int, Long, Long, String, Int)): KafkaRDDPartition = { + new KafkaRDDPartition( + tuple._1, + tuple._2, + tuple._3, + tuple._4, + tuple._5, + tuple._6, + tuple._7 + ) + } +} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 00353ba065fa0..537bcadd7e610 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -17,7 +17,9 @@ package org.apache.spark.streaming.kafka + import scala.annotation.tailrec +import scala.collection.mutable import scala.reflect.{classTag, ClassTag} import kafka.common.TopicAndPartition @@ -26,7 +28,7 @@ import kafka.serializer.Decoder import org.apache.spark.Logging import org.apache.spark.rdd.RDD -import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD} +import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition} import org.apache.spark.rdd.kafka.KafkaCluster.LeaderOffset import org.apache.spark.streaming.{StreamingContext, Time} import org.apache.spark.streaming.dstream._ @@ -62,6 +64,8 @@ class DeterministicKafkaInputDStream[ maxRetries: Int = 1 ) extends InputDStream[R](ssc_) with Logging { + protected[streaming] override val checkpointData = new DeterministicKafkaInputDStreamCheckpointData + private val kc = new KafkaCluster(kafkaParams) private val maxMessagesPerPartition: Option[Long] = { @@ -117,4 +121,29 @@ class DeterministicKafkaInputDStream[ def stop(): Unit = { } + + private[streaming] + class DeterministicKafkaInputDStreamCheckpointData extends DStreamCheckpointData(this) { + def batchForTime = data.asInstanceOf[mutable.HashMap[ + Time, Array[(Int, String, Int, Long, Long, String, Int)]]] + + override def update(time: Time) { + batchForTime.clear() + generatedRDDs.foreach { kv => + val a = kv._2.asInstanceOf[KafkaRDD[K, V, U, T, R]].batch.map(_.toTuple).toArray + batchForTime += kv._1 -> a + } + } + + override def cleanup(time: Time) { } + + override def restore() { + batchForTime.toSeq.sortBy(_._1)(Time.ordering).foreach { case (t, b) => + logInfo(s"Restoring KafkaRDD for time $t ${b.mkString("[", ", ", "]")}") + generatedRDDs += t -> new KafkaRDD[K, V, U, T, R]( + context.sparkContext, kafkaParams, b.map(KafkaRDDPartition(_)), messageHandler) + } + } + } + } diff --git a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala index 955cbad603adf..63581c252390f 100644 --- a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.rdd.kafka import scala.util.Random import kafka.serializer.StringDecoder +import kafka.common.TopicAndPartition import org.scalatest.BeforeAndAfter import org.apache.spark._ @@ -57,7 +58,9 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { assert(rdd.isDefined) assert(rdd.get.count === sent.values.sum) - kc.setConsumerOffsets(kafkaParams("group.id"), rdd.get.untilOffsets) + kc.setConsumerOffsets( + kafkaParams("group.id"), + rdd.get.batch.map(kp => TopicAndPartition(kp.topic, kp.partition) -> kp.untilOffset).toMap) val rdd2 = getRdd(kc, Set(topic)) val sent2 = Map("d" -> 1) From c1bd6d9d78802607e1c23b0dcfe98ddfc0b71c08 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 14 Jan 2015 14:07:18 -0600 Subject: [PATCH 26/42] [SPARK-4964] use newly available attemptNumber for correct retry behavior --- .../src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 31601538435a6..59efd1ac84da7 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -88,11 +88,10 @@ class KafkaRDD[ var requestOffset = part.fromOffset var iter: Iterator[MessageAndOffset] = null - // TODO broken until SPARK-4014 is resolved and attemptId / attemptNumber is meaningful. // The idea is to use the provided preferred host, except on task retry atttempts, // to minimize number of kafka metadata requests private def connectLeader: SimpleConsumer = { - if (context.attemptId > 0) { + if (context.attemptNumber > 0) { kc.connectLeader(part.topic, part.partition).fold( errs => throw new Exception( s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + From d4a7cf7ae3360e73572124567f85b775566ae05e Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 14 Jan 2015 14:32:22 -0600 Subject: [PATCH 27/42] [SPARK-4964] allow for use cases that need to override compute for custom kafka dstreams --- .../kafka/DeterministicKafkaInputDStream.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 537bcadd7e610..d5c933f139b58 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -66,9 +66,9 @@ class DeterministicKafkaInputDStream[ protected[streaming] override val checkpointData = new DeterministicKafkaInputDStreamCheckpointData - private val kc = new KafkaCluster(kafkaParams) + protected val kc = new KafkaCluster(kafkaParams) - private val maxMessagesPerPartition: Option[Long] = { + protected val maxMessagesPerPartition: Option[Long] = { val ratePerSec = context.sparkContext.getConf.getInt("spark.streaming.receiver.maxRate", 0) if (ratePerSec > 0) { val secsPerBatch = context.graph.batchDuration.milliseconds.toDouble / 1000 @@ -78,10 +78,10 @@ class DeterministicKafkaInputDStream[ } } - private var currentOffsets = fromOffsets + protected var currentOffsets = fromOffsets @tailrec - private def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, LeaderOffset] = { + protected final def latestLeaderOffsets(retries: Int): Map[TopicAndPartition, LeaderOffset] = { val o = kc.getLatestLeaderOffsets(currentOffsets.keySet) // Either.fold would confuse @tailrec, do it manually if (o.isLeft) { @@ -98,7 +98,7 @@ class DeterministicKafkaInputDStream[ } } - private def clamp( + protected def clamp( leaderOffsets: Map[TopicAndPartition, LeaderOffset]): Map[TopicAndPartition, LeaderOffset] = { maxMessagesPerPartition.map { mmp => leaderOffsets.map { case (tp, lo) => From bb80bbea59d11af73430961c925c80dbb942d056 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 26 Jan 2015 16:07:44 -0600 Subject: [PATCH 28/42] [SPARK-4964] scalastyle line length --- .../spark/streaming/kafka/DeterministicKafkaInputDStream.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index d5c933f139b58..699b948aeba9c 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -64,7 +64,8 @@ class DeterministicKafkaInputDStream[ maxRetries: Int = 1 ) extends InputDStream[R](ssc_) with Logging { - protected[streaming] override val checkpointData = new DeterministicKafkaInputDStreamCheckpointData + protected[streaming] override val checkpointData = + new DeterministicKafkaInputDStreamCheckpointData protected val kc = new KafkaCluster(kafkaParams) From 2e67117d1d50d3a043c143670dd47594548c10be Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 27 Jan 2015 22:55:50 -0600 Subject: [PATCH 29/42] [SPARK-4964] one potential way of hiding most of the implementation, while still allowing access to offsets (but not subclassing) --- .../apache/spark/rdd/kafka/KafkaCluster.scala | 2 + .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 14 +- .../spark/rdd/kafka/KafkaRDDPartition.scala | 16 ++- .../apache/spark/rdd/kafka/OffsetRange.scala | 70 ++++++++++ .../DeterministicKafkaInputDStream.scala | 3 +- .../spark/streaming/kafka/KafkaUtils.scala | 120 +++++++++++++++++- 6 files changed, 210 insertions(+), 15 deletions(-) create mode 100644 external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala index 9c5699e075552..3be8c2eeb65be 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala @@ -32,6 +32,7 @@ import kafka.consumer.{ConsumerConfig, SimpleConsumer} * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form */ +private[spark] class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { import KafkaCluster.{Err, LeaderOffset} @@ -297,6 +298,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } +private[spark] object KafkaCluster { type Err = ArrayBuffer[Throwable] diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index 59efd1ac84da7..ca84d807e0bf6 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -34,8 +34,6 @@ import kafka.utils.VerifiableProperties /** A batch-oriented interface for consuming from Kafka. * Starting and ending offsets are specified in advance, * so that you can control exactly-once semantics. - * For an easy interface to Kafka-managed offsets, - * see {@link org.apache.spark.rdd.kafka.KafkaCluster} * @param kafkaParams Kafka * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), @@ -44,17 +42,20 @@ import kafka.utils.VerifiableProperties * range of offsets for a given Kafka topic/partition * @param messageHandler function for translating each message into the desired type */ +private[spark] class KafkaRDD[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, T <: Decoder[_]: ClassTag, - R: ClassTag]( + R: ClassTag] private[spark] ( sc: SparkContext, - val kafkaParams: Map[String, String], - val batch: Array[KafkaRDDPartition], + kafkaParams: Map[String, String], + private[spark] val batch: Array[KafkaRDDPartition], messageHandler: MessageAndMetadata[K, V] => R - ) extends RDD[R](sc, Nil) with Logging { + ) extends RDD[R](sc, Nil) with Logging with HasOffsetRanges { + + def offsetRanges: Array[OffsetRange] = batch.asInstanceOf[Array[OffsetRange]] override def getPartitions: Array[Partition] = batch.asInstanceOf[Array[Partition]] @@ -160,6 +161,7 @@ class KafkaRDD[ } +private[spark] object KafkaRDD { import KafkaCluster.LeaderOffset diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala index 946ad5f4a405d..791f8e72ce66d 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala @@ -26,15 +26,16 @@ import org.apache.spark.Partition * @param host preferred kafka host, i.e. the leader at the time the rdd was created * @param port preferred kafka host's port */ +private[spark] class KafkaRDDPartition( override val index: Int, - val topic: String, - val partition: Int, - val fromOffset: Long, - val untilOffset: Long, - val host: String, - val port: Int -) extends Partition { + override val topic: String, + override val partition: Int, + override val fromOffset: Long, + override val untilOffset: Long, + override val host: String, + override val port: Int +) extends Partition with OffsetRange { def toTuple: (Int, String, Int, Long, Long, String, Int) = ( index, topic, @@ -47,6 +48,7 @@ class KafkaRDDPartition( } +private[spark] object KafkaRDDPartition { def apply( index: Int, diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala new file mode 100644 index 0000000000000..58d0b703c2f69 --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.rdd.kafka + +/** Represents a range of offsets from a single Kafka TopicAndPartition */ +trait OffsetRange { + /** kafka topic name */ + def topic: String + + /** kafka partition id */ + def partition: Int + + /** inclusive starting offset */ + def fromOffset: Long + + /** exclusive ending offset */ + def untilOffset: Long + + /** preferred kafka host, i.e. the leader at the time of creation */ + def host: String + + /** preferred kafka host's port */ + def port: Int +} + +/** Something that has a collection of OffsetRanges */ +trait HasOffsetRanges { + def offsetRanges: Array[OffsetRange] +} + +private class OffsetRangeImpl( + override val topic: String, + override val partition: Int, + override val fromOffset: Long, + override val untilOffset: Long, + override val host: String, + override val port: Int +) extends OffsetRange + +object OffsetRange { + def apply( + topic: String, + partition: Int, + fromOffset: Long, + untilOffset: Long, + host: String, + port: Int): OffsetRange = + new OffsetRangeImpl( + topic, + partition, + fromOffset, + untilOffset, + host, + port) +} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 699b948aeba9c..c8b89498af6e4 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -51,6 +51,7 @@ import org.apache.spark.streaming.dstream._ * @param messageHandler function for translating each message into the desired type * @param maxRetries maximum number of times in a row to retry getting leaders' offsets */ +private[streaming] class DeterministicKafkaInputDStream[ K: ClassTag, V: ClassTag, @@ -61,7 +62,7 @@ class DeterministicKafkaInputDStream[ val kafkaParams: Map[String, String], val fromOffsets: Map[TopicAndPartition, Long], messageHandler: MessageAndMetadata[K, V] => R, - maxRetries: Int = 1 + maxRetries: Int ) extends InputDStream[R](ssc_) with Logging { protected[streaming] override val checkpointData = diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index df725f0c65a64..a4685c8cb3540 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -23,12 +23,18 @@ import java.util.{Map => JMap} import scala.reflect.ClassTag import scala.collection.JavaConversions._ +import kafka.common.TopicAndPartition +import kafka.message.MessageAndMetadata import kafka.serializer.{Decoder, StringDecoder} + +import org.apache.spark.SparkContext +import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.StreamingContext import org.apache.spark.streaming.api.java.{JavaPairReceiverInputDStream, JavaStreamingContext} -import org.apache.spark.streaming.dstream.ReceiverInputDStream +import org.apache.spark.streaming.dstream.{InputDStream, ReceiverInputDStream} +import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition, OffsetRange} object KafkaUtils { /** @@ -144,4 +150,116 @@ object KafkaUtils { createStream[K, V, U, T]( jssc.ssc, kafkaParams.toMap, Map(topics.mapValues(_.intValue()).toSeq: _*), storageLevel) } + + /** A batch-oriented interface for consuming from Kafka. + * Starting and ending offsets are specified in advance, + * so that you can control exactly-once semantics. + * @param sc SparkContext object + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param batch Each OffsetRange in the batch corresponds to a + * range of offsets for a given Kafka topic/partition + * @param messageHandler function for translating each message into the desired type + */ + def createRDD[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag] ( + sc: SparkContext, + kafkaParams: Map[String, String], + batch: Array[OffsetRange], + messageHandler: MessageAndMetadata[K, V] => R + ): RDD[R] = { + val parts = batch.zipWithIndex.map { case (o, i) => + new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, o.host, o.port) + }.toArray + new KafkaRDD[K, V, U, T, R](sc, kafkaParams, parts, messageHandler) + } + + /** + * This DOES NOT guarantee that side-effects of an action will see each message exactly once. + * If you need that guarantee, get the offsets from this stream and store them with your output. + * Nor does this store offsets in Kafka / Zookeeper. + * If checkpointed, it will store offset ranges in the checkpoint, such that each message + * will be transformed effectively exactly once even after failure, + * provided you have sufficient Kafka log retention. + * + * @param ssc StreamingContext object + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param messageHandler function for translating each message into the desired type + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the stream + * @param maxRetries maximum number of times in a row to retry getting leaders' offsets + */ + def createExactlyOnceStream[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag] ( + ssc: StreamingContext, + kafkaParams: Map[String, String], + fromOffsets: Map[TopicAndPartition, Long], + messageHandler: MessageAndMetadata[K, V] => R, + maxRetries: Int + ): InputDStream[R] = { + new DeterministicKafkaInputDStream[K, V, U, T, R]( + ssc, kafkaParams, fromOffsets, messageHandler, maxRetries) + } + + /** + * This DOES NOT guarantee that side-effects of an action will see each message exactly once. + * If you need that guarantee, get the offsets from this stream and store them with your output. + * Nor does this store offsets in Kafka / Zookeeper. + * If checkpointed, it will store offset ranges in the checkpoint, such that each message + * will be transformed effectively exactly once even after failure, + * provided you have sufficient Kafka log retention. + * + * @param ssc StreamingContext object + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * If starting without a checkpoint, "auto.offset.reset" may be set to "largest" or "smallest" + * to determine where the stream starts (defaults to "largest") + * @param topics names of the topics to consume + */ + def createExactlyOnceStream[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag] ( + ssc: StreamingContext, + kafkaParams: Map[String, String], + topics: Set[String] + ): InputDStream[(K, V)] = { + val messageHandler = (mmd: MessageAndMetadata[K, V]) => (mmd.key, mmd.message) + val kc = new KafkaCluster(kafkaParams) + val reset = kafkaParams.get("auto.offset.reset").map(_.toLowerCase) + + (for { + topicPartitions <- kc.getPartitions(topics).right + leaderOffsets <- (if (reset == Some("smallest")) { + kc.getEarliestLeaderOffsets(topicPartitions) + } else { + kc.getLatestLeaderOffsets(topicPartitions) + }).right + } yield { + val fromOffsets = leaderOffsets.map { case (tp, lo) => + (tp, lo.offset) + } + new DeterministicKafkaInputDStream[K, V, U, T, (K, V)]( + ssc, kafkaParams, fromOffsets, messageHandler, 1) + }).fold( + errs => throw new Exception(errs.mkString("\n")), + ok => ok + ) + } } From 99d2eba6d0e2d3d549782079a8c31343a5b2a615 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Thu, 29 Jan 2015 00:07:00 -0600 Subject: [PATCH 30/42] [SPARK-4964] Reduce level of nesting. If beginning is past end, its actually an error (may happen if Kafka topic was deleted and recreated) --- .../org/apache/spark/rdd/kafka/KafkaRDD.scala | 189 ++++++++++-------- 1 file changed, 106 insertions(+), 83 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala index ca84d807e0bf6..484ebaefd08ad 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala @@ -19,7 +19,7 @@ package org.apache.spark.rdd.kafka import scala.reflect.{classTag, ClassTag} -import org.apache.spark.{Logging, Partition, SparkContext, TaskContext} +import org.apache.spark.{Logging, Partition, SparkContext, SparkException, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.util.NextIterator @@ -65,100 +65,123 @@ class KafkaRDD[ Seq(part.host) } + private def assertStartedCleanly(part: KafkaRDDPartition) { + assert(part.fromOffset <= part.untilOffset, + s"Beginning offset ${part.fromOffset} is after the ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition}. " + + "You either provided an invalid fromOffset, or the Kafka topic has been damaged") + } + override def compute(thePart: Partition, context: TaskContext): Iterator[R] = { val part = thePart.asInstanceOf[KafkaRDDPartition] - if (part.fromOffset >= part.untilOffset) { - log.warn("Beginning offset is same or after ending offset " + + assertStartedCleanly(part) + if (part.fromOffset == part.untilOffset) { + log.warn("Beginning offset ${part.fromOffset} is the same as ending offset " + s"skipping ${part.topic} ${part.partition}") Iterator.empty } else { - new NextIterator[R] { - context.addTaskCompletionListener{ context => closeIfNeeded() } - - log.info(s"Computing topic ${part.topic}, partition ${part.partition} " + - s"offsets ${part.fromOffset} -> ${part.untilOffset}") - - val kc = new KafkaCluster(kafkaParams) - val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(kc.config.props) - .asInstanceOf[Decoder[K]] - val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) - .newInstance(kc.config.props) - .asInstanceOf[Decoder[V]] - val consumer = connectLeader - var requestOffset = part.fromOffset - var iter: Iterator[MessageAndOffset] = null - - // The idea is to use the provided preferred host, except on task retry atttempts, - // to minimize number of kafka metadata requests - private def connectLeader: SimpleConsumer = { - if (context.attemptNumber > 0) { - kc.connectLeader(part.topic, part.partition).fold( - errs => throw new Exception( - s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + - errs.mkString("\n")), - consumer => consumer - ) - } else { - kc.connect(part.host, part.port) - } - } + new KafkaRDDIterator(part, context) + } + } - private def handleErr(resp: FetchResponse) { - if (resp.hasError) { - val err = resp.errorCode(part.topic, part.partition) - if (err == ErrorMapping.LeaderNotAvailableCode || - err == ErrorMapping.NotLeaderForPartitionCode) { - log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, " + - s" sleeping for ${kc.config.refreshLeaderBackoffMs}ms") - Thread.sleep(kc.config.refreshLeaderBackoffMs) - } - // Let normal rdd retry sort out reconnect attempts - throw ErrorMapping.exceptionFor(err) - } + private class KafkaRDDIterator( + part: KafkaRDDPartition, context: TaskContext) extends NextIterator[R] { + + context.addTaskCompletionListener{ context => closeIfNeeded() } + + log.info(s"Computing topic ${part.topic}, partition ${part.partition} " + + s"offsets ${part.fromOffset} -> ${part.untilOffset}") + + val kc = new KafkaCluster(kafkaParams) + val keyDecoder = classTag[U].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(kc.config.props) + .asInstanceOf[Decoder[K]] + val valueDecoder = classTag[T].runtimeClass.getConstructor(classOf[VerifiableProperties]) + .newInstance(kc.config.props) + .asInstanceOf[Decoder[V]] + val consumer = connectLeader + var requestOffset = part.fromOffset + var iter: Iterator[MessageAndOffset] = null + + // The idea is to use the provided preferred host, except on task retry atttempts, + // to minimize number of kafka metadata requests + private def connectLeader: SimpleConsumer = { + if (context.attemptNumber > 0) { + kc.connectLeader(part.topic, part.partition).fold( + errs => throw new SparkException( + s"Couldn't connect to leader for topic ${part.topic} ${part.partition}: " + + errs.mkString("\n")), + consumer => consumer + ) + } else { + kc.connect(part.host, part.port) + } + } + + private def handleFetchErr(resp: FetchResponse) { + if (resp.hasError) { + val err = resp.errorCode(part.topic, part.partition) + if (err == ErrorMapping.LeaderNotAvailableCode || + err == ErrorMapping.NotLeaderForPartitionCode) { + log.error(s"Lost leader for topic ${part.topic} partition ${part.partition}, " + + s" sleeping for ${kc.config.refreshLeaderBackoffMs}ms") + Thread.sleep(kc.config.refreshLeaderBackoffMs) } + // Let normal rdd retry sort out reconnect attempts + throw ErrorMapping.exceptionFor(err) + } + } - override def close() = consumer.close() - - override def getNext: R = { - if (iter == null || !iter.hasNext) { - val req = new FetchRequestBuilder(). - addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). - build() - val resp = consumer.fetch(req) - handleErr(resp) - // kafka may return a batch that starts before the requested offset - iter = resp.messageSet(part.topic, part.partition) - .iterator - .dropWhile(_.offset < requestOffset) - } - if (!iter.hasNext) { - assert(requestOffset == part.untilOffset, - s"ran out of messages before reaching ending offset ${part.untilOffset} " + - s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + - " This should not happen, and indicates that messages may have been lost") - finished = true - null.asInstanceOf[R] - } else { - val item = iter.next - if (item.offset >= part.untilOffset) { - assert(item.offset == part.untilOffset, - s"got ${item.offset} > ending offset ${part.untilOffset} " + - s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + - " This should not happen, and indicates a message may have been skipped") - finished = true - null.asInstanceOf[R] - } else { - requestOffset = item.nextOffset - messageHandler(new MessageAndMetadata( - part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) - } - } + private def assertFinishedEmpty(requestOffset: Long) { + assert(requestOffset == part.untilOffset, + s"ran out of messages before reaching ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates that messages may have been lost") + } + + private def assertFinishedWithoutOvershoot(itemOffset: Long) { + assert(itemOffset == part.untilOffset, + s"got ${itemOffset} > ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates a message may have been skipped") + } + + private def fetchBatch: Iterator[MessageAndOffset] = { + val req = new FetchRequestBuilder(). + addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). + build() + val resp = consumer.fetch(req) + handleFetchErr(resp) + // kafka may return a batch that starts before the requested offset + resp.messageSet(part.topic, part.partition) + .iterator + .dropWhile(_.offset < requestOffset) + } + + override def close() = consumer.close() + + override def getNext: R = { + if (iter == null || !iter.hasNext) { + iter = fetchBatch + } + if (!iter.hasNext) { + assertFinishedEmpty(requestOffset) + finished = true + null.asInstanceOf[R] + } else { + val item = iter.next + if (item.offset >= part.untilOffset) { + assertFinishedWithoutOvershoot(item.offset) + finished = true + null.asInstanceOf[R] + } else { + requestOffset = item.nextOffset + messageHandler(new MessageAndMetadata( + part.topic, part.partition, item.message, item.offset, keyDecoder, valueDecoder)) } } } } - } private[spark] From 80fd6aefde56299ba7651560e52df20b884c474e Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Thu, 29 Jan 2015 00:09:30 -0600 Subject: [PATCH 31/42] [SPARK-4964] Rename createExactlyOnceStream so it isnt over-promising, change doc --- .../spark/streaming/kafka/KafkaUtils.scala | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index a4685c8cb3540..94405bb273163 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -28,13 +28,13 @@ import kafka.message.MessageAndMetadata import kafka.serializer.{Decoder, StringDecoder} -import org.apache.spark.SparkContext +import org.apache.spark.{SparkContext, SparkException} import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.StreamingContext import org.apache.spark.streaming.api.java.{JavaPairReceiverInputDStream, JavaStreamingContext} import org.apache.spark.streaming.dstream.{InputDStream, ReceiverInputDStream} -import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition, OffsetRange} +import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition, OffsetRange, HasOffsetRanges} object KafkaUtils { /** @@ -173,7 +173,7 @@ object KafkaUtils { kafkaParams: Map[String, String], batch: Array[OffsetRange], messageHandler: MessageAndMetadata[K, V] => R - ): RDD[R] = { + ): RDD[R] with HasOffsetRanges = { val parts = batch.zipWithIndex.map { case (o, i) => new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, o.host, o.port) }.toArray @@ -181,12 +181,26 @@ object KafkaUtils { } /** - * This DOES NOT guarantee that side-effects of an action will see each message exactly once. - * If you need that guarantee, get the offsets from this stream and store them with your output. - * Nor does this store offsets in Kafka / Zookeeper. - * If checkpointed, it will store offset ranges in the checkpoint, such that each message - * will be transformed effectively exactly once even after failure, - * provided you have sufficient Kafka log retention. + * Compared to `createStream`, the stream created by this can guarantee that each message + * from Kafka is included in transformations (as opposed to output actions) exactly once, + * even in most failure situations. + * + * Points to note: + * + * Failure Recovery - You must checkpoint this stream, or save offsets yourself and provide them + * as the fromOffsets parameter on restart. + * Kafka must have sufficient log retention to obtain messages after failure. + * + * Getting offsets from the stream - see programming guide + * +. * Zookeeper - This does not use Zookeeper to store offsets. For interop with Kafka monitors + * that depend on Zookeeper, you must store offsets in ZK yourself. + * + * End-to-end semantics - This does not guarantee that any output operation will push each record + * exactly once. To ensure end-to-end exactly-once semantics (that is, receiving exactly once and + * outputting exactly once), you have to either ensure that the output operation is + * idempotent, or transactionally store offsets with the output. See the programming guide for + * more details. * * @param ssc StreamingContext object * @param kafkaParams Kafka @@ -198,7 +212,7 @@ object KafkaUtils { * starting point of the stream * @param maxRetries maximum number of times in a row to retry getting leaders' offsets */ - def createExactlyOnceStream[ + def createNewStream[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, @@ -215,12 +229,23 @@ object KafkaUtils { } /** - * This DOES NOT guarantee that side-effects of an action will see each message exactly once. - * If you need that guarantee, get the offsets from this stream and store them with your output. - * Nor does this store offsets in Kafka / Zookeeper. - * If checkpointed, it will store offset ranges in the checkpoint, such that each message - * will be transformed effectively exactly once even after failure, - * provided you have sufficient Kafka log retention. + * Compared to `createStream`, the stream created by this can guarantee that each message + * from Kafka is included in transformations (as opposed to output actions) exactly once, + * even in most failure situations. + * + * Points to note: + * + * Failure Recovery - You must checkpoint this stream. + * Kafka must have sufficient log retention to obtain messages after failure. + * + * Getting offsets from the stream - see programming guide + * +. * Zookeeper - This does not use Zookeeper to store offsets. For interop with Kafka monitors + * that depend on Zookeeper, you must store offsets in ZK yourself. + * + * End-to-end semantics - This does not guarantee that any output operation will push each record + * exactly once. To ensure end-to-end exactly-once semantics (that is, receiving exactly once and + * outputting exactly once), you have to ensure that the output operation is idempotent. * * @param ssc StreamingContext object * @param kafkaParams Kafka @@ -231,7 +256,7 @@ object KafkaUtils { * to determine where the stream starts (defaults to "largest") * @param topics names of the topics to consume */ - def createExactlyOnceStream[ + def createNewStream[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, @@ -258,7 +283,7 @@ object KafkaUtils { new DeterministicKafkaInputDStream[K, V, U, T, (K, V)]( ssc, kafkaParams, fromOffsets, messageHandler, 1) }).fold( - errs => throw new Exception(errs.mkString("\n")), + errs => throw new SparkException(errs.mkString("\n")), ok => ok ) } From 2b340d86987507ad5712e4d44114d0810351402d Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Thu, 29 Jan 2015 19:10:36 -0600 Subject: [PATCH 32/42] [SPARK-4964] refactor per TD feedback --- .../main/scala/org/apache/spark/rdd/RDD.scala | 14 ---- .../DeterministicKafkaInputDStream.scala | 3 +- .../kafka/KafkaCluster.scala | 3 +- .../{rdd => streaming}/kafka/KafkaRDD.scala | 2 +- .../kafka/KafkaRDDPartition.scala | 6 +- .../spark/streaming/kafka/KafkaUtils.scala | 48 +++++++++++-- .../apache/spark/streaming/kafka/Leader.scala | 68 +++++++++++++++++++ .../kafka/OffsetRange.scala | 35 +++++----- .../kafka/KafkaRDDSuite.scala | 10 ++- 9 files changed, 145 insertions(+), 44 deletions(-) rename external/kafka/src/main/scala/org/apache/spark/{rdd => streaming}/kafka/KafkaCluster.scala (99%) rename external/kafka/src/main/scala/org/apache/spark/{rdd => streaming}/kafka/KafkaRDD.scala (99%) rename external/kafka/src/main/scala/org/apache/spark/{rdd => streaming}/kafka/KafkaRDDPartition.scala (95%) create mode 100644 external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala rename external/kafka/src/main/scala/org/apache/spark/{rdd => streaming}/kafka/OffsetRange.scala (78%) rename external/kafka/src/test/scala/org/apache/spark/{rdd => streaming}/kafka/KafkaRDDSuite.scala (86%) diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index ea4277a433b00..ab7410a1f7f99 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -804,20 +804,6 @@ abstract class RDD[T: ClassTag]( sc.runJob(this, (iter: Iterator[T]) => cleanF(iter)) } - /** - * Applies a function to each partition of this RDD, while tracking the index - * of the original partition. - */ - def foreachPartitionWithIndex( - f: (Int, Iterator[T]) => Unit) { - val func = (index: Int, iter: Iterator[T]) => { - f(index, iter) - Iterator.empty - } - sc.runJob( - mapPartitionsWithIndex(func, true), (iter: Iterator[T]) => ()) - } - /** * Return an array that contains all of the elements in this RDD. */ diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index c8b89498af6e4..85893347c0108 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -28,8 +28,7 @@ import kafka.serializer.Decoder import org.apache.spark.Logging import org.apache.spark.rdd.RDD -import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition} -import org.apache.spark.rdd.kafka.KafkaCluster.LeaderOffset +import org.apache.spark.streaming.kafka.KafkaCluster.LeaderOffset import org.apache.spark.streaming.{StreamingContext, Time} import org.apache.spark.streaming.dstream._ diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala similarity index 99% rename from external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala rename to external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala index 3be8c2eeb65be..1f72cb2a55670 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.rdd.kafka +package org.apache.spark.streaming.kafka import scala.util.control.NonFatal import scala.util.Random @@ -302,6 +302,7 @@ private[spark] object KafkaCluster { type Err = ArrayBuffer[Throwable] + private[spark] case class LeaderOffset(host: String, port: Int, offset: Long) /** Make a consumer config without requiring group.id or zookeeper.connect, diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala similarity index 99% rename from external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala rename to external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala index 484ebaefd08ad..44cf9678a6f04 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.rdd.kafka +package org.apache.spark.streaming.kafka import scala.reflect.{classTag, ClassTag} diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala similarity index 95% rename from external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala rename to external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala index 791f8e72ce66d..bf4c6ebf69556 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/KafkaRDDPartition.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.rdd.kafka +package org.apache.spark.streaming.kafka import org.apache.spark.Partition @@ -33,8 +33,8 @@ class KafkaRDDPartition( override val partition: Int, override val fromOffset: Long, override val untilOffset: Long, - override val host: String, - override val port: Int + val host: String, + val port: Int ) extends Partition with OffsetRange { def toTuple: (Int, String, Int, Long, Long, String, Int) = ( index, diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 94405bb273163..82bc83692cf56 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -34,7 +34,6 @@ import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.StreamingContext import org.apache.spark.streaming.api.java.{JavaPairReceiverInputDStream, JavaStreamingContext} import org.apache.spark.streaming.dstream.{InputDStream, ReceiverInputDStream} -import org.apache.spark.rdd.kafka.{KafkaCluster, KafkaRDD, KafkaRDDPartition, OffsetRange, HasOffsetRanges} object KafkaUtils { /** @@ -161,6 +160,43 @@ object KafkaUtils { * NOT zookeeper servers, specified in host1:port1,host2:port2 form. * @param batch Each OffsetRange in the batch corresponds to a * range of offsets for a given Kafka topic/partition + */ + def createRDD[ + K: ClassTag, + V: ClassTag, + U <: Decoder[_]: ClassTag, + T <: Decoder[_]: ClassTag, + R: ClassTag] ( + sc: SparkContext, + kafkaParams: Map[String, String], + batch: Array[OffsetRange] + ): RDD[(K, V)] with HasOffsetRanges = { + val messageHandler = (mmd: MessageAndMetadata[K, V]) => (mmd.key, mmd.message) + val kc = new KafkaCluster(kafkaParams) + val topics = batch.map(o => TopicAndPartition(o.topic, o.partition)).toSet + val leaderMap = kc.findLeaders(topics).fold( + errs => throw new SparkException(errs.mkString("\n")), + ok => ok + ) + val rddParts = batch.zipWithIndex.map { case (o, i) => + val tp = TopicAndPartition(o.topic, o.partition) + val (host, port) = leaderMap(tp) + new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, host, port) + }.toArray + new KafkaRDD[K, V, U, T, (K, V)](sc, kafkaParams, rddParts, messageHandler) + } + + /** A batch-oriented interface for consuming from Kafka. + * Starting and ending offsets are specified in advance, + * so that you can control exactly-once semantics. + * @param sc SparkContext object + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param batch Each OffsetRange in the batch corresponds to a + * range of offsets for a given Kafka topic/partition + * @param leaders Kafka leaders for each offset range in batch * @param messageHandler function for translating each message into the desired type */ def createRDD[ @@ -172,12 +208,16 @@ object KafkaUtils { sc: SparkContext, kafkaParams: Map[String, String], batch: Array[OffsetRange], + leaders: Array[Leader], messageHandler: MessageAndMetadata[K, V] => R ): RDD[R] with HasOffsetRanges = { - val parts = batch.zipWithIndex.map { case (o, i) => - new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, o.host, o.port) + val leaderMap = leaders.map(l => (l.topic, l.partition) -> (l.host, l.port)).toMap + val rddParts = batch.zipWithIndex.map { case (o, i) => + val (host, port) = leaderMap((o.topic, o.partition)) + new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, host, port) }.toArray - new KafkaRDD[K, V, U, T, R](sc, kafkaParams, parts, messageHandler) + + new KafkaRDD[K, V, U, T, R](sc, kafkaParams, rddParts, messageHandler) } /** diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala new file mode 100644 index 0000000000000..360c359ae4dcd --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import kafka.common.TopicAndPartition + +/** Host info for the leader of a Kafka TopicAndPartition */ + +trait Leader { + /** kafka topic name */ + def topic: String + + /** kafka partition id */ + def partition: Int + + /** kafka hostname */ + def host: String + + /** kafka host's port */ + def port: Int +} + +private class LeaderImpl( + override val topic: String, + override val partition: Int, + override val host: String, + override val port: Int +) extends Leader + +object Leader { + def create( + topic: String, + partition: Int, + host: String, + port: Int): Leader = + new LeaderImpl( + topic, + partition, + host, + port) + + def create( + topicAndPartition: TopicAndPartition, + host: String, + port: Int): Leader = + new LeaderImpl( + topicAndPartition.topic, + topicAndPartition.partition, + host, + port) + +} + diff --git a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala similarity index 78% rename from external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala rename to external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala index 58d0b703c2f69..ea3ef067ea4ac 100644 --- a/external/kafka/src/main/scala/org/apache/spark/rdd/kafka/OffsetRange.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala @@ -15,7 +15,9 @@ * limitations under the License. */ -package org.apache.spark.rdd.kafka +package org.apache.spark.streaming.kafka + +import kafka.common.TopicAndPartition /** Represents a range of offsets from a single Kafka TopicAndPartition */ trait OffsetRange { @@ -30,12 +32,6 @@ trait OffsetRange { /** exclusive ending offset */ def untilOffset: Long - - /** preferred kafka host, i.e. the leader at the time of creation */ - def host: String - - /** preferred kafka host's port */ - def port: Int } /** Something that has a collection of OffsetRanges */ @@ -47,24 +43,29 @@ private class OffsetRangeImpl( override val topic: String, override val partition: Int, override val fromOffset: Long, - override val untilOffset: Long, - override val host: String, - override val port: Int + override val untilOffset: Long ) extends OffsetRange object OffsetRange { - def apply( + def create( topic: String, partition: Int, fromOffset: Long, - untilOffset: Long, - host: String, - port: Int): OffsetRange = + untilOffset: Long): OffsetRange = new OffsetRangeImpl( topic, partition, fromOffset, - untilOffset, - host, - port) + untilOffset) + + def create( + topicAndPartition: TopicAndPartition, + fromOffset: Long, + untilOffset: Long): OffsetRange = + new OffsetRangeImpl( + topicAndPartition.topic, + topicAndPartition.partition, + fromOffset, + untilOffset) + } diff --git a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala similarity index 86% rename from external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala rename to external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala index 63581c252390f..e1ade6fea11e4 100644 --- a/external/kafka/src/test/scala/org/apache/spark/rdd/kafka/KafkaRDDSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.rdd.kafka +package org.apache.spark.streaming.kafka import scala.util.Random @@ -25,7 +25,6 @@ import org.scalatest.BeforeAndAfter import org.apache.spark._ import org.apache.spark.SparkContext._ -import org.apache.spark.streaming.kafka.KafkaStreamSuiteBase class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { var sc: SparkContext = _ @@ -55,6 +54,8 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { val kc = new KafkaCluster(kafkaParams) val rdd = getRdd(kc, Set(topic)) + // this is the "lots of messages" case + // make sure we get all of them assert(rdd.isDefined) assert(rdd.get.count === sent.values.sum) @@ -65,16 +66,21 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { val rdd2 = getRdd(kc, Set(topic)) val sent2 = Map("d" -> 1) produceAndSendMessage(topic, sent2) + // this is the "0 messages" case + // make sure we dont get anything, since messages were sent after rdd was defined assert(rdd2.isDefined) assert(rdd2.get.count === 0) val rdd3 = getRdd(kc, Set(topic)) produceAndSendMessage(topic, Map("extra" -> 22)) + // this is the "exactly 1 message" case + // make sure we get exactly one message, despite there being lots more available assert(rdd3.isDefined) assert(rdd3.get.count === sent2.values.sum) } + // get an rdd from the committed consumer offsets until the latest leader offsets, private def getRdd(kc: KafkaCluster, topics: Set[String]) = { val groupId = kc.kafkaParams("group.id") for { From 9a838c29fa40906c97172d248380f316cdf7183c Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Thu, 29 Jan 2015 23:05:51 -0600 Subject: [PATCH 33/42] [SPARK-4964] code cleanup, add more tests --- .../spark/streaming/kafka/KafkaCluster.scala | 109 ++++++++++-------- .../spark/streaming/kafka/KafkaRDD.scala | 96 ++++++++------- .../spark/streaming/kafka/KafkaUtils.scala | 15 ++- .../streaming/kafka/KafkaClusterSuite.scala | 73 ++++++++++++ .../streaming/kafka/KafkaNewStreamSuite.scala | 92 +++++++++++++++ 5 files changed, 283 insertions(+), 102 deletions(-) create mode 100644 external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaClusterSuite.scala create mode 100644 external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala index 1f72cb2a55670..f0484de35de54 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala @@ -26,12 +26,12 @@ import kafka.common.{ErrorMapping, OffsetMetadataAndError, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} /** - * Convenience methods for interacting with a Kafka cluster. - * @param kafkaParams Kafka - * configuration parameters. - * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), - * NOT zookeeper servers, specified in host1:port1,host2:port2 form - */ + * Convenience methods for interacting with a Kafka cluster. + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form + */ private[spark] class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { import KafkaCluster.{Err, LeaderOffset} @@ -59,11 +59,13 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { new SimpleConsumer(host, port, config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId) - def connect(hostAndPort: (String, Int)): SimpleConsumer = - connect(hostAndPort._1, hostAndPort._2) - def connectLeader(topic: String, partition: Int): Either[Err, SimpleConsumer] = - findLeader(topic, partition).right.map(connect) + findLeader(topic, partition).right.map(hp => connect(hp._1, hp._2)) + + // Metadata api + // scalastyle:off + // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI + // scalastyle:on def findLeader(topic: String, partition: Int): Either[Err, (String, Int)] = { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, @@ -71,10 +73,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val errs = new Err withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) - resp.topicsMetadata.find(_.topic == topic).flatMap { t => - t.partitionsMetadata.find(_.partitionId == partition) - }.foreach { partitionMeta => - partitionMeta.leader.foreach { leader => + resp.topicsMetadata.find(_.topic == topic).flatMap { tm: TopicMetadata => + tm.partitionsMetadata.find(_.partitionId == partition) + }.foreach { pm: PartitionMetadata => + pm.leader.foreach { leader => return Right((leader.host, leader.port)) } } @@ -85,9 +87,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def findLeaders( topicAndPartitions: Set[TopicAndPartition] ): Either[Err, Map[TopicAndPartition, (String, Int)]] = { - getPartitionMetadata(topicAndPartitions.map(_.topic)).right.flatMap { tms => + val topics = topicAndPartitions.map(_.topic) + getPartitionMetadata(topics).right.flatMap { tms: Set[TopicMetadata] => val result = tms.flatMap { tm: TopicMetadata => - tm.partitionsMetadata.flatMap { pm => + tm.partitionsMetadata.flatMap { pm: PartitionMetadata => val tp = TopicAndPartition(tm.topic, pm.partitionId) if (topicAndPartitions(tp)) { pm.leader.map { l => @@ -112,15 +115,15 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def getPartitions(topics: Set[String]): Either[Err, Set[TopicAndPartition]] = getPartitionMetadata(topics).right.map { r => r.flatMap { tm: TopicMetadata => - tm.partitionsMetadata.map { pm => + tm.partitionsMetadata.map { pm: PartitionMetadata => TopicAndPartition(tm.topic, pm.partitionId) } } } def getPartitionMetadata(topics: Set[String]): Either[Err, Set[TopicMetadata]] = { - val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, - 0, config.clientId, topics.toSeq) + val req = TopicMetadataRequest( + TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) val errs = new Err withBrokers(Random.shuffle(seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) @@ -131,6 +134,11 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { Left(errs) } + // Leader offset api + // scalastyle:off + // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI + // scalastyle:on + def getLatestLeaderOffsets( topicAndPartitions: Set[TopicAndPartition] ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = @@ -170,17 +178,17 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { withBrokers(leaders, errs) { consumer => val needed: Seq[TopicAndPartition] = leaderToTp((consumer.host, consumer.port)) val req = OffsetRequest( - needed.map { tp => + needed.map { tp: TopicAndPartition => tp -> PartitionOffsetRequestInfo(before, maxNumOffsets) }.toMap ) val resp = consumer.getOffsetsBefore(req) val respMap = resp.partitionErrorAndOffsets - needed.foreach { tp => - respMap.get(tp).foreach { errAndOffsets => - if (errAndOffsets.error == ErrorMapping.NoError) { - if (errAndOffsets.offsets.nonEmpty) { - result += tp -> errAndOffsets.offsets.map { off => + needed.foreach { tp: TopicAndPartition => + respMap.get(tp).foreach { por: PartitionOffsetsResponse => + if (por.error == ErrorMapping.NoError) { + if (por.offsets.nonEmpty) { + result += tp -> por.offsets.map { off => LeaderOffset(consumer.host, consumer.port, off) } } else { @@ -188,7 +196,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { s"Empty offsets for ${tp}, is ${before} before log beginning?")) } } else { - errs.append(ErrorMapping.exceptionFor(errAndOffsets.error)) + errs.append(ErrorMapping.exceptionFor(por.error)) } } } @@ -202,9 +210,14 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } + // Consumer offset api + // scalastyle:off + // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + // scalastyle:on + def getConsumerOffsets( - groupId: String, - topicAndPartitions: Set[TopicAndPartition] + groupId: String, + topicAndPartitions: Set[TopicAndPartition] ): Either[Err, Map[TopicAndPartition, Long]] = { getConsumerOffsetMetadata(groupId, topicAndPartitions).right.map { r => r.map { kv => @@ -214,8 +227,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } def getConsumerOffsetMetadata( - groupId: String, - topicAndPartitions: Set[TopicAndPartition] + groupId: String, + topicAndPartitions: Set[TopicAndPartition] ): Either[Err, Map[TopicAndPartition, OffsetMetadataAndError]] = { var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) @@ -224,12 +237,12 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) - needed.foreach { tp => - respMap.get(tp).foreach { offsetMeta => - if (offsetMeta.error == ErrorMapping.NoError) { - result += tp -> offsetMeta + needed.foreach { tp: TopicAndPartition => + respMap.get(tp).foreach { ome: OffsetMetadataAndError => + if (ome.error == ErrorMapping.NoError) { + result += tp -> ome } else { - errs.append(ErrorMapping.exceptionFor(offsetMeta.error)) + errs.append(ErrorMapping.exceptionFor(ome.error)) } } } @@ -243,8 +256,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } def setConsumerOffsets( - groupId: String, - offsets: Map[TopicAndPartition, Long] + groupId: String, + offsets: Map[TopicAndPartition, Long] ): Either[Err, Map[TopicAndPartition, Short]] = { setConsumerOffsetMetadata(groupId, offsets.map { kv => kv._1 -> OffsetMetadataAndError(kv._2) @@ -252,8 +265,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } def setConsumerOffsetMetadata( - groupId: String, - metadata: Map[TopicAndPartition, OffsetMetadataAndError] + groupId: String, + metadata: Map[TopicAndPartition, OffsetMetadataAndError] ): Either[Err, Map[TopicAndPartition, Short]] = { var result = Map[TopicAndPartition, Short]() val req = OffsetCommitRequest(groupId, metadata) @@ -263,8 +276,8 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val resp = consumer.commitOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) - needed.foreach { tp => - respMap.get(tp).foreach { err => + needed.foreach { tp: TopicAndPartition => + respMap.get(tp).foreach { err: Short => if (err == ErrorMapping.NoError) { result += tp -> err } else { @@ -281,18 +294,21 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { Left(errs) } + // Try a call against potentially multiple brokers, accumulating errors private def withBrokers(brokers: Iterable[(String, Int)], errs: Err) (fn: SimpleConsumer => Any): Unit = { brokers.foreach { hp => var consumer: SimpleConsumer = null try { - consumer = connect(hp) + consumer = connect(hp._1, hp._2) fn(consumer) } catch { case NonFatal(e) => errs.append(e) } finally { - if (consumer != null) consumer.close() + if (consumer != null) { + consumer.close() + } } } } @@ -305,9 +321,10 @@ object KafkaCluster { private[spark] case class LeaderOffset(host: String, port: Int, offset: Long) - /** Make a consumer config without requiring group.id or zookeeper.connect, - * since communicating with brokers also needs common settings such as timeout - */ + /** + * Make a consumer config without requiring group.id or zookeeper.connect, + * since communicating with brokers also needs common settings such as timeout + */ def consumerConfig(kafkaParams: Map[String, String]): ConsumerConfig = { val props = new Properties() kafkaParams.foreach(param => props.put(param._1, param._2)) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala index 44cf9678a6f04..358525d30e5e0 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala @@ -31,17 +31,18 @@ import kafka.message.{MessageAndMetadata, MessageAndOffset} import kafka.serializer.Decoder import kafka.utils.VerifiableProperties -/** A batch-oriented interface for consuming from Kafka. - * Starting and ending offsets are specified in advance, - * so that you can control exactly-once semantics. - * @param kafkaParams Kafka - * configuration parameters. - * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), - * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param batch Each KafkaRDDPartition in the batch corresponds to a - * range of offsets for a given Kafka topic/partition - * @param messageHandler function for translating each message into the desired type - */ +/** + * A batch-oriented interface for consuming from Kafka. + * Starting and ending offsets are specified in advance, + * so that you can control exactly-once semantics. + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param batch Each KafkaRDDPartition in the batch corresponds to a + * range of offsets for a given Kafka topic/partition + * @param messageHandler function for translating each message into the desired type + */ private[spark] class KafkaRDD[ K: ClassTag, @@ -65,16 +66,24 @@ class KafkaRDD[ Seq(part.host) } - private def assertStartedCleanly(part: KafkaRDDPartition) { - assert(part.fromOffset <= part.untilOffset, - s"Beginning offset ${part.fromOffset} is after the ending offset ${part.untilOffset} " + - s"for topic ${part.topic} partition ${part.partition}. " + - "You either provided an invalid fromOffset, or the Kafka topic has been damaged") - } + private def errBeginAfterEnd(part: KafkaRDDPartition): String = + s"Beginning offset ${part.fromOffset} is after the ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition}. " + + "You either provided an invalid fromOffset, or the Kafka topic has been damaged" + + private def errRanOutBeforeEnd(part: KafkaRDDPartition): String = + s"Ran out of messages before reaching ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates that messages may have been lost" + + private def errOvershotEnd(itemOffset: Long, part: KafkaRDDPartition): String = + s"Got ${itemOffset} > ending offset ${part.untilOffset} " + + s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + + " This should not happen, and indicates a message may have been skipped" override def compute(thePart: Partition, context: TaskContext): Iterator[R] = { val part = thePart.asInstanceOf[KafkaRDDPartition] - assertStartedCleanly(part) + assert(part.fromOffset <= part.untilOffset, errBeginAfterEnd(part)) if (part.fromOffset == part.untilOffset) { log.warn("Beginning offset ${part.fromOffset} is the same as ending offset " + s"skipping ${part.topic} ${part.partition}") @@ -85,7 +94,8 @@ class KafkaRDD[ } private class KafkaRDDIterator( - part: KafkaRDDPartition, context: TaskContext) extends NextIterator[R] { + part: KafkaRDDPartition, + context: TaskContext) extends NextIterator[R] { context.addTaskCompletionListener{ context => closeIfNeeded() } @@ -132,24 +142,10 @@ class KafkaRDD[ } } - private def assertFinishedEmpty(requestOffset: Long) { - assert(requestOffset == part.untilOffset, - s"ran out of messages before reaching ending offset ${part.untilOffset} " + - s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + - " This should not happen, and indicates that messages may have been lost") - } - - private def assertFinishedWithoutOvershoot(itemOffset: Long) { - assert(itemOffset == part.untilOffset, - s"got ${itemOffset} > ending offset ${part.untilOffset} " + - s"for topic ${part.topic} partition ${part.partition} start ${part.fromOffset}." + - " This should not happen, and indicates a message may have been skipped") - } - private def fetchBatch: Iterator[MessageAndOffset] = { - val req = new FetchRequestBuilder(). - addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes). - build() + val req = new FetchRequestBuilder() + .addFetch(part.topic, part.partition, requestOffset, kc.config.fetchMessageMaxBytes) + .build() val resp = consumer.fetch(req) handleFetchErr(resp) // kafka may return a batch that starts before the requested offset @@ -160,18 +156,18 @@ class KafkaRDD[ override def close() = consumer.close() - override def getNext: R = { + override def getNext(): R = { if (iter == null || !iter.hasNext) { iter = fetchBatch } if (!iter.hasNext) { - assertFinishedEmpty(requestOffset) + assert(requestOffset == part.untilOffset, errRanOutBeforeEnd(part)) finished = true null.asInstanceOf[R] } else { - val item = iter.next + val item = iter.next() if (item.offset >= part.untilOffset) { - assertFinishedWithoutOvershoot(item.offset) + assert(item.offset == part.untilOffset, errOvershotEnd(item.offset, part)) finished = true null.asInstanceOf[R] } else { @@ -189,16 +185,16 @@ object KafkaRDD { import KafkaCluster.LeaderOffset /** - * @param kafkaParams Kafka - * configuration parameters. - * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), - * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) - * starting point of the batch - * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) - * ending point of the batch - * @param messageHandler function for translating each message into the desired type - */ + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the batch + * @param untilOffsets per-topic/partition Kafka offsets defining the (exclusive) + * ending point of the batch + * @param messageHandler function for translating each message into the desired type + */ def apply[ K: ClassTag, V: ClassTag, diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 82bc83692cf56..7dd5e3fba1ba0 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -29,6 +29,7 @@ import kafka.serializer.{Decoder, StringDecoder} import org.apache.spark.{SparkContext, SparkException} +import org.apache.spark.annotation.Experimental import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.StreamingContext @@ -161,6 +162,7 @@ object KafkaUtils { * @param batch Each OffsetRange in the batch corresponds to a * range of offsets for a given Kafka topic/partition */ + @Experimental def createRDD[ K: ClassTag, V: ClassTag, @@ -199,6 +201,7 @@ object KafkaUtils { * @param leaders Kafka leaders for each offset range in batch * @param messageHandler function for translating each message into the desired type */ + @Experimental def createRDD[ K: ClassTag, V: ClassTag, @@ -221,9 +224,8 @@ object KafkaUtils { } /** - * Compared to `createStream`, the stream created by this can guarantee that each message - * from Kafka is included in transformations (as opposed to output actions) exactly once, - * even in most failure situations. + * This stream can guarantee that each message from Kafka is included in transformations + * (as opposed to output actions) exactly once, even in most failure situations. * * Points to note: * @@ -252,6 +254,7 @@ object KafkaUtils { * starting point of the stream * @param maxRetries maximum number of times in a row to retry getting leaders' offsets */ + @Experimental def createNewStream[ K: ClassTag, V: ClassTag, @@ -269,9 +272,8 @@ object KafkaUtils { } /** - * Compared to `createStream`, the stream created by this can guarantee that each message - * from Kafka is included in transformations (as opposed to output actions) exactly once, - * even in most failure situations. + * This stream can guarantee that each message from Kafka is included in transformations + * (as opposed to output actions) exactly once, even in most failure situations. * * Points to note: * @@ -296,6 +298,7 @@ object KafkaUtils { * to determine where the stream starts (defaults to "largest") * @param topics names of the topics to consume */ + @Experimental def createNewStream[ K: ClassTag, V: ClassTag, diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaClusterSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaClusterSuite.scala new file mode 100644 index 0000000000000..e57c8f6987fdc --- /dev/null +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaClusterSuite.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import scala.util.Random + +import org.scalatest.BeforeAndAfter +import kafka.common.TopicAndPartition + +class KafkaClusterSuite extends KafkaStreamSuiteBase with BeforeAndAfter { + val brokerHost = "localhost" + + val kafkaParams = Map("metadata.broker.list" -> s"$brokerHost:$brokerPort") + + val kc = new KafkaCluster(kafkaParams) + + val topic = "kcsuitetopic" + Random.nextInt(10000) + + val topicAndPartition = TopicAndPartition(topic, 0) + + before { + setupKafka() + createTopic(topic) + produceAndSendMessage(topic, Map("a" -> 1)) + } + + after { + tearDownKafka() + } + + test("metadata apis") { + val leader = kc.findLeaders(Set(topicAndPartition)).right.get + assert(leader(topicAndPartition) === (brokerHost, brokerPort), "didn't get leader") + + val parts = kc.getPartitions(Set(topic)).right.get + assert(parts(topicAndPartition), "didn't get partitions") + } + + test("leader offset apis") { + val earliest = kc.getEarliestLeaderOffsets(Set(topicAndPartition)).right.get + assert(earliest(topicAndPartition).offset === 0, "didn't get earliest") + + val latest = kc.getLatestLeaderOffsets(Set(topicAndPartition)).right.get + assert(latest(topicAndPartition).offset === 1, "didn't get latest") + } + + test("consumer offset apis") { + val group = "kcsuitegroup" + Random.nextInt(10000) + + val offset = Random.nextInt(10000) + + val set = kc.setConsumerOffsets(group, Map(topicAndPartition -> offset)) + assert(set.isRight, "didn't set consumer offsets") + + val get = kc.getConsumerOffsets(group, Set(topicAndPartition)).right.get + assert(get(topicAndPartition) === offset, "didn't get consumer offsets") + } +} diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala new file mode 100644 index 0000000000000..f78695ba30697 --- /dev/null +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import scala.util.Random +import scala.concurrent.duration._ + +import org.scalatest.BeforeAndAfter +import org.scalatest.concurrent.Eventually + +import kafka.serializer.StringDecoder + +import org.apache.spark.SparkConf +import org.apache.spark.storage.StorageLevel +import org.apache.spark.streaming.{Milliseconds, StreamingContext} + +class KafkaNewStreamSuite extends KafkaStreamSuiteBase with BeforeAndAfter with Eventually { + val sparkConf = new SparkConf() + .setMaster("local[4]") + .setAppName(this.getClass.getSimpleName) + + val brokerHost = "localhost" + + val kafkaParams = Map( + "metadata.broker.list" -> s"$brokerHost:$brokerPort", + "auto.offset.reset" -> "smallest" + ) + + var ssc: StreamingContext = _ + + before { + setupKafka() + + ssc = new StreamingContext(sparkConf, Milliseconds(500)) + } + + after { + if (ssc != null) { + ssc.stop() + } + tearDownKafka() + } + + test("multi topic stream") { + val topics = Set("newA", "newB") + val data = Map("a" -> 7, "b" -> 9) + topics.foreach { t => + createTopic(t) + produceAndSendMessage(t, data) + } + val stream = KafkaUtils.createNewStream[String, String, StringDecoder, StringDecoder]( + ssc, kafkaParams, topics) + var total = 0L; + + stream.foreachRDD { rdd => + val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges + val collected = rdd.mapPartitionsWithIndex { (i, iter) => + val off = offsets(i) + val all = iter.toSeq + val partSize = all.size + val rangeSize = off.untilOffset - off.fromOffset + all.map { _ => + (partSize, rangeSize) + }.toIterator + }.collect + collected.foreach { case (partSize, rangeSize) => + assert(partSize === rangeSize, "offset ranges are wrong") + } + total += collected.size + } + ssc.start() + eventually(timeout(20000.milliseconds), interval(200.milliseconds)) { + assert(total === data.values.sum * topics.size, "didn't get all messages") + } + ssc.stop() + } +} From 0090553eba09240b6ad4cf508ea33503705b12d9 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 30 Jan 2015 09:20:30 -0600 Subject: [PATCH 34/42] [SPARK-4964] javafication of interfaces --- .../streaming/kafka/HasOffsetRanges.java | 24 +++++++ .../apache/spark/streaming/kafka/Leader.java} | 49 ++----------- .../spark/streaming/kafka/OffsetRange.java} | 50 ++----------- .../spark/streaming/kafka/KafkaUtils.scala | 71 +++++++++++++++++++ 4 files changed, 108 insertions(+), 86 deletions(-) create mode 100644 external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java rename external/kafka/src/main/{scala/org/apache/spark/streaming/kafka/Leader.scala => java/org/apache/spark/streaming/kafka/Leader.java} (55%) rename external/kafka/src/main/{scala/org/apache/spark/streaming/kafka/OffsetRange.scala => java/org/apache/spark/streaming/kafka/OffsetRange.java} (51%) diff --git a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java new file mode 100644 index 0000000000000..13548ab296764 --- /dev/null +++ b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka; + +/** Something that has a collection of OffsetRanges */ +public interface HasOffsetRanges { + /** array of OffsetRanges */ + public OffsetRange[] offsetRanges(); +} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java similarity index 55% rename from external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala rename to external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java index 360c359ae4dcd..f45f2d208f3ad 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala +++ b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java @@ -15,54 +15,19 @@ * limitations under the License. */ -package org.apache.spark.streaming.kafka - -import kafka.common.TopicAndPartition +package org.apache.spark.streaming.kafka; /** Host info for the leader of a Kafka TopicAndPartition */ - -trait Leader { - /** kafka topic name */ - def topic: String +public interface Leader { + /** kafka topic name */ + public String topic(); /** kafka partition id */ - def partition: Int + public int partition(); /** kafka hostname */ - def host: String + public String host(); /** kafka host's port */ - def port: Int -} - -private class LeaderImpl( - override val topic: String, - override val partition: Int, - override val host: String, - override val port: Int -) extends Leader - -object Leader { - def create( - topic: String, - partition: Int, - host: String, - port: Int): Leader = - new LeaderImpl( - topic, - partition, - host, - port) - - def create( - topicAndPartition: TopicAndPartition, - host: String, - port: Int): Leader = - new LeaderImpl( - topicAndPartition.topic, - topicAndPartition.partition, - host, - port) - + public int port(); } - diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java similarity index 51% rename from external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala rename to external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java index ea3ef067ea4ac..ee79ba663476c 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala +++ b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java @@ -15,57 +15,19 @@ * limitations under the License. */ -package org.apache.spark.streaming.kafka - -import kafka.common.TopicAndPartition +package org.apache.spark.streaming.kafka; /** Represents a range of offsets from a single Kafka TopicAndPartition */ -trait OffsetRange { +public interface OffsetRange { /** kafka topic name */ - def topic: String + public String topic(); /** kafka partition id */ - def partition: Int + public int partition(); /** inclusive starting offset */ - def fromOffset: Long + public long fromOffset(); /** exclusive ending offset */ - def untilOffset: Long -} - -/** Something that has a collection of OffsetRanges */ -trait HasOffsetRanges { - def offsetRanges: Array[OffsetRange] -} - -private class OffsetRangeImpl( - override val topic: String, - override val partition: Int, - override val fromOffset: Long, - override val untilOffset: Long -) extends OffsetRange - -object OffsetRange { - def create( - topic: String, - partition: Int, - fromOffset: Long, - untilOffset: Long): OffsetRange = - new OffsetRangeImpl( - topic, - partition, - fromOffset, - untilOffset) - - def create( - topicAndPartition: TopicAndPartition, - fromOffset: Long, - untilOffset: Long): OffsetRange = - new OffsetRangeImpl( - topicAndPartition.topic, - topicAndPartition.partition, - fromOffset, - untilOffset) - + public long untilOffset(); } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 7dd5e3fba1ba0..458e2823f88cd 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -330,4 +330,75 @@ object KafkaUtils { ok => ok ) } + + private class OffsetRangeImpl( + override val topic: String, + override val partition: Int, + override val fromOffset: Long, + override val untilOffset: Long) extends OffsetRange + + /** + * Behaviorless container for a range of offsets from a single Kafka TopicAndPartition + * @param topic kafka topic name + * @param partition kafka partition id + * @param fromOffset inclusive starting offset + * @param untilOffset exclusive ending offset + */ + @Experimental + def createOffsetRange( + topic: String, + partition: Int, + fromOffset: Long, + untilOffset: Long): OffsetRange = + new OffsetRangeImpl(topic, partition, fromOffset, untilOffset) + + /** + * Behaviorless container for a range of offsets from a single Kafka TopicAndPartition + * @param topicAndPartition kafka TopicAndPartition + * @param fromOffset inclusive starting offset + * @param untilOffset exclusive ending offset + */ + @Experimental + def createOffsetRange( + topicAndPartition: TopicAndPartition, + fromOffset: Long, + untilOffset: Long): OffsetRange = + new OffsetRangeImpl( + topicAndPartition.topic, topicAndPartition.partition, fromOffset, untilOffset) + + private class LeaderImpl( + override val topic: String, + override val partition: Int, + override val host: String, + override val port: Int) extends Leader + + /** + * Behaviorless container of host info for the leader of a Kafka TopicAndPartition + * @param topic kafka topic name + * @param partition kafka partition id + * @param host kafka hostname + * @param port kafka host's port + */ + @Experimental + def createLeader(topic: String, partition: Int, host: String, port: Int): Leader = + new LeaderImpl(topic,partition, + host, + port) + + /** + * Behaviorless container of host info for the leader of a Kafka TopicAndPartition + * @param topicAndPartition kafka TopicAndPartition + * @param host kafka hostname + * @param port kafka host's port + */ + @Experimental + def createLeader( + topicAndPartition: TopicAndPartition, + host: String, + port: Int): Leader = + new LeaderImpl( + topicAndPartition.topic, + topicAndPartition.partition, + host, + port) } From 9adaa0a0e13a1c7eef80569b90ff7eaa2b6db190 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Fri, 30 Jan 2015 09:34:35 -0600 Subject: [PATCH 35/42] [SPARK-4964] formatting --- .../spark/streaming/kafka/KafkaUtils.scala | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 458e2823f88cd..98524efffc3f3 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -381,9 +381,7 @@ object KafkaUtils { */ @Experimental def createLeader(topic: String, partition: Int, host: String, port: Int): Leader = - new LeaderImpl(topic,partition, - host, - port) + new LeaderImpl(topic,partition, host, port) /** * Behaviorless container of host info for the leader of a Kafka TopicAndPartition @@ -393,12 +391,8 @@ object KafkaUtils { */ @Experimental def createLeader( - topicAndPartition: TopicAndPartition, - host: String, - port: Int): Leader = - new LeaderImpl( - topicAndPartition.topic, - topicAndPartition.partition, - host, - port) + topicAndPartition: TopicAndPartition, + host: String, + port: Int): Leader = + new LeaderImpl(topicAndPartition.topic, topicAndPartition.partition, host, port) } From 4354bced65a7f37a51bde9081d8d19dc9b9316cd Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 2 Feb 2015 19:24:20 -0600 Subject: [PATCH 36/42] [SPARK-4964] per td, remove java interfaces, replace with final classes, corresponding changes to KafkaRDD constructor and checkpointing --- .../streaming/kafka/HasOffsetRanges.java | 24 ----- .../apache/spark/streaming/kafka/Leader.java | 33 ------- .../spark/streaming/kafka/OffsetRange.java | 33 ------- .../DeterministicKafkaInputDStream.scala | 19 ++-- .../spark/streaming/kafka/KafkaRDD.scala | 26 ++--- .../streaming/kafka/KafkaRDDPartition.scala | 35 ++----- .../spark/streaming/kafka/KafkaUtils.scala | 97 +++---------------- .../apache/spark/streaming/kafka/Leader.scala | 46 +++++++++ .../spark/streaming/kafka/OffsetRange.scala | 69 +++++++++++++ .../spark/streaming/kafka/KafkaRDDSuite.scala | 2 +- 10 files changed, 162 insertions(+), 222 deletions(-) delete mode 100644 external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java delete mode 100644 external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java delete mode 100644 external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java create mode 100644 external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala create mode 100644 external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala diff --git a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java deleted file mode 100644 index 13548ab296764..0000000000000 --- a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/HasOffsetRanges.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.streaming.kafka; - -/** Something that has a collection of OffsetRanges */ -public interface HasOffsetRanges { - /** array of OffsetRanges */ - public OffsetRange[] offsetRanges(); -} diff --git a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java deleted file mode 100644 index f45f2d208f3ad..0000000000000 --- a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/Leader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.streaming.kafka; - -/** Host info for the leader of a Kafka TopicAndPartition */ -public interface Leader { - /** kafka topic name */ - public String topic(); - - /** kafka partition id */ - public int partition(); - - /** kafka hostname */ - public String host(); - - /** kafka host's port */ - public int port(); -} diff --git a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java b/external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java deleted file mode 100644 index ee79ba663476c..0000000000000 --- a/external/kafka/src/main/java/org/apache/spark/streaming/kafka/OffsetRange.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.streaming.kafka; - -/** Represents a range of offsets from a single Kafka TopicAndPartition */ -public interface OffsetRange { - /** kafka topic name */ - public String topic(); - - /** kafka partition id */ - public int partition(); - - /** inclusive starting offset */ - public long fromOffset(); - - /** exclusive ending offset */ - public long untilOffset(); -} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 85893347c0108..1c4ccd187eb2c 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -26,13 +26,13 @@ import kafka.common.TopicAndPartition import kafka.message.MessageAndMetadata import kafka.serializer.Decoder -import org.apache.spark.Logging +import org.apache.spark.{Logging, SparkException} import org.apache.spark.rdd.RDD import org.apache.spark.streaming.kafka.KafkaCluster.LeaderOffset import org.apache.spark.streaming.{StreamingContext, Time} import org.apache.spark.streaming.dstream._ -/** A stream of {@link org.apache.spark.rdd.kafka.KafkaRDD} where +/** A stream of {@link org.apache.spark.streaming.kafka.KafkaRDD} where * each given Kafka topic/partition corresponds to an RDD partition. * The spark configuration spark.streaming.receiver.maxRate gives the maximum number of messages * per second that each '''partition''' will accept. @@ -40,7 +40,7 @@ import org.apache.spark.streaming.dstream._ * and this DStream is not responsible for committing offsets, * so that you can control exactly-once semantics. * For an easy interface to Kafka-managed offsets, - * see {@link org.apache.spark.rdd.kafka.KafkaCluster} + * see {@link org.apache.spark.streaming.kafka.KafkaCluster} * @param kafkaParams Kafka * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), @@ -126,12 +126,12 @@ class DeterministicKafkaInputDStream[ private[streaming] class DeterministicKafkaInputDStreamCheckpointData extends DStreamCheckpointData(this) { def batchForTime = data.asInstanceOf[mutable.HashMap[ - Time, Array[(Int, String, Int, Long, Long, String, Int)]]] + Time, Array[OffsetRange.OffsetRangeTuple]]] override def update(time: Time) { batchForTime.clear() generatedRDDs.foreach { kv => - val a = kv._2.asInstanceOf[KafkaRDD[K, V, U, T, R]].batch.map(_.toTuple).toArray + val a = kv._2.asInstanceOf[KafkaRDD[K, V, U, T, R]].offsetRanges.map(_.toTuple).toArray batchForTime += kv._1 -> a } } @@ -139,10 +139,17 @@ class DeterministicKafkaInputDStream[ override def cleanup(time: Time) { } override def restore() { + // this is assuming that the topics don't change during execution, which is true currently + val topics = fromOffsets.keySet + val leaders = kc.findLeaders(topics).fold( + errs => throw new SparkException(errs.mkString("\n")), + ok => ok + ) + batchForTime.toSeq.sortBy(_._1)(Time.ordering).foreach { case (t, b) => logInfo(s"Restoring KafkaRDD for time $t ${b.mkString("[", ", ", "]")}") generatedRDDs += t -> new KafkaRDD[K, V, U, T, R]( - context.sparkContext, kafkaParams, b.map(KafkaRDDPartition(_)), messageHandler) + context.sparkContext, kafkaParams, b.map(OffsetRange(_)), leaders, messageHandler) } } } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala index 358525d30e5e0..50bf7cbdb8dbf 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala @@ -52,13 +52,16 @@ class KafkaRDD[ R: ClassTag] private[spark] ( sc: SparkContext, kafkaParams: Map[String, String], - private[spark] val batch: Array[KafkaRDDPartition], + val offsetRanges: Array[OffsetRange], + leaders: Map[TopicAndPartition, (String, Int)], messageHandler: MessageAndMetadata[K, V] => R ) extends RDD[R](sc, Nil) with Logging with HasOffsetRanges { - - def offsetRanges: Array[OffsetRange] = batch.asInstanceOf[Array[OffsetRange]] - - override def getPartitions: Array[Partition] = batch.asInstanceOf[Array[Partition]] + override def getPartitions: Array[Partition] = { + offsetRanges.zipWithIndex.map { case (o, i) => + val (host, port) = leaders(TopicAndPartition(o.topic, o.partition)) + new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, host, port) + }.toArray + } override def getPreferredLocations(thePart: Partition): Seq[String] = { val part = thePart.asInstanceOf[KafkaRDDPartition] @@ -207,14 +210,15 @@ object KafkaRDD { untilOffsets: Map[TopicAndPartition, LeaderOffset], messageHandler: MessageAndMetadata[K, V] => R ): KafkaRDD[K, V, U, T, R] = { - assert(fromOffsets.keys == untilOffsets.keys, - "Must provide both from and until offsets for each topic/partition") + val leaders = untilOffsets.map { case (tp, lo) => + tp -> (lo.host, lo.port) + }.toMap - val partitions = fromOffsets.zipWithIndex.map { case ((tp, from), index) => - val lo = untilOffsets(tp) - new KafkaRDDPartition(index, tp.topic, tp.partition, from, lo.offset, lo.host, lo.port) + val offsetRanges = fromOffsets.map { case (tp, fo) => + val uo = untilOffsets(tp) + OffsetRange(tp.topic, tp.partition, fo, uo.offset) }.toArray - new KafkaRDD[K, V, U, T, R](sc, kafkaParams, partitions, messageHandler) + new KafkaRDD[K, V, U, T, R](sc, kafkaParams, offsetRanges, leaders, messageHandler) } } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala index bf4c6ebf69556..36372e08f65f6 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDPartition.scala @@ -28,25 +28,14 @@ import org.apache.spark.Partition */ private[spark] class KafkaRDDPartition( - override val index: Int, - override val topic: String, - override val partition: Int, - override val fromOffset: Long, - override val untilOffset: Long, + val index: Int, + val topic: String, + val partition: Int, + val fromOffset: Long, + val untilOffset: Long, val host: String, val port: Int -) extends Partition with OffsetRange { - def toTuple: (Int, String, Int, Long, Long, String, Int) = ( - index, - topic, - partition, - fromOffset, - untilOffset, - host, - port - ) - -} +) extends Partition private[spark] object KafkaRDDPartition { @@ -67,16 +56,4 @@ object KafkaRDDPartition { host, port ) - - def apply(tuple: (Int, String, Int, Long, Long, String, Int)): KafkaRDDPartition = { - new KafkaRDDPartition( - tuple._1, - tuple._2, - tuple._3, - tuple._4, - tuple._5, - tuple._6, - tuple._7 - ) - } } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 98524efffc3f3..46f2b386586ea 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -159,7 +159,7 @@ object KafkaUtils { * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param batch Each OffsetRange in the batch corresponds to a + * @param offsetRanges Each OffsetRange in the batch corresponds to a * range of offsets for a given Kafka topic/partition */ @Experimental @@ -167,25 +167,19 @@ object KafkaUtils { K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, - T <: Decoder[_]: ClassTag, - R: ClassTag] ( + T <: Decoder[_]: ClassTag] ( sc: SparkContext, kafkaParams: Map[String, String], - batch: Array[OffsetRange] + offsetRanges: Array[OffsetRange] ): RDD[(K, V)] with HasOffsetRanges = { val messageHandler = (mmd: MessageAndMetadata[K, V]) => (mmd.key, mmd.message) val kc = new KafkaCluster(kafkaParams) - val topics = batch.map(o => TopicAndPartition(o.topic, o.partition)).toSet - val leaderMap = kc.findLeaders(topics).fold( + val topics = offsetRanges.map(o => TopicAndPartition(o.topic, o.partition)).toSet + val leaders = kc.findLeaders(topics).fold( errs => throw new SparkException(errs.mkString("\n")), ok => ok ) - val rddParts = batch.zipWithIndex.map { case (o, i) => - val tp = TopicAndPartition(o.topic, o.partition) - val (host, port) = leaderMap(tp) - new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, host, port) - }.toArray - new KafkaRDD[K, V, U, T, (K, V)](sc, kafkaParams, rddParts, messageHandler) + new KafkaRDD[K, V, U, T, (K, V)](sc, kafkaParams, offsetRanges, leaders, messageHandler) } /** A batch-oriented interface for consuming from Kafka. @@ -196,7 +190,7 @@ object KafkaUtils { * configuration parameters. * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param batch Each OffsetRange in the batch corresponds to a + * @param offsetRanges Each OffsetRange in the batch corresponds to a * range of offsets for a given Kafka topic/partition * @param leaders Kafka leaders for each offset range in batch * @param messageHandler function for translating each message into the desired type @@ -210,17 +204,15 @@ object KafkaUtils { R: ClassTag] ( sc: SparkContext, kafkaParams: Map[String, String], - batch: Array[OffsetRange], + offsetRanges: Array[OffsetRange], leaders: Array[Leader], messageHandler: MessageAndMetadata[K, V] => R ): RDD[R] with HasOffsetRanges = { - val leaderMap = leaders.map(l => (l.topic, l.partition) -> (l.host, l.port)).toMap - val rddParts = batch.zipWithIndex.map { case (o, i) => - val (host, port) = leaderMap((o.topic, o.partition)) - new KafkaRDDPartition(i, o.topic, o.partition, o.fromOffset, o.untilOffset, host, port) - }.toArray - new KafkaRDD[K, V, U, T, R](sc, kafkaParams, rddParts, messageHandler) + val leaderMap = leaders + .map(l => TopicAndPartition(l.topic, l.partition) -> (l.host, l.port)) + .toMap + new KafkaRDD[K, V, U, T, R](sc, kafkaParams, offsetRanges, leaderMap, messageHandler) } /** @@ -330,69 +322,4 @@ object KafkaUtils { ok => ok ) } - - private class OffsetRangeImpl( - override val topic: String, - override val partition: Int, - override val fromOffset: Long, - override val untilOffset: Long) extends OffsetRange - - /** - * Behaviorless container for a range of offsets from a single Kafka TopicAndPartition - * @param topic kafka topic name - * @param partition kafka partition id - * @param fromOffset inclusive starting offset - * @param untilOffset exclusive ending offset - */ - @Experimental - def createOffsetRange( - topic: String, - partition: Int, - fromOffset: Long, - untilOffset: Long): OffsetRange = - new OffsetRangeImpl(topic, partition, fromOffset, untilOffset) - - /** - * Behaviorless container for a range of offsets from a single Kafka TopicAndPartition - * @param topicAndPartition kafka TopicAndPartition - * @param fromOffset inclusive starting offset - * @param untilOffset exclusive ending offset - */ - @Experimental - def createOffsetRange( - topicAndPartition: TopicAndPartition, - fromOffset: Long, - untilOffset: Long): OffsetRange = - new OffsetRangeImpl( - topicAndPartition.topic, topicAndPartition.partition, fromOffset, untilOffset) - - private class LeaderImpl( - override val topic: String, - override val partition: Int, - override val host: String, - override val port: Int) extends Leader - - /** - * Behaviorless container of host info for the leader of a Kafka TopicAndPartition - * @param topic kafka topic name - * @param partition kafka partition id - * @param host kafka hostname - * @param port kafka host's port - */ - @Experimental - def createLeader(topic: String, partition: Int, host: String, port: Int): Leader = - new LeaderImpl(topic,partition, host, port) - - /** - * Behaviorless container of host info for the leader of a Kafka TopicAndPartition - * @param topicAndPartition kafka TopicAndPartition - * @param host kafka hostname - * @param port kafka host's port - */ - @Experimental - def createLeader( - topicAndPartition: TopicAndPartition, - host: String, - port: Int): Leader = - new LeaderImpl(topicAndPartition.topic, topicAndPartition.partition, host, port) } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala new file mode 100644 index 0000000000000..3454d92e72b47 --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/Leader.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import kafka.common.TopicAndPartition + +/** Host info for the leader of a Kafka TopicAndPartition */ +final class Leader private( + /** kafka topic name */ + val topic: String, + /** kafka partition id */ + val partition: Int, + /** kafka hostname */ + val host: String, + /** kafka host's port */ + val port: Int) extends Serializable + +object Leader { + def create(topic: String, partition: Int, host: String, port: Int): Leader = + new Leader(topic, partition, host, port) + + def create(topicAndPartition: TopicAndPartition, host: String, port: Int): Leader = + new Leader(topicAndPartition.topic, topicAndPartition.partition, host, port) + + def apply(topic: String, partition: Int, host: String, port: Int): Leader = + new Leader(topic, partition, host, port) + + def apply(topicAndPartition: TopicAndPartition, host: String, port: Int): Leader = + new Leader(topicAndPartition.topic, topicAndPartition.partition, host, port) + +} diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala new file mode 100644 index 0000000000000..132bd81e2364d --- /dev/null +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.kafka + +import kafka.common.TopicAndPartition + +/** Something that has a collection of OffsetRanges */ +trait HasOffsetRanges { + def offsetRanges: Array[OffsetRange] +} + +/** Represents a range of offsets from a single Kafka TopicAndPartition */ +final class OffsetRange private( + /** kafka topic name */ + val topic: String, + /** kafka partition id */ + val partition: Int, + /** inclusive starting offset */ + val fromOffset: Long, + /** exclusive ending offset */ + val untilOffset: Long) extends Serializable { + import OffsetRange.OffsetRangeTuple + + /** this is to avoid ClassNotFoundException during checkpoint restore */ + private[streaming] + def toTuple: OffsetRangeTuple = (topic, partition, fromOffset, untilOffset) +} + +object OffsetRange { + private[spark] + type OffsetRangeTuple = (String, Int, Long, Long) + + def create(topic: String, partition: Int, fromOffset: Long, untilOffset: Long): OffsetRange = + new OffsetRange(topic, partition, fromOffset, untilOffset) + + def create( + topicAndPartition: TopicAndPartition, + fromOffset: Long, + untilOffset: Long): OffsetRange = + new OffsetRange(topicAndPartition.topic, topicAndPartition.partition, fromOffset, untilOffset) + + def apply(topic: String, partition: Int, fromOffset: Long, untilOffset: Long): OffsetRange = + new OffsetRange(topic, partition, fromOffset, untilOffset) + + def apply( + topicAndPartition: TopicAndPartition, + fromOffset: Long, + untilOffset: Long): OffsetRange = + new OffsetRange(topicAndPartition.topic, topicAndPartition.partition, fromOffset, untilOffset) + + private[streaming] + def apply(t: OffsetRangeTuple) = + new OffsetRange(t._1, t._2, t._3, t._4) +} diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala index e1ade6fea11e4..9b9e3f5fce8bd 100644 --- a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaRDDSuite.scala @@ -61,7 +61,7 @@ class KafkaRDDSuite extends KafkaStreamSuiteBase with BeforeAndAfter { kc.setConsumerOffsets( kafkaParams("group.id"), - rdd.get.batch.map(kp => TopicAndPartition(kp.topic, kp.partition) -> kp.untilOffset).toMap) + rdd.get.offsetRanges.map(o => TopicAndPartition(o.topic, o.partition) -> o.untilOffset).toMap) val rdd2 = getRdd(kc, Set(topic)) val sent2 = Map("d" -> 1) From 825110fc22abc8ba3bb8c0e685a35ed5e939ffb9 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 2 Feb 2015 21:18:51 -0600 Subject: [PATCH 37/42] [SPARK-4964] rename stuff per TD --- .../DeterministicKafkaInputDStream.scala | 13 ++-- .../spark/streaming/kafka/KafkaCluster.scala | 78 ++++++++++--------- .../spark/streaming/kafka/KafkaUtils.scala | 8 +- 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index 1c4ccd187eb2c..ba1ed39c980e9 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -34,7 +34,7 @@ import org.apache.spark.streaming.dstream._ /** A stream of {@link org.apache.spark.streaming.kafka.KafkaRDD} where * each given Kafka topic/partition corresponds to an RDD partition. - * The spark configuration spark.streaming.receiver.maxRate gives the maximum number of messages + * The spark configuration spark.streaming.kafka.maxRatePerPartition gives the maximum number of messages * per second that each '''partition''' will accept. * Starting offsets are specified in advance, * and this DStream is not responsible for committing offsets, @@ -60,9 +60,10 @@ class DeterministicKafkaInputDStream[ @transient ssc_ : StreamingContext, val kafkaParams: Map[String, String], val fromOffsets: Map[TopicAndPartition, Long], - messageHandler: MessageAndMetadata[K, V] => R, - maxRetries: Int + messageHandler: MessageAndMetadata[K, V] => R ) extends InputDStream[R](ssc_) with Logging { + val maxRetries = context.sparkContext.getConf.getInt( + "spark.streaming.kafka.maxRetries", 1) protected[streaming] override val checkpointData = new DeterministicKafkaInputDStreamCheckpointData @@ -70,7 +71,8 @@ class DeterministicKafkaInputDStream[ protected val kc = new KafkaCluster(kafkaParams) protected val maxMessagesPerPartition: Option[Long] = { - val ratePerSec = context.sparkContext.getConf.getInt("spark.streaming.receiver.maxRate", 0) + val ratePerSec = context.sparkContext.getConf.getInt( + "spark.streaming.kafka.maxRatePerPartition", 0) if (ratePerSec > 0) { val secsPerBatch = context.graph.batchDuration.milliseconds.toDouble / 1000 Some((secsPerBatch * ratePerSec).toLong) @@ -88,7 +90,7 @@ class DeterministicKafkaInputDStream[ if (o.isLeft) { val err = o.left.get.toString if (retries <= 0) { - throw new Exception(err) + throw new SparkException(err) } else { log.error(err) Thread.sleep(kc.config.refreshLeaderBackoffMs) @@ -99,6 +101,7 @@ class DeterministicKafkaInputDStream[ } } + // limits the maximum number of messages per partition protected def clamp( leaderOffsets: Map[TopicAndPartition, LeaderOffset]): Map[TopicAndPartition, LeaderOffset] = { maxMessagesPerPartition.map { mmp => diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala index f0484de35de54..54c8fe4821ce7 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala @@ -24,6 +24,7 @@ import java.util.Properties import kafka.api._ import kafka.common.{ErrorMapping, OffsetMetadataAndError, TopicAndPartition} import kafka.consumer.{ConsumerConfig, SimpleConsumer} +import org.apache.spark.SparkException /** * Convenience methods for interacting with a Kafka cluster. @@ -39,7 +40,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val seedBrokers: Array[(String, Int)] = kafkaParams.get("metadata.broker.list") .orElse(kafkaParams.get("bootstrap.servers")) - .getOrElse(throw new Exception("Must specify metadata.broker.list or bootstrap.servers")) + .getOrElse(throw new SparkException("Must specify metadata.broker.list or bootstrap.servers")) .split(",").map { hp => val hpa = hp.split(":") (hpa(0), hpa(1).toInt) @@ -85,11 +86,12 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } def findLeaders( - topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, (String, Int)]] = { + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, (String, Int)]] = { val topics = topicAndPartitions.map(_.topic) - getPartitionMetadata(topics).right.flatMap { tms: Set[TopicMetadata] => - val result = tms.flatMap { tm: TopicMetadata => + val response = getPartitionMetadata(topics).right + val answer = response.flatMap { tms: Set[TopicMetadata] => + val leaderMap = tms.flatMap { tm: TopicMetadata => tm.partitionsMetadata.flatMap { pm: PartitionMetadata => val tp = TopicAndPartition(tm.topic, pm.partitionId) if (topicAndPartitions(tp)) { @@ -101,18 +103,20 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } }.toMap - if (result.keys.size == topicAndPartitions.size) { - Right(result) + + if (leaderMap.keys.size == topicAndPartitions.size) { + Right(leaderMap) } else { - val missing = topicAndPartitions.diff(result.keySet) + val missing = topicAndPartitions.diff(leaderMap.keySet) val err = new Err - err.append(new Exception(s"Couldn't find leaders for ${missing}")) + err.append(new SparkException(s"Couldn't find leaders for ${missing}")) Left(err) } } + answer } - def getPartitions(topics: Set[String]): Either[Err, Set[TopicAndPartition]] = + def getPartitions(topics: Set[String]): Either[Err, Set[TopicAndPartition]] = { getPartitionMetadata(topics).right.map { r => r.flatMap { tm: TopicMetadata => tm.partitionsMetadata.map { pm: PartitionMetadata => @@ -120,6 +124,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } } + } def getPartitionMetadata(topics: Set[String]): Either[Err, Set[TopicMetadata]] = { val req = TopicMetadataRequest( @@ -140,25 +145,26 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { // scalastyle:on def getLatestLeaderOffsets( - topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.LatestTime) def getEarliestLeaderOffsets( - topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = + topicAndPartitions: Set[TopicAndPartition] + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = getLeaderOffsets(topicAndPartitions, OffsetRequest.EarliestTime) def getLeaderOffsets( - topicAndPartitions: Set[TopicAndPartition], - before: Long - ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = + topicAndPartitions: Set[TopicAndPartition], + before: Long + ): Either[Err, Map[TopicAndPartition, LeaderOffset]] = { getLeaderOffsets(topicAndPartitions, before, 1).right.map { r => r.map { kv => // mapValues isnt serializable, see SI-7005 kv._1 -> kv._2.head } } + } private def flip[K, V](m: Map[K, V]): Map[V, Seq[K]] = m.groupBy(_._2).map { kv => @@ -166,25 +172,25 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } def getLeaderOffsets( - topicAndPartitions: Set[TopicAndPartition], - before: Long, - maxNumOffsets: Int - ): Either[Err, Map[TopicAndPartition, Seq[LeaderOffset]]] = { + topicAndPartitions: Set[TopicAndPartition], + before: Long, + maxNumOffsets: Int + ): Either[Err, Map[TopicAndPartition, Seq[LeaderOffset]]] = { findLeaders(topicAndPartitions).right.flatMap { tpToLeader => val leaderToTp: Map[(String, Int), Seq[TopicAndPartition]] = flip(tpToLeader) val leaders = leaderToTp.keys var result = Map[TopicAndPartition, Seq[LeaderOffset]]() val errs = new Err withBrokers(leaders, errs) { consumer => - val needed: Seq[TopicAndPartition] = leaderToTp((consumer.host, consumer.port)) - val req = OffsetRequest( - needed.map { tp: TopicAndPartition => - tp -> PartitionOffsetRequestInfo(before, maxNumOffsets) - }.toMap - ) + val partitionsToGetOffsets: Seq[TopicAndPartition] = + leaderToTp((consumer.host, consumer.port)) + val reqMap = partitionsToGetOffsets.map { tp: TopicAndPartition => + tp -> PartitionOffsetRequestInfo(before, maxNumOffsets) + }.toMap + val req = OffsetRequest(reqMap) val resp = consumer.getOffsetsBefore(req) val respMap = resp.partitionErrorAndOffsets - needed.foreach { tp: TopicAndPartition => + partitionsToGetOffsets.foreach { tp: TopicAndPartition => respMap.get(tp).foreach { por: PartitionOffsetsResponse => if (por.error == ErrorMapping.NoError) { if (por.offsets.nonEmpty) { @@ -192,7 +198,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { LeaderOffset(consumer.host, consumer.port, off) } } else { - errs.append(new Exception( + errs.append(new SparkException( s"Empty offsets for ${tp}, is ${before} before log beginning?")) } } else { @@ -205,7 +211,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } val missing = topicAndPartitions.diff(result.keySet) - errs.append(new Exception(s"Couldn't find leader offsets for ${missing}")) + errs.append(new SparkException(s"Couldn't find leader offsets for ${missing}")) Left(errs) } } @@ -218,7 +224,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def getConsumerOffsets( groupId: String, topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, Long]] = { + ): Either[Err, Map[TopicAndPartition, Long]] = { getConsumerOffsetMetadata(groupId, topicAndPartitions).right.map { r => r.map { kv => kv._1 -> kv._2.offset @@ -229,7 +235,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def getConsumerOffsetMetadata( groupId: String, topicAndPartitions: Set[TopicAndPartition] - ): Either[Err, Map[TopicAndPartition, OffsetMetadataAndError]] = { + ): Either[Err, Map[TopicAndPartition, OffsetMetadataAndError]] = { var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) val errs = new Err @@ -251,14 +257,14 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } val missing = topicAndPartitions.diff(result.keySet) - errs.append(new Exception(s"Couldn't find consumer offsets for ${missing}")) + errs.append(new SparkException(s"Couldn't find consumer offsets for ${missing}")) Left(errs) } def setConsumerOffsets( groupId: String, offsets: Map[TopicAndPartition, Long] - ): Either[Err, Map[TopicAndPartition, Short]] = { + ): Either[Err, Map[TopicAndPartition, Short]] = { setConsumerOffsetMetadata(groupId, offsets.map { kv => kv._1 -> OffsetMetadataAndError(kv._2) }) @@ -267,7 +273,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { def setConsumerOffsetMetadata( groupId: String, metadata: Map[TopicAndPartition, OffsetMetadataAndError] - ): Either[Err, Map[TopicAndPartition, Short]] = { + ): Either[Err, Map[TopicAndPartition, Short]] = { var result = Map[TopicAndPartition, Short]() val req = OffsetCommitRequest(groupId, metadata) val errs = new Err @@ -290,7 +296,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } val missing = topicAndPartitions.diff(result.keySet) - errs.append(new Exception(s"Couldn't set offsets for ${missing}")) + errs.append(new SparkException(s"Couldn't set offsets for ${missing}")) Left(errs) } diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index 46f2b386586ea..f25669c7647a5 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -244,7 +244,6 @@ object KafkaUtils { * @param messageHandler function for translating each message into the desired type * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) * starting point of the stream - * @param maxRetries maximum number of times in a row to retry getting leaders' offsets */ @Experimental def createNewStream[ @@ -256,11 +255,10 @@ object KafkaUtils { ssc: StreamingContext, kafkaParams: Map[String, String], fromOffsets: Map[TopicAndPartition, Long], - messageHandler: MessageAndMetadata[K, V] => R, - maxRetries: Int + messageHandler: MessageAndMetadata[K, V] => R ): InputDStream[R] = { new DeterministicKafkaInputDStream[K, V, U, T, R]( - ssc, kafkaParams, fromOffsets, messageHandler, maxRetries) + ssc, kafkaParams, fromOffsets, messageHandler) } /** @@ -316,7 +314,7 @@ object KafkaUtils { (tp, lo.offset) } new DeterministicKafkaInputDStream[K, V, U, T, (K, V)]( - ssc, kafkaParams, fromOffsets, messageHandler, 1) + ssc, kafkaParams, fromOffsets, messageHandler) }).fold( errs => throw new SparkException(errs.mkString("\n")), ok => ok From 8991017cdf6c33527cbabab5dcda29519579692f Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Mon, 2 Feb 2015 21:33:06 -0600 Subject: [PATCH 38/42] [SPARK-4964] formatting --- .../DeterministicKafkaInputDStream.scala | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala index ba1ed39c980e9..2832107857d81 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala @@ -32,24 +32,26 @@ import org.apache.spark.streaming.kafka.KafkaCluster.LeaderOffset import org.apache.spark.streaming.{StreamingContext, Time} import org.apache.spark.streaming.dstream._ -/** A stream of {@link org.apache.spark.streaming.kafka.KafkaRDD} where - * each given Kafka topic/partition corresponds to an RDD partition. - * The spark configuration spark.streaming.kafka.maxRatePerPartition gives the maximum number of messages - * per second that each '''partition''' will accept. - * Starting offsets are specified in advance, - * and this DStream is not responsible for committing offsets, - * so that you can control exactly-once semantics. - * For an easy interface to Kafka-managed offsets, - * see {@link org.apache.spark.streaming.kafka.KafkaCluster} - * @param kafkaParams Kafka - * configuration parameters. - * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), - * NOT zookeeper servers, specified in host1:port1,host2:port2 form. - * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) - * starting point of the stream - * @param messageHandler function for translating each message into the desired type - * @param maxRetries maximum number of times in a row to retry getting leaders' offsets - */ +/** + * A stream of {@link org.apache.spark.streaming.kafka.KafkaRDD} where + * each given Kafka topic/partition corresponds to an RDD partition. + * The spark configuration spark.streaming.kafka.maxRatePerPartition gives the maximum number + * of messages + * per second that each '''partition''' will accept. + * Starting offsets are specified in advance, + * and this DStream is not responsible for committing offsets, + * so that you can control exactly-once semantics. + * For an easy interface to Kafka-managed offsets, + * see {@link org.apache.spark.streaming.kafka.KafkaCluster} + * @param kafkaParams Kafka + * configuration parameters. + * Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), + * NOT zookeeper servers, specified in host1:port1,host2:port2 form. + * @param fromOffsets per-topic/partition Kafka offsets defining the (inclusive) + * starting point of the stream + * @param messageHandler function for translating each message into the desired type + * @param maxRetries maximum number of times in a row to retry getting leaders' offsets + */ private[streaming] class DeterministicKafkaInputDStream[ K: ClassTag, From 0df3ebe1eed5b149c03a828db621fbc60e5555df Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 3 Feb 2015 11:34:15 -0600 Subject: [PATCH 39/42] [SPARK-4964] add comments per pwendell / dibbhatt --- .../org/apache/spark/streaming/kafka/KafkaCluster.scala | 4 ++++ .../org/apache/spark/streaming/kafka/OffsetRange.scala | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala index 54c8fe4821ce7..068120891f9fa 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala @@ -221,6 +221,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI // scalastyle:on + /** Requires Kafka >= 0.8.1.1 */ def getConsumerOffsets( groupId: String, topicAndPartitions: Set[TopicAndPartition] @@ -232,6 +233,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { } } + /** Requires Kafka >= 0.8.1.1 */ def getConsumerOffsetMetadata( groupId: String, topicAndPartitions: Set[TopicAndPartition] @@ -261,6 +263,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { Left(errs) } + /** Requires Kafka >= 0.8.1.1 */ def setConsumerOffsets( groupId: String, offsets: Map[TopicAndPartition, Long] @@ -270,6 +273,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { }) } + /** Requires Kafka >= 0.8.1.1 */ def setConsumerOffsetMetadata( groupId: String, metadata: Map[TopicAndPartition, OffsetMetadataAndError] diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala index 132bd81e2364d..334c12e4627b4 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala @@ -42,9 +42,6 @@ final class OffsetRange private( } object OffsetRange { - private[spark] - type OffsetRangeTuple = (String, Int, Long, Long) - def create(topic: String, partition: Int, fromOffset: Long, untilOffset: Long): OffsetRange = new OffsetRange(topic, partition, fromOffset, untilOffset) @@ -63,6 +60,10 @@ object OffsetRange { untilOffset: Long): OffsetRange = new OffsetRange(topicAndPartition.topic, topicAndPartition.partition, fromOffset, untilOffset) + /** this is to avoid ClassNotFoundException during checkpoint restore */ + private[spark] + type OffsetRangeTuple = (String, Int, Long, Long) + private[streaming] def apply(t: OffsetRangeTuple) = new OffsetRange(t._1, t._2, t._3, t._4) From 8c31855cf6b7327c6b6611e715457ba15bb79355 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 3 Feb 2015 19:47:00 -0600 Subject: [PATCH 40/42] [SPARK-4964] remove HasOffsetRanges interface from return types --- .../scala/org/apache/spark/streaming/kafka/KafkaUtils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index f25669c7647a5..eef729c4f9b9b 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -171,7 +171,7 @@ object KafkaUtils { sc: SparkContext, kafkaParams: Map[String, String], offsetRanges: Array[OffsetRange] - ): RDD[(K, V)] with HasOffsetRanges = { + ): RDD[(K, V)] = { val messageHandler = (mmd: MessageAndMetadata[K, V]) => (mmd.key, mmd.message) val kc = new KafkaCluster(kafkaParams) val topics = offsetRanges.map(o => TopicAndPartition(o.topic, o.partition)).toSet @@ -207,7 +207,7 @@ object KafkaUtils { offsetRanges: Array[OffsetRange], leaders: Array[Leader], messageHandler: MessageAndMetadata[K, V] => R - ): RDD[R] with HasOffsetRanges = { + ): RDD[R] = { val leaderMap = leaders .map(l => TopicAndPartition(l.topic, l.partition) -> (l.host, l.port)) From 59e29f61cd6a730eeea4e47a5316cbbe47615618 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Tue, 3 Feb 2015 20:19:45 -0600 Subject: [PATCH 41/42] [SPARK-4964] settle on "Direct" as a naming convention for the new stream --- ...kaInputDStream.scala => DirectKafkaInputDStream.scala} | 6 +++--- .../org/apache/spark/streaming/kafka/KafkaUtils.scala | 8 ++++---- ...aNewStreamSuite.scala => KafkaDirectStreamSuite.scala} | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) rename external/kafka/src/main/scala/org/apache/spark/streaming/kafka/{DeterministicKafkaInputDStream.scala => DirectKafkaInputDStream.scala} (96%) rename external/kafka/src/test/scala/org/apache/spark/streaming/kafka/{KafkaNewStreamSuite.scala => KafkaDirectStreamSuite.scala} (93%) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DirectKafkaInputDStream.scala similarity index 96% rename from external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala rename to external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DirectKafkaInputDStream.scala index 2832107857d81..c7bca43eb889d 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DeterministicKafkaInputDStream.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/DirectKafkaInputDStream.scala @@ -53,7 +53,7 @@ import org.apache.spark.streaming.dstream._ * @param maxRetries maximum number of times in a row to retry getting leaders' offsets */ private[streaming] -class DeterministicKafkaInputDStream[ +class DirectKafkaInputDStream[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, @@ -68,7 +68,7 @@ class DeterministicKafkaInputDStream[ "spark.streaming.kafka.maxRetries", 1) protected[streaming] override val checkpointData = - new DeterministicKafkaInputDStreamCheckpointData + new DirectKafkaInputDStreamCheckpointData protected val kc = new KafkaCluster(kafkaParams) @@ -129,7 +129,7 @@ class DeterministicKafkaInputDStream[ } private[streaming] - class DeterministicKafkaInputDStreamCheckpointData extends DStreamCheckpointData(this) { + class DirectKafkaInputDStreamCheckpointData extends DStreamCheckpointData(this) { def batchForTime = data.asInstanceOf[mutable.HashMap[ Time, Array[OffsetRange.OffsetRangeTuple]]] diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala index eef729c4f9b9b..f8aa6c5c6263c 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala @@ -246,7 +246,7 @@ object KafkaUtils { * starting point of the stream */ @Experimental - def createNewStream[ + def createDirectStream[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, @@ -257,7 +257,7 @@ object KafkaUtils { fromOffsets: Map[TopicAndPartition, Long], messageHandler: MessageAndMetadata[K, V] => R ): InputDStream[R] = { - new DeterministicKafkaInputDStream[K, V, U, T, R]( + new DirectKafkaInputDStream[K, V, U, T, R]( ssc, kafkaParams, fromOffsets, messageHandler) } @@ -289,7 +289,7 @@ object KafkaUtils { * @param topics names of the topics to consume */ @Experimental - def createNewStream[ + def createDirectStream[ K: ClassTag, V: ClassTag, U <: Decoder[_]: ClassTag, @@ -313,7 +313,7 @@ object KafkaUtils { val fromOffsets = leaderOffsets.map { case (tp, lo) => (tp, lo.offset) } - new DeterministicKafkaInputDStream[K, V, U, T, (K, V)]( + new DirectKafkaInputDStream[K, V, U, T, (K, V)]( ssc, kafkaParams, fromOffsets, messageHandler) }).fold( errs => throw new SparkException(errs.mkString("\n")), diff --git a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaDirectStreamSuite.scala similarity index 93% rename from external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala rename to external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaDirectStreamSuite.scala index f78695ba30697..0891ce344f16a 100644 --- a/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaNewStreamSuite.scala +++ b/external/kafka/src/test/scala/org/apache/spark/streaming/kafka/KafkaDirectStreamSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.SparkConf import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.{Milliseconds, StreamingContext} -class KafkaNewStreamSuite extends KafkaStreamSuiteBase with BeforeAndAfter with Eventually { +class KafkaDirectStreamSuite extends KafkaStreamSuiteBase with BeforeAndAfter with Eventually { val sparkConf = new SparkConf() .setMaster("local[4]") .setAppName(this.getClass.getSimpleName) @@ -63,7 +63,7 @@ class KafkaNewStreamSuite extends KafkaStreamSuiteBase with BeforeAndAfter with createTopic(t) produceAndSendMessage(t, data) } - val stream = KafkaUtils.createNewStream[String, String, StringDecoder, StringDecoder]( + val stream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder]( ssc, kafkaParams, topics) var total = 0L; From 1dc29415e3c0ac23a4207513686dfe5ee5ab2725 Mon Sep 17 00:00:00 2001 From: cody koeninger Date: Wed, 4 Feb 2015 10:59:42 -0600 Subject: [PATCH 42/42] [SPARK-4964] silence ConsumerConfig warnings about broker connection props --- .../spark/streaming/kafka/KafkaCluster.scala | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala index 068120891f9fa..ccc62bfe8f057 100644 --- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala +++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala @@ -35,23 +35,14 @@ import org.apache.spark.SparkException */ private[spark] class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { - import KafkaCluster.{Err, LeaderOffset} - - val seedBrokers: Array[(String, Int)] = - kafkaParams.get("metadata.broker.list") - .orElse(kafkaParams.get("bootstrap.servers")) - .getOrElse(throw new SparkException("Must specify metadata.broker.list or bootstrap.servers")) - .split(",").map { hp => - val hpa = hp.split(":") - (hpa(0), hpa(1).toInt) - } + import KafkaCluster.{Err, LeaderOffset, SimpleConsumerConfig} // ConsumerConfig isn't serializable - @transient private var _config: ConsumerConfig = null + @transient private var _config: SimpleConsumerConfig = null - def config: ConsumerConfig = this.synchronized { + def config: SimpleConsumerConfig = this.synchronized { if (_config == null) { - _config = KafkaCluster.consumerConfig(kafkaParams) + _config = SimpleConsumerConfig(kafkaParams) } _config } @@ -72,7 +63,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = TopicMetadataRequest(TopicMetadataRequest.CurrentVersion, 0, config.clientId, Seq(topic)) val errs = new Err - withBrokers(Random.shuffle(seedBrokers), errs) { consumer => + withBrokers(Random.shuffle(config.seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) resp.topicsMetadata.find(_.topic == topic).flatMap { tm: TopicMetadata => tm.partitionsMetadata.find(_.partitionId == partition) @@ -130,7 +121,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = TopicMetadataRequest( TopicMetadataRequest.CurrentVersion, 0, config.clientId, topics.toSeq) val errs = new Err - withBrokers(Random.shuffle(seedBrokers), errs) { consumer => + withBrokers(Random.shuffle(config.seedBrokers), errs) { consumer => val resp: TopicMetadataResponse = consumer.send(req) // error codes here indicate missing / just created topic, // repeating on a different broker wont be useful @@ -241,7 +232,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { var result = Map[TopicAndPartition, OffsetMetadataAndError]() val req = OffsetFetchRequest(groupId, topicAndPartitions.toSeq) val errs = new Err - withBrokers(Random.shuffle(seedBrokers), errs) { consumer => + withBrokers(Random.shuffle(config.seedBrokers), errs) { consumer => val resp = consumer.fetchOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) @@ -282,7 +273,7 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable { val req = OffsetCommitRequest(groupId, metadata) val errs = new Err val topicAndPartitions = metadata.keySet - withBrokers(Random.shuffle(seedBrokers), errs) { consumer => + withBrokers(Random.shuffle(config.seedBrokers), errs) { consumer => val resp = consumer.commitOffsets(req) val respMap = resp.requestInfo val needed = topicAndPartitions.diff(result.keySet) @@ -332,17 +323,47 @@ object KafkaCluster { case class LeaderOffset(host: String, port: Int, offset: Long) /** - * Make a consumer config without requiring group.id or zookeeper.connect, - * since communicating with brokers also needs common settings such as timeout + * High-level kafka consumers connect to ZK. ConsumerConfig assumes this use case. + * Simple consumers connect directly to brokers, but need many of the same configs. + * This subclass won't warn about missing ZK params, or presence of broker params. */ - def consumerConfig(kafkaParams: Map[String, String]): ConsumerConfig = { - val props = new Properties() - kafkaParams.foreach(param => props.put(param._1, param._2)) - Seq("zookeeper.connect", "group.id").foreach { s => - if (!props.contains(s)) { - props.setProperty(s, "") + private[spark] + class SimpleConsumerConfig private(brokers: String, originalProps: Properties) + extends ConsumerConfig(originalProps) { + val seedBrokers: Array[(String, Int)] = brokers.split(",").map { hp => + val hpa = hp.split(":") + (hpa(0), hpa(1).toInt) + } + } + + private[spark] + object SimpleConsumerConfig { + /** + * Make a consumer config without requiring group.id or zookeeper.connect, + * since communicating with brokers also needs common settings such as timeout + */ + def apply(kafkaParams: Map[String, String]): SimpleConsumerConfig = { + // These keys are from other pre-existing kafka configs for specifying brokers, accept either + val brokers = kafkaParams.get("metadata.broker.list") + .orElse(kafkaParams.get("bootstrap.servers")) + .getOrElse(throw new SparkException( + "Must specify metadata.broker.list or bootstrap.servers")) + + val props = new Properties() + kafkaParams.foreach { case (key, value) => + // prevent warnings on parameters ConsumerConfig doesn't know about + if (key != "metadata.broker.list" && key != "bootstrap.servers") { + props.put(key, value) + } + } + + Seq("zookeeper.connect", "group.id").foreach { s => + if (!props.contains(s)) { + props.setProperty(s, "") + } } + + new SimpleConsumerConfig(brokers, props) } - new ConsumerConfig(props) } }