All,

I have resurrected some old code that includes a Kafka Producer class as well as a storm topology that includes a Kafka Spout to ingest the messages coming from the aforementioned producer.

I was using the storm-kafka library with Scala 2.11, but when I changed to the newer code base, which is using Scala 2.12, I found that the older library wouldn't work...thus I wanted to switch to the new storm-kafka-client, but am not sure what version I should be using.  here is my build.sbt file, as well as the error messages I am seeing when the Spout actually runs (my favorite assistant, Google, tells me it is due to a version mismatch between Storm and Kafka).  I am using HDP 2.6.2, for whatever that is worth...

java.lang.NoSuchMethodError: kafka.javaapi.consumer.SimpleConsumer.<init>(Ljava/lang/String;IIILjava/lang/String;Ljava/lang/String;)


name := "apachestormstreaming"

version := "0.1"

scalaVersion := "2.12.4"

scalacOptions := Seq("-unchecked", "-deprecation")

resolvers ++= Seq(
"clojars" at "http://clojars.org/repo/",
"HDP" at "http://repo.hortonworks.com/content/repositories/releases/",
"Hortonworks Jetty" at "http://repo.hortonworks.com/content/repositories/jetty-hadoop/"
)

libraryDependencies ++= Seq(
"org.apache.storm" % "flux-core" % "1.1.0.2.6.2.0-205",
"org.apache.storm" % "storm-core" % "1.1.0.2.6.2.0-205" % Provided
exclude("org.slf4j", "slf4j-log4j12")
exclude("log4j","log4j"),
"org.pcap4j" % "pcap4j-core" % "2.0.0-alpha",
"org.pcap4j" % "pcap4j-packetfactory-static" % "2.0.0-alpha",
"org.apache.hadoop" % "hadoop-common" % "2.7.3.2.6.2.0-205"
exclude("org.slf4j", "slf4j-log4j12")
exclude("log4j","log4j")
exclude("commons-beanutils", "commons-beanutils-core")
exclude("commons-beanutils", "commons-beanutils")
exclude("commons-collections", "commons-collections"),
"org.apache.storm" % "storm-kafka-client" % "1.1.0.2.6.2.0-205"
exclude("org.apache.kafka","kafka-clients")
exclude("org.slf4j", "slf4j-log4j12")
exclude("log4j","log4j")
exclude("org.apache.zookeeper","zookeeper"),
"org.apache.kafka" % "kafka-clients" % "0.10.1.1",
"org.apache.kafka" %% "kafka" % "0.10.1.1"
exclude("org.slf4j", "slf4j-log4j12")
exclude("log4j","log4j")
exclude("org.apache.zookeeper","zookeeper"),
"org.apache.hbase" % "hbase-common" % "1.1.2.2.6.2.0-205"
exclude("org.slf4j","slf4j-log4j12")
exclude("log4j","log4j"),
"org.apache.hbase" % "hbase-client" % "1.1.2.2.6.2.0-205"
exclude("org.slf4j","slf4j-log4j12")
exclude("log4j","log4j"),
"com.paulgoldbaum" %% "scala-influxdb-client" % "0.5.2",
"org.apache.commons" % "commons-lang3" % "3.6",
"org.influxdb" % "influxdb-java" % "2.7"
)

assemblyMergeStrategy in assembly := {
case PathList(ps @ _*) if ps.last endsWith "project.clj" =>
MergeStrategy.rename
case PathList(ps @ _*) if ps.last endsWith "UnknownPacketFactory.class" =>
MergeStrategy.rename
case PathList("org","apache","http", _ @ _*) =>
MergeStrategy.first
case PathList("org","apache","commons","codec", _ @ _*) =>
MergeStrategy.first
case PathList("io","netty", _ @ _*) =>
MergeStrategy.last
case PathList(ps @ _*) if ps.last equalsIgnoreCase "io.netty.versions.properties" =>
MergeStrategy.first
case PathList(ps @ _*) if ps.last equalsIgnoreCase "libnetty-transport-native-epoll.so" =>
MergeStrategy.first
case x => val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}