Refactor more Kafka subscription into OfftopicClient to allow multiple streams at once

WOOOHOOO

Fixes #7
This commit is contained in:
R. Tyler Croy 2014-11-24 21:07:55 -08:00
parent d04d228b28
commit 2677018943
4 changed files with 102 additions and 24 deletions

View File

@ -49,7 +49,12 @@ class KafkaSubscriber {
consumerMap.get(this.topic).each { stream ->
def iterator = stream.iterator()
while (iterator.hasNext()) {
this.callback.call(iterator.next())
def message = iterator.next()
def data = ['raw' : new String(message.message()),
'b64' : message.message().encodeBase64().toString(),
'topic' : message.topic(),
'tstamp' : System.currentTimeMillis()]
this.callback.call(data)
}
}
}

View File

@ -1,9 +1,60 @@
package offtopic
/**
* OfftopicClient coordinates the interactions between KafkaSubscriber objects
* and the websocket interactions
*/
class OfftopicClient {
int clientId = 0
public int clientId = 0
public OfftopicClient() {
private Closure messageCallback = null
private String topicsPattern = null
private ArrayList<KafkaSubscriber> subscribers = null
private Configuration config = null
public OfftopicClient(Configuration configuration) {
this.clientId = new Random().nextInt()
this.config = configuration
this.subscribers = new ArrayList<KafkaSubscriber>()
}
public ArrayList<KafkaSubscriber> getSubscribers() {
return this.subscribers
}
public void createSubscribersFor(String topicsPattern) {
topicsPattern.split("\\+").each { topic ->
if (topic.length() == 0) {
return
}
KafkaSubscriber subscriber = new KafkaSubscriber(Configuration.instance.zookeepers,
topic,
"offtopic-${clientId}")
subscriber.callback = this.messageCallback
this.subscribers.add(subscriber)
}
}
public void setOnMessageCallback(Closure c) {
this.messageCallback = c
}
public void startSubscribers() {
this.subscribers.each { subscriber ->
Thread runner = new Thread({
subscriber.connect()
println "subscriber connected"
subscriber.consume()
println "consume over!"
})
runner.start()
}
}
public void shutdown() {
this.subscribers.each { subscriber ->
subscriber.shutdown()
}
}
}

View File

@ -36,7 +36,6 @@ ratpack {
}
get('topics/:name') {
println offtopic.Configuration.instance['offtopic.zookeepers']
render "Fetching info for ${pathTokens.name}"
}
@ -45,32 +44,23 @@ ratpack {
}
get('topics/:name/websocket') { ctx ->
client = new OfftopicClient()
subscriber = new KafkaSubscriber(Configuration.instance.zookeepers,
pathTokens.name,
"offtopic-${client.clientId}")
runner = new Thread({
subscriber.connect()
println "subscriber connected"
subscriber.consume()
println "consume over!"
})
println "creating thingies"
def client = new OfftopicClient(Configuration.instance)
println "client: ${client}"
websocket(ctx) { ws ->
println "Connected ${ws} ${subscriber}"
subscriber.callback = { msg ->
println "called back with: ${msg}"
ws.send(groovy.json.JsonOutput.toJson(['raw' : new String(msg.message()),
'b64' : msg.message().encodeBase64().toString(),
'topic' : msg.topic(),
'tstamp' : System.currentTimeMillis()]))
println "Connected ${ws}"
client.onMessageCallback = { m ->
println "called back with ${m}"
ws.send(groovy.json.JsonOutput.toJson(m))
println "sent message"
}
runner.start()
client.createSubscribersFor(pathTokens.name)
print "subscribers created for ${pathTokens.name}"
client.startSubscribers()
} connect { sock ->
sock.onClose {
println "closing up ${subscriber}"
subscriber.shutdown()
client.shutdown()
}
sock.onMessage { msg ->
println "client sent ${msg}"

View File

@ -10,4 +10,36 @@ class OfftopicClientSpec extends Specification {
then:
client.clientId != 0
}
def "initialization should create a subscribers ArrayList"() {
when:
def client = new OfftopicClient()
then:
client.subscribers.size() == 0
}
}
class OfftopicClientCreateSubscribersSpec extends Specification {
def client = null
def setup() {
this.client = new OfftopicClient()
}
def "createSubscribersFor with an empty string"() {
when:
this.client.createSubscribersFor('')
then:
this.client.subscribers.size() == 0
}
def "createSubscribersFor with a single topic"() {
when:
this.client.createSubscribersFor('spock-topic')
then:
this.client.subscribers.size() == 1
}
}