mirror of https://github.com/reiseburo/offtopic
Refactor more Kafka subscription into OfftopicClient to allow multiple streams at once
WOOOHOOO Fixes #7
This commit is contained in:
parent
d04d228b28
commit
2677018943
|
@ -49,7 +49,12 @@ class KafkaSubscriber {
|
||||||
consumerMap.get(this.topic).each { stream ->
|
consumerMap.get(this.topic).each { stream ->
|
||||||
def iterator = stream.iterator()
|
def iterator = stream.iterator()
|
||||||
while (iterator.hasNext()) {
|
while (iterator.hasNext()) {
|
||||||
this.callback.call(iterator.next())
|
def message = iterator.next()
|
||||||
|
def data = ['raw' : new String(message.message()),
|
||||||
|
'b64' : message.message().encodeBase64().toString(),
|
||||||
|
'topic' : message.topic(),
|
||||||
|
'tstamp' : System.currentTimeMillis()]
|
||||||
|
this.callback.call(data)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,60 @@
|
||||||
package offtopic
|
package offtopic
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OfftopicClient coordinates the interactions between KafkaSubscriber objects
|
||||||
|
* and the websocket interactions
|
||||||
|
*/
|
||||||
class OfftopicClient {
|
class OfftopicClient {
|
||||||
int clientId = 0
|
public int clientId = 0
|
||||||
|
|
||||||
public OfftopicClient() {
|
private Closure messageCallback = null
|
||||||
|
private String topicsPattern = null
|
||||||
|
private ArrayList<KafkaSubscriber> subscribers = null
|
||||||
|
private Configuration config = null
|
||||||
|
|
||||||
|
public OfftopicClient(Configuration configuration) {
|
||||||
this.clientId = new Random().nextInt()
|
this.clientId = new Random().nextInt()
|
||||||
|
this.config = configuration
|
||||||
|
this.subscribers = new ArrayList<KafkaSubscriber>()
|
||||||
|
}
|
||||||
|
|
||||||
|
public ArrayList<KafkaSubscriber> getSubscribers() {
|
||||||
|
return this.subscribers
|
||||||
|
}
|
||||||
|
|
||||||
|
public void createSubscribersFor(String topicsPattern) {
|
||||||
|
topicsPattern.split("\\+").each { topic ->
|
||||||
|
if (topic.length() == 0) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
KafkaSubscriber subscriber = new KafkaSubscriber(Configuration.instance.zookeepers,
|
||||||
|
topic,
|
||||||
|
"offtopic-${clientId}")
|
||||||
|
subscriber.callback = this.messageCallback
|
||||||
|
this.subscribers.add(subscriber)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setOnMessageCallback(Closure c) {
|
||||||
|
this.messageCallback = c
|
||||||
|
}
|
||||||
|
|
||||||
|
public void startSubscribers() {
|
||||||
|
this.subscribers.each { subscriber ->
|
||||||
|
Thread runner = new Thread({
|
||||||
|
subscriber.connect()
|
||||||
|
println "subscriber connected"
|
||||||
|
subscriber.consume()
|
||||||
|
println "consume over!"
|
||||||
|
})
|
||||||
|
runner.start()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void shutdown() {
|
||||||
|
this.subscribers.each { subscriber ->
|
||||||
|
subscriber.shutdown()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,6 @@ ratpack {
|
||||||
}
|
}
|
||||||
|
|
||||||
get('topics/:name') {
|
get('topics/:name') {
|
||||||
println offtopic.Configuration.instance['offtopic.zookeepers']
|
|
||||||
render "Fetching info for ${pathTokens.name}"
|
render "Fetching info for ${pathTokens.name}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -45,32 +44,23 @@ ratpack {
|
||||||
}
|
}
|
||||||
|
|
||||||
get('topics/:name/websocket') { ctx ->
|
get('topics/:name/websocket') { ctx ->
|
||||||
client = new OfftopicClient()
|
println "creating thingies"
|
||||||
subscriber = new KafkaSubscriber(Configuration.instance.zookeepers,
|
def client = new OfftopicClient(Configuration.instance)
|
||||||
pathTokens.name,
|
println "client: ${client}"
|
||||||
"offtopic-${client.clientId}")
|
|
||||||
runner = new Thread({
|
|
||||||
subscriber.connect()
|
|
||||||
println "subscriber connected"
|
|
||||||
subscriber.consume()
|
|
||||||
println "consume over!"
|
|
||||||
})
|
|
||||||
|
|
||||||
websocket(ctx) { ws ->
|
websocket(ctx) { ws ->
|
||||||
println "Connected ${ws} ${subscriber}"
|
println "Connected ${ws}"
|
||||||
subscriber.callback = { msg ->
|
client.onMessageCallback = { m ->
|
||||||
println "called back with: ${msg}"
|
println "called back with ${m}"
|
||||||
ws.send(groovy.json.JsonOutput.toJson(['raw' : new String(msg.message()),
|
ws.send(groovy.json.JsonOutput.toJson(m))
|
||||||
'b64' : msg.message().encodeBase64().toString(),
|
|
||||||
'topic' : msg.topic(),
|
|
||||||
'tstamp' : System.currentTimeMillis()]))
|
|
||||||
println "sent message"
|
println "sent message"
|
||||||
}
|
}
|
||||||
runner.start()
|
client.createSubscribersFor(pathTokens.name)
|
||||||
|
print "subscribers created for ${pathTokens.name}"
|
||||||
|
client.startSubscribers()
|
||||||
} connect { sock ->
|
} connect { sock ->
|
||||||
sock.onClose {
|
sock.onClose {
|
||||||
println "closing up ${subscriber}"
|
client.shutdown()
|
||||||
subscriber.shutdown()
|
|
||||||
}
|
}
|
||||||
sock.onMessage { msg ->
|
sock.onMessage { msg ->
|
||||||
println "client sent ${msg}"
|
println "client sent ${msg}"
|
||||||
|
|
|
@ -10,4 +10,36 @@ class OfftopicClientSpec extends Specification {
|
||||||
then:
|
then:
|
||||||
client.clientId != 0
|
client.clientId != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def "initialization should create a subscribers ArrayList"() {
|
||||||
|
when:
|
||||||
|
def client = new OfftopicClient()
|
||||||
|
|
||||||
|
then:
|
||||||
|
client.subscribers.size() == 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class OfftopicClientCreateSubscribersSpec extends Specification {
|
||||||
|
def client = null
|
||||||
|
|
||||||
|
def setup() {
|
||||||
|
this.client = new OfftopicClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
def "createSubscribersFor with an empty string"() {
|
||||||
|
when:
|
||||||
|
this.client.createSubscribersFor('')
|
||||||
|
|
||||||
|
then:
|
||||||
|
this.client.subscribers.size() == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
def "createSubscribersFor with a single topic"() {
|
||||||
|
when:
|
||||||
|
this.client.createSubscribersFor('spock-topic')
|
||||||
|
|
||||||
|
then:
|
||||||
|
this.client.subscribers.size() == 1
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue