2014-10-29 01:00:29 +00:00
|
|
|
require 'thread_safe'
|
|
|
|
|
2014-08-29 00:14:38 +00:00
|
|
|
require 'hermann'
|
2014-09-03 21:19:09 +00:00
|
|
|
require 'hermann/result'
|
2014-10-03 18:56:14 +00:00
|
|
|
|
|
|
|
if RUBY_PLATFORM == "java"
|
2014-10-06 15:08:30 +00:00
|
|
|
require 'hermann/provider/java_producer'
|
2014-10-03 18:56:14 +00:00
|
|
|
else
|
2015-06-23 16:01:31 +00:00
|
|
|
require 'hermann_rdkafka'
|
2014-10-03 18:56:14 +00:00
|
|
|
end
|
2014-08-29 00:14:38 +00:00
|
|
|
|
|
|
|
module Hermann
|
|
|
|
class Producer
|
2014-09-03 21:19:09 +00:00
|
|
|
attr_reader :topic, :brokers, :internal, :children
|
2014-08-29 00:14:38 +00:00
|
|
|
|
2014-10-14 16:51:24 +00:00
|
|
|
# Initialize a producer object with a default topic and broker list
|
|
|
|
#
|
|
|
|
# @param [String] topic The default topic to use for pushing messages
|
|
|
|
# @param [Array] brokers An array of "host:port" strings for the brokers
|
2014-10-14 17:26:02 +00:00
|
|
|
def initialize(topic, brokers, opts={})
|
2014-08-29 00:14:38 +00:00
|
|
|
@topic = topic
|
2014-10-29 01:13:20 +00:00
|
|
|
@brokers = ThreadSafe::Array.new(brokers)
|
2014-10-29 03:47:25 +00:00
|
|
|
if Hermann.jruby?
|
2014-10-29 23:41:25 +00:00
|
|
|
@internal = Hermann::Provider::JavaProducer.new(brokers.join(','), opts)
|
2014-10-03 18:56:14 +00:00
|
|
|
else
|
2015-06-23 16:01:31 +00:00
|
|
|
@internal = Hermann::Provider::RDKafka::Producer.new(brokers.join(','))
|
2014-10-03 18:56:14 +00:00
|
|
|
end
|
2014-09-03 21:19:09 +00:00
|
|
|
# We're tracking children so we can make sure that at Producer exit we
|
|
|
|
# make a reasonable attempt to clean up outstanding result objects
|
2014-10-29 01:00:29 +00:00
|
|
|
@children = ThreadSafe::Array.new
|
2014-08-29 00:14:38 +00:00
|
|
|
end
|
|
|
|
|
2014-09-09 21:40:00 +00:00
|
|
|
# @return [Boolean] True if our underlying producer object thinks it's
|
|
|
|
# connected to a Kafka broker
|
|
|
|
def connected?
|
|
|
|
return @internal.connected?
|
|
|
|
end
|
|
|
|
|
2014-09-10 00:36:27 +00:00
|
|
|
# @return [Boolean] True if the underlying producer object has errored
|
|
|
|
def errored?
|
|
|
|
return @internal.errored?
|
|
|
|
end
|
|
|
|
|
2014-09-09 21:40:00 +00:00
|
|
|
def connect(timeout=0)
|
|
|
|
return @internal.connect(timeout * 1000)
|
|
|
|
end
|
|
|
|
|
2014-08-29 00:14:38 +00:00
|
|
|
# Push a value onto the Kafka topic passed to this +Producer+
|
|
|
|
#
|
|
|
|
# @param [Object] value A single object to push
|
2014-10-13 21:05:58 +00:00
|
|
|
# @param [Hash] opts to pass to push method
|
2014-10-14 16:51:24 +00:00
|
|
|
# @option opts [String] :topic The topic to push messages to
|
2015-04-03 01:35:10 +00:00
|
|
|
# :partition_key The string to partition by
|
2014-10-13 21:05:58 +00:00
|
|
|
#
|
2014-09-03 21:19:09 +00:00
|
|
|
# @return [Hermann::Result] A future-like object which will store the
|
|
|
|
# result from the broker
|
2014-10-13 21:05:58 +00:00
|
|
|
def push(value, opts={})
|
|
|
|
topic = opts[:topic] || @topic
|
2014-10-14 16:51:24 +00:00
|
|
|
result = nil
|
2014-10-06 15:08:30 +00:00
|
|
|
|
2014-08-29 00:14:38 +00:00
|
|
|
if value.kind_of? Array
|
2014-10-14 17:54:52 +00:00
|
|
|
return value.map { |e| self.push(e, opts) }
|
2014-10-14 16:51:24 +00:00
|
|
|
end
|
|
|
|
|
2014-10-29 03:47:25 +00:00
|
|
|
if Hermann.jruby?
|
2015-09-16 15:54:42 +00:00
|
|
|
result = @internal.push_single(value, topic, opts[:partition_key], nil)
|
2014-10-14 17:19:10 +00:00
|
|
|
unless result.nil?
|
|
|
|
@children << result
|
|
|
|
end
|
2014-10-14 16:51:24 +00:00
|
|
|
# Reaping children on the push just to make sure that it does get
|
|
|
|
# called correctly and we don't leak memory
|
|
|
|
reap_children
|
2014-08-29 00:14:38 +00:00
|
|
|
else
|
2014-10-29 03:47:25 +00:00
|
|
|
# Ticking reactor to make sure that we don't inadvertantly let the
|
|
|
|
# librdkafka callback queue overflow
|
|
|
|
tick_reactor
|
2014-10-14 16:51:24 +00:00
|
|
|
result = create_result
|
2015-08-19 14:47:09 +00:00
|
|
|
@internal.push_single(value, topic, opts[:partition_key].to_s, result)
|
2014-08-29 00:14:38 +00:00
|
|
|
end
|
2014-10-06 15:08:30 +00:00
|
|
|
|
Tie Hermann::Result#value to the underlying reactor to bring values up to Ruby
This ensures that we're getting async values out of librdkafka into the calling
Ruby thread. Currently errors aren't being brought up properly, but we're getting there
Example:
[14:47:30] tyler:Hermann git:(issues/11-producer-feedback*) $ pry -I lib -r 'hermann/producer'
[1] pry(main)> p = Hermann::Producer.new('topic', 'kafka0.REDACTED.com:6667')
=> #<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">
[2] pry(main)> r = p.push('hello world!')
=> #<Hermann::Result:0x00000803b8cb20
@producer=
#<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[#<Hermann::Result:0x00000803b8cb20 ...>],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">,
@reason=nil,
@state=:unfulfilled,
@value=nil>
[3] pry(main)> r.state
=> :unfulfilled
[4] pry(main)> r.value
ticking rdkafka reactor
ticked
=> "hello world!"
[5] pry(main)> r.state
=> :fulfilled
[6] pry(main)> r.rejected?
=> false
[7] pry(main)>
[14:47:56] tyler:Hermann git:(issues/11-producer-feedback*) $
Fixes #11
2014-09-04 21:29:19 +00:00
|
|
|
return result
|
2014-08-29 00:14:38 +00:00
|
|
|
end
|
2014-09-03 21:19:09 +00:00
|
|
|
|
|
|
|
# Create a +Hermann::Result+ that is tracked in the Producer's children
|
|
|
|
# array
|
|
|
|
#
|
|
|
|
# @return [Hermann::Result] A new, unused, result
|
|
|
|
def create_result
|
|
|
|
@children << Hermann::Result.new(self)
|
|
|
|
return @children.last
|
|
|
|
end
|
|
|
|
|
|
|
|
# Tick the underlying librdkafka reacter and clean up any unreaped but
|
|
|
|
# reapable children results
|
|
|
|
#
|
2014-09-09 21:38:30 +00:00
|
|
|
# @param [FixNum] timeout Seconds to block on the internal reactor
|
2014-09-09 18:11:07 +00:00
|
|
|
# @return [FixNum] Number of +Hermann::Result+ children reaped
|
2014-09-03 21:19:09 +00:00
|
|
|
def tick_reactor(timeout=0)
|
2014-09-10 18:01:01 +00:00
|
|
|
begin
|
|
|
|
execute_tick(rounded_timeout(timeout))
|
|
|
|
rescue StandardError => ex
|
|
|
|
@children.each do |child|
|
|
|
|
# Skip over any children that should already be reaped for other
|
|
|
|
# reasons
|
2016-02-23 10:00:38 +00:00
|
|
|
next if (Hermann.jruby? ? child.fulfilled? : child.completed?)
|
|
|
|
|
2014-09-10 18:01:01 +00:00
|
|
|
# Propagate errors to the remaining children
|
|
|
|
child.internal_set_error(ex)
|
2014-09-09 21:38:30 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2014-09-10 18:01:01 +00:00
|
|
|
# Reaping the children at this point will also reap any children marked
|
|
|
|
# as errored by an exception out of #execute_tick
|
2014-09-09 21:38:30 +00:00
|
|
|
return reap_children
|
|
|
|
end
|
|
|
|
|
|
|
|
# @return [FixNum] number of children reaped
|
|
|
|
def reap_children
|
2014-09-03 21:19:09 +00:00
|
|
|
# Filter all children who are no longer pending/fulfilled
|
2014-09-09 18:11:07 +00:00
|
|
|
total_children = @children.size
|
2014-10-15 18:09:16 +00:00
|
|
|
|
2016-02-23 10:00:38 +00:00
|
|
|
@children = @children.reject { |c| Hermann.jruby? ? c.fulfilled? : c.completed? }
|
2014-09-10 18:01:01 +00:00
|
|
|
|
|
|
|
return (total_children - children.size)
|
2014-09-03 21:19:09 +00:00
|
|
|
end
|
Tie Hermann::Result#value to the underlying reactor to bring values up to Ruby
This ensures that we're getting async values out of librdkafka into the calling
Ruby thread. Currently errors aren't being brought up properly, but we're getting there
Example:
[14:47:30] tyler:Hermann git:(issues/11-producer-feedback*) $ pry -I lib -r 'hermann/producer'
[1] pry(main)> p = Hermann::Producer.new('topic', 'kafka0.REDACTED.com:6667')
=> #<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">
[2] pry(main)> r = p.push('hello world!')
=> #<Hermann::Result:0x00000803b8cb20
@producer=
#<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[#<Hermann::Result:0x00000803b8cb20 ...>],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">,
@reason=nil,
@state=:unfulfilled,
@value=nil>
[3] pry(main)> r.state
=> :unfulfilled
[4] pry(main)> r.value
ticking rdkafka reactor
ticked
=> "hello world!"
[5] pry(main)> r.state
=> :fulfilled
[6] pry(main)> r.rejected?
=> false
[7] pry(main)>
[14:47:56] tyler:Hermann git:(issues/11-producer-feedback*) $
Fixes #11
2014-09-04 21:29:19 +00:00
|
|
|
|
2014-09-09 22:30:19 +00:00
|
|
|
private
|
|
|
|
|
|
|
|
def rounded_timeout(timeout)
|
|
|
|
# Handle negative numbers, those can be zero
|
|
|
|
return 0 if (timeout < 0)
|
|
|
|
# Since we're going to sleep for each second, round any potential floats
|
|
|
|
# off
|
|
|
|
return timeout.round if timeout.kind_of?(Float)
|
|
|
|
return timeout
|
Tie Hermann::Result#value to the underlying reactor to bring values up to Ruby
This ensures that we're getting async values out of librdkafka into the calling
Ruby thread. Currently errors aren't being brought up properly, but we're getting there
Example:
[14:47:30] tyler:Hermann git:(issues/11-producer-feedback*) $ pry -I lib -r 'hermann/producer'
[1] pry(main)> p = Hermann::Producer.new('topic', 'kafka0.REDACTED.com:6667')
=> #<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">
[2] pry(main)> r = p.push('hello world!')
=> #<Hermann::Result:0x00000803b8cb20
@producer=
#<Hermann::Producer:0x00000803b3b450
@brokers="kafka0.REDACTED.com:6667",
@children=[#<Hermann::Result:0x00000803b8cb20 ...>],
@internal=#<Hermann::Lib::Producer:0x00000803b3b3d8>,
@topic="topic">,
@reason=nil,
@state=:unfulfilled,
@value=nil>
[3] pry(main)> r.state
=> :unfulfilled
[4] pry(main)> r.value
ticking rdkafka reactor
ticked
=> "hello world!"
[5] pry(main)> r.state
=> :fulfilled
[6] pry(main)> r.rejected?
=> false
[7] pry(main)>
[14:47:56] tyler:Hermann git:(issues/11-producer-feedback*) $
Fixes #11
2014-09-04 21:29:19 +00:00
|
|
|
end
|
2014-09-10 18:01:01 +00:00
|
|
|
|
|
|
|
# Perform the actual reactor tick
|
2014-10-14 17:26:02 +00:00
|
|
|
# @raises [StandardError] in case of underlying failures in librdkafka
|
2014-09-10 18:01:01 +00:00
|
|
|
def execute_tick(timeout)
|
|
|
|
if timeout == 0
|
|
|
|
@internal.tick(0)
|
|
|
|
else
|
|
|
|
(timeout * 2).times do
|
|
|
|
# We're going to Thread#sleep in Ruby to avoid a
|
|
|
|
# pthread_cond_timedwait(3) inside of librdkafka
|
|
|
|
events = @internal.tick(0)
|
|
|
|
# If we find events, break out early
|
|
|
|
break if events > 0
|
|
|
|
sleep 0.5
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2014-08-29 00:14:38 +00:00
|
|
|
end
|
|
|
|
end
|