hermann/lib/hermann/producer.rb

require 'thread_safe'

require 'hermann'
require 'hermann/result'

if RUBY_PLATFORM == "java"
  require 'hermann/provider/java_producer'
else
  require 'hermann_rdkafka'
end

module Hermann
  class Producer
    attr_reader :topic, :brokers, :internal, :children

    # Initialize a producer object with a default topic and broker list
    #
    # @param [String] topic The default topic to use for pushing messages
    # @param [Array] brokers An array of "host:port" strings for the brokers
    def initialize(topic, brokers, opts={})
      @topic = topic
      @brokers = ThreadSafe::Array.new(brokers)
      if Hermann.jruby?
        @internal = Hermann::Provider::JavaProducer.new(brokers.join(','), opts)
      else
        @internal = Hermann::Provider::RDKafka::Producer.new(brokers.join(','))
      end
      # We're tracking children so we can make sure that at Producer exit we
      # make a reasonable attempt to clean up outstanding result objects
      @children = ThreadSafe::Array.new
    end

    # @return [Boolean] True if our underlying producer object thinks it's
    #   connected to a Kafka broker
    def connected?
      return @internal.connected?
    end

    # @return [Boolean] True if the underlying producer object has errored
    def errored?
      return @internal.errored?
    end

    def connect(timeout=0)
      return @internal.connect(timeout * 1000)
    end

    # Push a value onto the Kafka topic passed to this +Producer+
    #
    # @param [Object] value A single object to push
    # @param [Hash] opts to pass to push method
    # @option opts [String] :topic The topic to push messages to
    #                       :partition_key The string to partition by
    #
    # @return [Hermann::Result] A future-like object which will store the
    #   result from the broker
    def push(value, opts={})
      topic = opts[:topic] || @topic
      result = nil

      if value.kind_of? Array
        return value.map { |e| self.push(e, opts) }
      end

      if Hermann.jruby?
        result = @internal.push_single(value, topic, opts[:partition_key], nil)
        unless result.nil?
          @children << result
        end
        # Reaping children on the push just to make sure that it does get
        # called correctly and we don't leak memory
        reap_children
      else
        # Ticking reactor to make sure that we don't inadvertantly let the
        # librdkafka callback queue overflow
        tick_reactor
        result = create_result
        @internal.push_single(value, topic, opts[:partition_key].to_s, result)
      end

      return result
    end

    # Create a +Hermann::Result+ that is tracked in the Producer's children
    # array
    #
    # @return [Hermann::Result] A new, unused, result
    def create_result
      @children << Hermann::Result.new(self)
      return @children.last
    end

    # Tick the underlying librdkafka reacter and clean up any unreaped but
    # reapable children results
    #
    # @param [FixNum] timeout Seconds to block on the internal reactor
    # @return [FixNum] Number of +Hermann::Result+ children reaped
    def tick_reactor(timeout=0)
      begin
        execute_tick(rounded_timeout(timeout))
      rescue StandardError => ex
        @children.each do |child|
          # Skip over any children that should already be reaped for other
          # reasons
          next if (Hermann.jruby? ? child.fulfilled? : child.completed?)

          # Propagate errors to the remaining children
          child.internal_set_error(ex)
        end
      end

      # Reaping the children at this point will also reap any children marked
      # as errored by an exception out of #execute_tick
      return reap_children
    end

    # @return [FixNum] number of children reaped
    def reap_children
      # Filter all children who are no longer pending/fulfilled
      total_children = @children.size

      @children = @children.reject { |c| Hermann.jruby? ? c.fulfilled? : c.completed? }

      return (total_children - children.size)
    end

    private

    def rounded_timeout(timeout)
      # Handle negative numbers, those can be zero
      return 0 if (timeout < 0)
      # Since we're going to sleep for each second, round any potential floats
      # off
      return timeout.round if timeout.kind_of?(Float)
      return timeout
    end

    # Perform the actual reactor tick
    # @raises [StandardError] in case of underlying failures in librdkafka
    def execute_tick(timeout)
      if timeout == 0
        @internal.tick(0)
      else
        (timeout * 2).times do
          # We're going to Thread#sleep in Ruby to avoid a
          # pthread_cond_timedwait(3) inside of librdkafka
          events = @internal.tick(0)
          # If we find events, break out early
          break if events > 0
          sleep 0.5
        end
      end
    end
  end
end