Bundle a known good version of librdkafka (0.8.4) with Hermann

This reduces the need for the librdkafka system dependency and results in
statically linking librdkafka.a against the C extension.

There's a few hacks in extconf.rb specifically to deal with librdkafka's build
system, as well as apply checksumming to the downloaded file

Fixes #8

Gemfile

@@ -1,8 +1,12 @@
 source "https://rubygems.org"
 
-gem 'rake'
-gem 'rake-compiler'
-gem 'pry'
+gemspec
+
+group :development do
+  gem 'rake'
+  gem 'rake-compiler'
+  gem 'pry'
+end
 
 group :test do
   gem 'rspec', '~> 3.0.0'

Gemfile.lock

@@ -1,9 +1,16 @@
+PATH
+  remote: .
+  specs:
+    hermann (0.16.0)
+      mini_portile (~> 0.6.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.0)
     diff-lcs (1.2.5)
     method_source (0.8.2)
+    mini_portile (0.6.0)
     pry (0.9.12.6)
       coderay (~> 1.0)
       method_source (~> 0.8)
@@ -33,6 +40,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  hermann!
   pry
   rake
   rake-compiler

Rakefile

@@ -1,4 +1,5 @@
 require 'rubygems'
+require 'fileutils'
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
 require 'rake/extensiontask'
@@ -20,6 +21,13 @@ namespace :spec do
   end
 end
 
+desc 'Remove the entire ./tmp directory'
+task :removetmp do
+  FileUtils.rm_rf('tmp')
+end
+
+
 task :build => [:compile]
+task :clean => [:removetmp]
 task :default => [:clean, :build, :spec]
 

ext/hermann/extconf.rb

@@ -1,34 +1,74 @@
 # External configuration for Hermann Gem
 
+require 'rubygems'
 require 'mkmf'
+require 'mini_portile'
+require 'digest/md5'
 
 RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
 
 LIBDIR      = RbConfig::CONFIG['libdir']
 INCLUDEDIR  = RbConfig::CONFIG['includedir']
+BASE_DIR = File.expand_path(File.dirname(__FILE__) + '/../../')
 
 puts "Library Dir: #{LIBDIR}\n"
 puts "Include Dir: #{INCLUDEDIR}"
 
-HEADER_DIRS = [INCLUDEDIR]
+################################################################################
+#   MiniPortile overrides
+################################################################################
+# RdKafkaRecipe is a class that adds some librdkafka specific customizations to
+# make sure that we can safely build librdkafka when installing this gem
+class RdKafkaRecipe < MiniPortile
+  attr_accessor :checksum
 
-LIB_DIRS = [LIBDIR, './librdkafka']
+  def configure
+    execute('configure', %Q(bash configure #{computed_options}))
+  end
 
+  def configured?
+    File.exists?(File.join(work_path, 'Makefile.config'))
+  end
+
+  # Overriding this from MiniPortile because it includes autoconf defaults that
+  # don't apply to librdkafka's mklove-based configure script
+  def configure_defaults
+    []
+  end
+
+  def download_file(url, full_path, count=3)
+    super(url, full_path, count)
+
+    # Support some simple checksumming
+    unless Digest::MD5.hexdigest(File.read(full_path)) == checksum
+      raise 'Checksum error!'
+    end
+  end
+end
+################################################################################
+
+librdkafka = RdKafkaRecipe.new('librdkafka', '0.8.4')
+librdkafka.files = ["https://github.com/edenhill/librdkafka/archive/#{librdkafka.version}.tar.gz"]
+librdkafka.checksum = '28a3252fd0f31d4a38bea9cd25083a06'
+librdkafka.patch_files = Dir["#{File.join(BASE_DIR, 'ext', 'patches', 'librdkafka')}/*.patch"]
+checkpoint = ".librdkafka.#{librdkafka.version}.cooked"
+
+unless File.exists?(checkpoint)
+  librdkafka.cook
+  File.open(checkpoint, 'w+') do |f|
+    f.write("Cooked: #{Time.now}\n")
+  end
+end
+
+librdkafka.activate
+
+HEADER_DIRS = [INCLUDEDIR, File.join(librdkafka.path, 'include')]
+LIB_DIRS = [LIBDIR]
 dir_config('rdkafka', HEADER_DIRS, LIB_DIRS)
 
-ROOTDIR = File.expand_path(File.dirname(__FILE__) + '/../../')
-RDK = File.join(ROOTDIR, 'librdkafka/target')
-unless find_header('librdkafka/rdkafka.h', File.join(RDK, 'include'))
-  abort "librdkafka not installed"
-end
+# Tell mkmf to staticallly link our mini_portile generated static library,
+# courtesty of:
+#   <http://blog.zachallett.com/howto-ruby-c-extension-with-a-static-library>
+$LOCAL_LIBS << File.join(librdkafka.path, 'lib', 'librdkafka.a')
 
-unless find_library('rdkafka', 'rd_kafka_conf_new', File.join(RDK, 'lib'))
-   abort "librdkafka not installed"
-end
-
-#unless have_func "rb_thread_blocking_region"
-#  abort "rb_thread_blocking_region function missing"
-#end
-
-# create_header('hermann_lib.h')
 create_makefile('hermann/hermann_lib')

ext/patches/librdkafka/0006-Update-some-headers-to-include-the-right-headers-to-.patch

@@ -0,0 +1,57 @@
+From 888ca33b571d99e877d665235b822f7c961c8fdb Mon Sep 17 00:00:00 2001
+From: "R. Tyler Croy" <tyler@monkeypox.org>
+Date: Thu, 28 Aug 2014 16:24:04 -0700
+Subject: [PATCH 6/8] Update some headers to include the right headers to build
+ on FreeBSD
+
+---
+ src/rd.h     | 9 +++++++++
+ src/rdaddr.h | 4 ++++
+ 2 files changed, 13 insertions(+)
+
+diff --git a/src/rd.h b/src/rd.h
+index c31501e..4789493 100644
+--- a/src/rd.h
++++ b/src/rd.h
+@@ -37,7 +37,11 @@
+ #include <errno.h>
+ #include <time.h>
+ #include <sys/time.h>
++
++#ifndef __FreeBSD__
++/* alloca(3) is in stdlib on FreeBSD */
+ #include <alloca.h>
++#endif
+ #include <assert.h>
+ #include <pthread.h>
+ 
+@@ -110,6 +114,11 @@
+ # endif
+ #endif /* sun */
+ 
++#ifdef __FreeBSD__
++/* FreeBSD defines be64toh() in sys/endian.h */
++#include <sys/endian.h>
++#endif
++
+ #ifndef be64toh
+ #ifndef __APPLE__
+ #ifndef sun
+diff --git a/src/rdaddr.h b/src/rdaddr.h
+index 0b37354..e55bd55 100644
+--- a/src/rdaddr.h
++++ b/src/rdaddr.h
+@@ -32,6 +32,10 @@
+ #include <arpa/inet.h>
+ #include <netdb.h>
+ 
++#ifdef __FreeBSD__
++#include <sys/socket.h>
++#endif
++
+ /**
+  * rd_sockaddr_inx_t is a union for either ipv4 or ipv6 sockaddrs.
+  * It provides conveniant abstraction of AF_INET* agnostic operations.
+-- 
+1.9.0
+

hermann.gemspec

@@ -23,5 +23,7 @@ SPEC = Gem::Specification.new do |s|
   s.require_paths = ["lib", "ext/hermann"]
   s.rubygems_version = '2.2.2'
 
+  s.add_dependency('mini_portile', '~> 0.6.0')
+
   s.specification_version = 3 if s.respond_to?(:specification_version)
 end

scripts/test-gem-install

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+GEMSET=${HERMANN_GEMSET:-hermantest}
+GEM=./pkg/hermann-0.16.0.gem
+
+######################
+
+
+echo ">> Sourcing RVM"
+source ~/.rvm/scripts/rvm
+
+echo ">> Gemset: ${GEMSET}"
+for ruby in "1.8.7" "1.9.3"; do
+  echo ">> Testing in ${ruby}"
+  rvm use ${ruby}@${GEMSET} --create
+
+  echo ">> Installing ${GEM}"
+  gem install ${GEM} --verbose
+  if [ $? -ne 0 ]; then
+    "FAILFAILFAIL installing ${GEM} on ${ruby}@${GEMSET} FAILFAILFAIL"
+    exit 1
+  fi;
+
+  echo ">>> done, deleting"
+  rvm gemset delete ${GEMSET} --force
+done;