diff --git a/Rakefile b/Rakefile index d56f7cd..d52f6c3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,130 +1,14 @@ -require 'ant' +load 'lib/tasks/red_storm.rake' + +task :default => :spec begin - # will work from gem, since lib dir is in gem require_paths - require 'red_storm' -rescue LoadError - # will work within RedStorm dev project - $:.unshift './lib' - require 'red_storm' -end - -CWD = Dir.pwd -TARGET_DIR = "#{CWD}/target" -TARGET_SRC_DIR = "#{TARGET_DIR}/src" -TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes" -TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency" -TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked" -TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar" - -JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main" -JRUBY_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/lib/red_storm" - -SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples" -DST_EXAMPLES = "#{CWD}/examples" - -task :default => [:clean, :build] - -task :launch, :env, :class_file do |t, args| - gem_home = ENV["GEM_HOME"].to_s.empty? ? " -Djruby.gem.home=`gem env home`" : "" - command = "java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}" - puts("launching #{command}") - system(command) -end - -task :clean do - ant.delete :dir => TARGET_DIR -end - -task :clean_jar do - ant.delete :dir => "#{TARGET_DIR}/cluster-topology.jar" -end - -task :setup do - ant.mkdir :dir => TARGET_DIR - ant.mkdir :dir => TARGET_CLASSES_DIR - ant.mkdir :dir => TARGET_SRC_DIR - ant.path :id => 'classpath' do - fileset :dir => TARGET_DEPENDENCY_DIR - fileset :dir => TARGET_CLASSES_DIR - end -end - -task :install => [:deps, :build] do - puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}") -end - -task :unpack do - system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR}") -end - -task :jar => [:unpack, :clean_jar] do - ant.jar :destfile => TARGET_CLUSTER_JAR do - fileset :dir => TARGET_CLASSES_DIR - fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR - fileset :dir => CWD do - exclude :name => "target/**/*" - end - manifest do - attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher" - end + require 'rspec/core/rake_task' + desc "run specs" + task :spec do + system("ruby -v") + RSpec::Core::RakeTask.new end - puts("\nRedStorm jar completed. Generated jar file #{TARGET_CLUSTER_JAR}") -end - -task :examples do - if File.identical?(SRC_EXAMPLES, DST_EXAMPLES) - STDERR.puts("error: cannot copy examples into itself") - exit(1) - end - if File.exist?(DST_EXAMPLES) - STDERR.puts("error: directory #{DST_EXAMPLES} already exists") - exit(1) - end - - puts("copying examples into #{DST_EXAMPLES}") - system("mkdir #{DST_EXAMPLES}") - system("cp -r #{SRC_EXAMPLES}/* #{DST_EXAMPLES}") - puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}") -end - -task :deps do - system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR}") -end - -task :build => :setup do - # compile the JRuby proxy classes to Java - build_jruby("#{JRUBY_SRC_DIR}/proxy") - - # compile the generated Java proxy classes - build_java_dir("#{TARGET_SRC_DIR}") - - # generate the JRuby topology launcher - build_jruby("#{JRUBY_SRC_DIR}/topology_launcher.rb") - - # compile the JRuby proxy classes - build_java_dir("#{JAVA_SRC_DIR}") - - # compile the JRuby proxy classes - build_java_dir("#{TARGET_SRC_DIR}") -end - -def build_java_dir(source_folder) - puts("\n--> Compiling Java") - ant.javac( - :srcdir => source_folder, - :destdir => TARGET_CLASSES_DIR, - :classpathref => 'classpath', - :source => "1.6", - :target => "1.6", - :debug => "yes", - :includeantruntime => "no", - :verbose => false, - :listfiles => true - ) -end - -def build_jruby(source_path) - puts("\n--> Compiling JRuby") - system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.5.3.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}") +rescue NameError, LoadError => e + puts e end diff --git a/lib/red_storm/application.rb b/lib/red_storm/application.rb index 8fda20a..b632f8b 100644 --- a/lib/red_storm/application.rb +++ b/lib/red_storm/application.rb @@ -3,6 +3,7 @@ require 'rake' module RedStorm class Application + TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" def usage puts("Usage: redstorm install|examples|jar") @@ -12,13 +13,11 @@ module RedStorm def run(args) if args.size > 0 - if ["install", "examples", "jar"].include?(args[0]) - task = args.shift - load("#{RedStorm::REDSTORM_HOME}/Rakefile") - Rake::Task[task].invoke(args) + load(TASKS_FILE) + Rake::Task[args.shift].invoke(*args) elsif args.size == 2 && ["local", "cluster"].include?(args[0]) && File.exist?(args[1]) - load("#{RedStorm::REDSTORM_HOME}/Rakefile") + load(TASKS_FILE) Rake::Task['launch'].invoke(*args) else usage diff --git a/lib/red_storm/simple_bolt.rb b/lib/red_storm/simple_bolt.rb index b515140..72b3f07 100644 --- a/lib/red_storm/simple_bolt.rb +++ b/lib/red_storm/simple_bolt.rb @@ -62,7 +62,7 @@ module RedStorm private def self.fields - @fields + @fields ||= [] end def self.receive_block diff --git a/lib/red_storm/simple_spout.rb b/lib/red_storm/simple_spout.rb index 41fffc5..139478f 100644 --- a/lib/red_storm/simple_spout.rb +++ b/lib/red_storm/simple_spout.rb @@ -5,6 +5,10 @@ module RedStorm # DSL class methods + def self.set(options = {}) + self.spout_options.merge!(options) + end + def self.output_fields(*fields) @fields = fields.map(&:to_s) end @@ -33,10 +37,6 @@ module RedStorm @fail_block = block_given? ? fail_block : lambda {|msg_id| self.send(method_name, msg_id)} end - def self.set(options = {}) - self.spout_options.merge!(options) - end - # DSL instance methods def emit(*values) @@ -87,7 +87,7 @@ module RedStorm private def self.fields - @fields + @fields ||= [] end def self.send_block diff --git a/lib/tasks/red_storm.rake b/lib/tasks/red_storm.rake new file mode 100644 index 0000000..2a3170a --- /dev/null +++ b/lib/tasks/red_storm.rake @@ -0,0 +1,128 @@ +require 'ant' + +begin + # will work from gem, since lib dir is in gem require_paths + require 'red_storm' +rescue LoadError + # will work within RedStorm dev project + $:.unshift './lib' + require 'red_storm' +end + +CWD = Dir.pwd +TARGET_DIR = "#{CWD}/target" +TARGET_SRC_DIR = "#{TARGET_DIR}/src" +TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes" +TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency" +TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked" +TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar" + +JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main" +JRUBY_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/lib/red_storm" + +SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples" +DST_EXAMPLES = "#{CWD}/examples" + +task :launch, :env, :class_file do |t, args| + gem_home = ENV["GEM_HOME"].to_s.empty? ? " -Djruby.gem.home=`gem env home`" : "" + command = "java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}" + puts("launching #{command}") + system(command) +end + +task :clean do + ant.delete :dir => TARGET_DIR +end + +task :clean_jar do + ant.delete :dir => "#{TARGET_DIR}/cluster-topology.jar" +end + +task :setup do + ant.mkdir :dir => TARGET_DIR + ant.mkdir :dir => TARGET_CLASSES_DIR + ant.mkdir :dir => TARGET_SRC_DIR + ant.path :id => 'classpath' do + fileset :dir => TARGET_DEPENDENCY_DIR + fileset :dir => TARGET_CLASSES_DIR + end +end + +task :install => [:deps, :build] do + puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}") +end + +task :unpack do + system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR}") +end + +task :jar => [:unpack, :clean_jar] do + ant.jar :destfile => TARGET_CLUSTER_JAR do + fileset :dir => TARGET_CLASSES_DIR + fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR + fileset :dir => CWD do + exclude :name => "target/**/*" + end + manifest do + attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher" + end + end + puts("\nRedStorm jar completed. Generated jar file #{TARGET_CLUSTER_JAR}") +end + +task :examples do + if File.identical?(SRC_EXAMPLES, DST_EXAMPLES) + STDERR.puts("error: cannot copy examples into itself") + exit(1) + end + if File.exist?(DST_EXAMPLES) + STDERR.puts("error: directory #{DST_EXAMPLES} already exists") + exit(1) + end + + puts("copying examples into #{DST_EXAMPLES}") + system("mkdir #{DST_EXAMPLES}") + system("cp -r #{SRC_EXAMPLES}/* #{DST_EXAMPLES}") + puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}") +end + +task :deps do + system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR}") +end + +task :build => :setup do + # compile the JRuby proxy classes to Java + build_jruby("#{JRUBY_SRC_DIR}/proxy") + + # compile the generated Java proxy classes + build_java_dir("#{TARGET_SRC_DIR}") + + # generate the JRuby topology launcher + build_jruby("#{JRUBY_SRC_DIR}/topology_launcher.rb") + + # compile the JRuby proxy classes + build_java_dir("#{JAVA_SRC_DIR}") + + # compile the JRuby proxy classes + build_java_dir("#{TARGET_SRC_DIR}") +end + +def build_java_dir(source_folder) + puts("\n--> Compiling Java") + ant.javac( + :srcdir => source_folder, + :destdir => TARGET_CLASSES_DIR, + :classpathref => 'classpath', + :source => "1.6", + :target => "1.6", + :debug => "yes", + :includeantruntime => "no", + :verbose => false, + :listfiles => true + ) +end + +def build_jruby(source_path) + puts("\n--> Compiling JRuby") + system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.5.3.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}") +end diff --git a/spec/red_storm/simple_spout_spec.rb b/spec/red_storm/simple_spout_spec.rb new file mode 100644 index 0000000..09e7e42 --- /dev/null +++ b/spec/red_storm/simple_spout_spec.rb @@ -0,0 +1,280 @@ +require 'spec_helper' +require 'red_storm/simple_spout' + +describe RedStorm::SimpleSpout do + + describe "interface" do + it "should implement spout proxy" do + spout = RedStorm::SimpleSpout.new + spout.should respond_to :next_tuple + spout.should respond_to :open + spout.should respond_to :close + spout.should respond_to :declare_output_fields + spout.should respond_to :is_distributed + spout.should respond_to :ack + spout.should respond_to :fail + end + + it "should implement dsl statement" do + RedStorm::SimpleSpout.should respond_to :set + RedStorm::SimpleSpout.should respond_to :output_fields + RedStorm::SimpleSpout.should respond_to :on_init + RedStorm::SimpleSpout.should respond_to :on_close + RedStorm::SimpleSpout.should respond_to :on_send + RedStorm::SimpleSpout.should respond_to :on_ack + RedStorm::SimpleSpout.should respond_to :on_fail + end + end + + describe "dsl" do + + describe "set statement" do + DEFAULT_SPOUT_OPTIONS = {:is_distributed => false} + + it "should have default options" do + RedStorm::SimpleSpout.send(:is_distributed?).should be_false + end + + it "should parse options" do + class IsDistributedClass < RedStorm::SimpleSpout + set :is_distributed => true + end + IsDistributedClass.send(:spout_options).should == DEFAULT_SPOUT_OPTIONS.merge(:is_distributed => true) + IsDistributedClass.send(:is_distributed?).should be_true + end + end + + describe "output_field statement" do + it "should parse single argument" do + class Test1 < RedStorm::SimpleSpout + output_fields :f1 + end + test1 = Test1.new + Test1.send(:fields).should == ["f1"] + end + + it "should parse multiple arguments" do + class Test2 < RedStorm::SimpleSpout + output_fields :f1, :f2 + end + Test2.send(:fields).should == ["f1", "f2"] + end + + it "should parse string and symbol arguments" do + class Test3 < RedStorm::SimpleSpout + output_fields :f1, "f2" + end + Test3.send(:fields).should == ["f1", "f2"] + end + + it "should not share state over mutiple classes" do + class Test4 < RedStorm::SimpleSpout + output_fields :f1 + end + class Test5 < RedStorm::SimpleSpout + output_fields :f2 + end + RedStorm::SimpleSpout.send(:fields).should == [] + Test4.send(:fields).should == ["f1"] + Test5.send(:fields).should == ["f2"] + end + end + + describe "on_send statement" do + DEFAULT_SEND_OPTIONS = {:emit => true} + + it "should emit by defaut" do + RedStorm::SimpleSpout.send(:emit?).should be_true + end + + describe "with block argument" do + + it "should parse without options" do + class BlockArgument1 < RedStorm::SimpleSpout + on_send {self.test_method} + end + + BlockArgument1.send_options.should == DEFAULT_SEND_OPTIONS + + spout = BlockArgument1.new + spout.should_receive(:test_method) + BlockArgument1.should_receive(:emit?).and_return(false) + spout.next_tuple + end + + it "should parse :emit option" do + class BlockArgument2 < RedStorm::SimpleSpout + on_send :emit => false do + self.test_method + end + end + + BlockArgument2.send_options.should == DEFAULT_SEND_OPTIONS.merge(:emit => false) + BlockArgument2.send(:emit?).should be_false + + spout = BlockArgument2.new + spout.should_receive(:test_method) + spout.next_tuple + end + end + + describe "with method name" do + + it "should parse without options" do + class MethodName1 < RedStorm::SimpleSpout + on_send :test_method + end + + MethodName1.send_options.should == DEFAULT_SEND_OPTIONS + + spout = MethodName1.new + spout.should_receive(:test_method) + MethodName1.should_receive(:emit?).and_return(false) + spout.next_tuple + end + + it "should parse :emit option" do + class MethodName2 < RedStorm::SimpleSpout + on_send :test_method, :emit => false + end + + MethodName2.send_options.should == DEFAULT_SEND_OPTIONS.merge(:emit => false) + MethodName2.send(:emit?).should be_false + + spout = MethodName2.new + spout.should_receive(:test_method) + spout.next_tuple + end + end + end + + describe "on_init statement" do + + it "should parse block argument" do + class OnInitBlockArgument1 < RedStorm::SimpleSpout + on_init {self.test_block_call} + end + + spout = OnInitBlockArgument1.new + spout.should_receive(:test_block_call) + spout.open(nil, nil, nil) + end + + it "should parse method name" do + class OnInitMethodName1 < RedStorm::SimpleSpout + on_init :test_method + end + + spout = OnInitMethodName1.new + spout.should_receive(:test_method) + spout.open(nil, nil, nil) + end + end + + describe "on_close statement" do + + it "should parse block argument" do + class OnCloseBlockArgument1 < RedStorm::SimpleSpout + on_close {self.test_block_call} + end + + spout = OnCloseBlockArgument1.new + spout.should_receive(:test_block_call) + spout.close + end + + it "should parse method name" do + class OnCloseMethodName1 < RedStorm::SimpleSpout + on_close :test_method + end + + spout = OnCloseMethodName1.new + spout.should_receive(:test_method) + spout.close + end + end + + describe "on_ack statement" do + + it "should parse block argument" do + class OnAckBlockArgument1 < RedStorm::SimpleSpout + on_ack {|msg_id| self.test_block_call(msg_id)} + end + + spout = OnAckBlockArgument1.new + spout.should_receive(:test_block_call).with("test") + spout.ack("test") + end + + it "should parse method name" do + class OnAckMethodName1 < RedStorm::SimpleSpout + on_ack :test_method + end + + spout = OnAckMethodName1.new + spout.should_receive(:test_method).with("test") + spout.ack("test") + end + end + + describe "on_fail statement" do + + it "should parse block argument" do + class OnFailBlockArgument1 < RedStorm::SimpleSpout + on_fail {|msg_id| self.test_block_call(msg_id)} + end + + spout = OnFailBlockArgument1.new + spout.should_receive(:test_block_call).with("test") + spout.fail("test") + end + + it "should parse method name" do + class OnFailMethodName1 < RedStorm::SimpleSpout + on_fail :test_method + end + + spout = OnFailMethodName1.new + spout.should_receive(:test_method).with("test") + spout.fail("test") + end + end + + end + + describe "spout" do + + describe "next_tuple" do + + it "should auto enit on single value output" do + class SpoutNextTuple1 < RedStorm::SimpleSpout + on_send {"output"} + end + collector = mock("Collector") + + class RedStorm::Values; end + RedStorm::Values.should_receive(:new).with("output").and_return("values") + collector.should_receive(:emit).with("values") + + spout = SpoutNextTuple1.new + spout.open(nil, nil, collector) + spout.next_tuple + end + + it "should auto enit on multiple values output" do + class SpoutNextTuple2 < RedStorm::SimpleSpout + on_send {["output1", "output2"]} + end + collector = mock("Collector") + + class RedStorm::Values; end + RedStorm::Values.should_receive(:new).with("output1", "output2").and_return("values") + collector.should_receive(:emit).with("values") + + spout = SpoutNextTuple2.new + spout.open(nil, nil, collector) + spout.next_tuple + end + end + end +end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..6fa3e2b --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,4 @@ +$:.unshift File.dirname(__FILE__) + '/../lib/' +$:.unshift File.dirname(__FILE__) + '/../spec' + +require 'rspec'