diff --git a/examples/native/Gemfile b/examples/native/Gemfile deleted file mode 100644 index c646e60..0000000 --- a/examples/native/Gemfile +++ /dev/null @@ -1,2 +0,0 @@ -source :rubygems -gem 'redis' \ No newline at end of file diff --git a/examples/native/cluster_word_count_topology.rb b/examples/native/cluster_word_count_topology.rb index 3fa8a9e..1b4882a 100644 --- a/examples/native/cluster_word_count_topology.rb +++ b/examples/native/cluster_word_count_topology.rb @@ -10,13 +10,13 @@ module RedStorm def start(base_class_path, env) builder = TopologyBuilder.new - builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5) - builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 4).shuffleGrouping('RandomSentenceSpout') - builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) + builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1) + builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout') + builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) conf = Backtype::Config.new conf.setDebug(true) - conf.setNumWorkers(20); + conf.setNumWorkers(4); conf.setMaxSpoutPending(1000); StormSubmitter.submitTopology("word_count", conf, builder.createTopology); end diff --git a/examples/native/local_exclamation_topology.rb b/examples/native/local_exclamation_topology.rb index ac061ad..3175c3d 100644 --- a/examples/native/local_exclamation_topology.rb +++ b/examples/native/local_exclamation_topology.rb @@ -1,6 +1,6 @@ java_import 'backtype.storm.testing.TestWordSpout' -require 'lib/red_storm' +require 'red_storm' require 'examples/native/exclamation_bolt' # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt @@ -13,9 +13,9 @@ module RedStorm def start(base_class_path, env) builder = TopologyBuilder.new - builder.setSpout('TestWordSpout', TestWordSpout.new, 10) - builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('TestWordSpout') - builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('ExclamationBolt1') + builder.setSpout('TestWordSpout', TestWordSpout.new, 1) + builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('TestWordSpout') + builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('ExclamationBolt1') conf = Backtype::Config.new conf.setDebug(true) diff --git a/examples/native/local_exclamation_topology2.rb b/examples/native/local_exclamation_topology2.rb index 89730b0..753797b 100644 --- a/examples/native/local_exclamation_topology2.rb +++ b/examples/native/local_exclamation_topology2.rb @@ -1,6 +1,6 @@ java_import 'backtype.storm.testing.TestWordSpout' -require 'lib/red_storm' +require 'red_storm' module RedStorm module Examples @@ -30,8 +30,8 @@ module RedStorm def start(base_class_path, env) builder = TopologyBuilder.new - builder.setSpout('TestWordSpout', TestWordSpout.new, 10) - builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 3).shuffleGrouping('TestWordSpout') + builder.setSpout('TestWordSpout', TestWordSpout.new, 2) + builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('TestWordSpout') builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('ExclamationBolt21') conf = Backtype::Config.new diff --git a/examples/native/local_redis_word_count_topology.rb b/examples/native/local_redis_word_count_topology.rb index ce93b85..7fdd4f6 100644 --- a/examples/native/local_redis_word_count_topology.rb +++ b/examples/native/local_redis_word_count_topology.rb @@ -1,8 +1,7 @@ -require 'bundler/setup' +require 'red_storm' +require 'examples/native/word_count_bolt' require 'redis' require 'thread' -require 'lib/red_storm' -require 'examples/native/word_count_bolt' module RedStorm module Examples diff --git a/examples/native/local_word_count_topology.rb b/examples/native/local_word_count_topology.rb index 46c698f..903de40 100644 --- a/examples/native/local_word_count_topology.rb +++ b/examples/native/local_word_count_topology.rb @@ -1,4 +1,4 @@ -require 'lib/red_storm' +require 'red_storm' require 'examples/native/random_sentence_spout' require 'examples/native/split_sentence_bolt' require 'examples/native/word_count_bolt' @@ -10,9 +10,9 @@ module Examples def start(base_class_path, env) builder = TopologyBuilder.new - builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5) - builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 8).shuffleGrouping('RandomSentenceSpout') - builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) + builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1) + builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout') + builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) conf = Backtype::Config.new conf.setDebug(true) diff --git a/examples/simple/exclamation_topology.rb b/examples/simple/exclamation_topology.rb index 4706a38..6365717 100644 --- a/examples/simple/exclamation_topology.rb +++ b/examples/simple/exclamation_topology.rb @@ -1,36 +1,32 @@ java_import 'backtype.storm.testing.TestWordSpout' +require 'red_storm' require 'examples/simple/exclamation_bolt' # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt module RedStorm module Examples - class ExclamationTopology < RedStorm::SimpleTopology - spout TestWordSpout, :parallelism => 5 do + class ExclamationTopology < SimpleTopology + spout TestWordSpout, :parallelism => 2 do debug true end bolt ExclamationBolt, :parallelism => 2 do source TestWordSpout, :shuffle - # max_task_parallelism 1 end bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do source ExclamationBolt, :shuffle - # max_task_parallelism 1 debug true end configure do |env| debug false - set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" - case env - when :local - max_task_parallelism 40 - when :cluster - num_workers 20 - max_spout_pending(1000); + max_task_parallelism 4 + if env == :cluster + num_workers 4 + max_spout_pending(1000) end end diff --git a/examples/simple/exclamation_topology2.rb b/examples/simple/exclamation_topology2.rb index 8a18ce5..9d25bb1 100644 --- a/examples/simple/exclamation_topology2.rb +++ b/examples/simple/exclamation_topology2.rb @@ -1,9 +1,10 @@ -java_import 'backtype.storm.testing.TestWordSpout' -require 'red_storm' - # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt # and a locally defined ExclamationBolt +java_import 'backtype.storm.testing.TestWordSpout' + +require 'red_storm' + module RedStorm module Examples class ExclamationBolt < RedStorm::SimpleBolt @@ -12,9 +13,9 @@ module RedStorm end class ExclamationTopology2 < RedStorm::SimpleTopology - spout TestWordSpout, :parallelism => 10 + spout TestWordSpout, :parallelism => 2 - bolt ExclamationBolt, :parallelism => 3 do + bolt ExclamationBolt, :parallelism => 2 do source TestWordSpout, :shuffle end @@ -24,13 +25,10 @@ module RedStorm configure do |env| debug true - set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" - case env - when :local - max_task_parallelism 3 - when :cluster - num_workers 20 - max_spout_pending(1000); + max_task_parallelism 4 + if env == :cluster + num_workers 4 + max_spout_pending(1000) end end diff --git a/examples/simple/kafka_topology.rb b/examples/simple/kafka_topology.rb index f5e8b05..8995cea 100644 --- a/examples/simple/kafka_topology.rb +++ b/examples/simple/kafka_topology.rb @@ -1,9 +1,10 @@ -require 'red_storm' java_import 'storm.kafka.KafkaConfig' java_import 'storm.kafka.SpoutConfig' java_import 'storm.kafka.StringScheme' java_import 'storm.kafka.KafkaSpout' +require 'red_storm' + # the KafkaTopology obviously requires a Kafka server running, you can ajust the # host and port below. # diff --git a/examples/simple/redis_word_count_topology.rb b/examples/simple/redis_word_count_topology.rb index bea331c..6bedb18 100644 --- a/examples/simple/redis_word_count_topology.rb +++ b/examples/simple/redis_word_count_topology.rb @@ -1,9 +1,7 @@ -require 'rubygems' require 'red_storm' - +require 'examples/simple/word_count_bolt' require 'redis' require 'thread' -require 'examples/simple/word_count_bolt' module RedStorm module Examples @@ -46,14 +44,14 @@ module RedStorm configure do |env| debug true - set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" + # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" case env when :local max_task_parallelism 3 when :cluster max_task_parallelism 5 num_workers 20 - max_spout_pending(1000); + max_spout_pending(1000) end end end diff --git a/examples/simple/ruby_version_topology.rb b/examples/simple/ruby_version_topology.rb index 0cfa787..d8cbfb1 100644 --- a/examples/simple/ruby_version_topology.rb +++ b/examples/simple/ruby_version_topology.rb @@ -6,7 +6,13 @@ module RedStorm module Examples class VersionSpout < RedStorm::SimpleSpout output_fields :dummy - on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")} + on_init do + log.info("***************** RUBY_VERSION=#{RUBY_VERSION}") + log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}") + log.info("***************** VERSION=#{VERSION}") + log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}") + log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}") + end on_send {} end diff --git a/examples/simple/word_count_topology.rb b/examples/simple/word_count_topology.rb index 08e191d..48abf78 100644 --- a/examples/simple/word_count_topology.rb +++ b/examples/simple/word_count_topology.rb @@ -1,3 +1,4 @@ +require 'red_storm' require 'examples/simple/random_sentence_spout' require 'examples/simple/split_sentence_bolt' require 'examples/simple/word_count_bolt' @@ -5,25 +6,22 @@ require 'examples/simple/word_count_bolt' module RedStorm module Examples class WordCountTopology < SimpleTopology - spout RandomSentenceSpout, :parallelism => 5 + spout RandomSentenceSpout, :parallelism => 2 - bolt SplitSentenceBolt, :parallelism => 8 do + bolt SplitSentenceBolt, :parallelism => 2 do source RandomSentenceSpout, :shuffle end - bolt WordCountBolt, :parallelism => 12 do + bolt WordCountBolt, :parallelism => 2 do source SplitSentenceBolt, :fields => ["word"] end configure :word_count do |env| debug true - set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" - case env - when :local - max_task_parallelism 3 - when :cluster - num_workers 20 - max_spout_pending(1000); + max_task_parallelism 4 + if env == :cluster + num_workers 6 + max_spout_pending(1000) end end