diff --git a/README.md b/README.md index da33e5e..1d196c9 100644 --- a/README.md +++ b/README.md @@ -65,23 +65,23 @@ This will install default Java jar dependencies in `target/dependency`, generate Create a subdirectory for your topology code and create your topology class **using this naming convention**: *underscore* topology_class_file_name.rb **MUST** correspond to its *CamelCase* class name. -Here's an example [hello_world_topology.rb](https://github.com/colinsurprenant/redstorm/tree/master/examples/simple/hello_world_topology.rb) +Here's an example [hello_world_topology.rb](https://github.com/colinsurprenant/redstorm/tree/master/examples/dsl/hello_world_topology.rb) ``` ruby require 'red_storm' -class HelloWorldSpout < RedStorm::SimpleSpout +class HelloWorldSpout < RedStorm::DSL::Spout on_init {@words = ["hello", "world"]} on_send {@words.shift unless @words.empty?} end -class HelloWorldBolt < RedStorm::SimpleBolt +class HelloWorldBolt < RedStorm::DSL::Bolt on_receive :emit => false do |tuple| log.info(tuple.getString(0)) end end -class HelloWorldTopology < RedStorm::SimpleTopology +class HelloWorldTopology < RedStorm::DSL::Topology spout HelloWorldSpout do output_fields :word end @@ -226,21 +226,21 @@ Install the [example files](https://github.com/colinsurprenant/redstorm/tree/mas $ redstorm examples ``` -All examples using the [DSL](https://github.com/colinsurprenant/redstorm/wiki/Ruby-DSL-Documentation) are located in `examples/simple`. Examples using the standard Java interface are in `examples/native`. +All examples using the [DSL](https://github.com/colinsurprenant/redstorm/wiki/Ruby-DSL-Documentation) are located in `examples/dsl`. Examples using the standard Java interface are in `examples/native`. ### Local mode #### Example topologies without gems ``` sh -$ redstorm local examples/simple/exclamation_topology.rb -$ redstorm local examples/simple/exclamation_topology2.rb -$ redstorm local examples/simple/word_count_topology.rb +$ redstorm local examples/dsl/exclamation_topology.rb +$ redstorm local examples/dsl/exclamation_topology2.rb +$ redstorm local examples/dsl/word_count_topology.rb ``` #### Example topologies with gems -For `examples/simple/redis_word_count_topology.rb` the `redis` gem is required and you need a [Redis](http://redis.io/) server running on `localhost:6379` +For `examples/dsl/redis_word_count_topology.rb` the `redis` gem is required and you need a [Redis](http://redis.io/) server running on `localhost:6379` 1. create a `Gemfile` @@ -262,7 +262,7 @@ For `examples/simple/redis_word_count_topology.rb` the `redis` gem is required a 3. run the topology in local mode ``` sh - $ redstorm local examples/simple/redis_word_count_topology.rb + $ redstorm local examples/dsl/redis_word_count_topology.rb ``` Using `redis-cli` push words into the `test` list and watch Storm pick them up @@ -283,15 +283,15 @@ All examples using the [DSL](https://github.com/colinsurprenant/redstorm/wiki/Ru 2. submit the cluster topology jar file to the cluster, assuming you have the Storm distribution installed and the Storm `bin/` directory in your path: ``` sh - $ redstorm cluster examples/simple/exclamation_topology.rb - $ redstorm cluster examples/simple/exclamation_topology2.rb - $ redstorm cluster examples/simple/word_count_topology.rb + $ redstorm cluster examples/dsl/exclamation_topology.rb + $ redstorm cluster examples/dsl/exclamation_topology2.rb + $ redstorm cluster examples/dsl/word_count_topology.rb ``` #### Topologies with gems -For `examples/simple/redis_word_count_topology.rb` the `redis` gem is required and you need a [Redis](http://redis.io/) server running on `localhost:6379` +For `examples/dsl/redis_word_count_topology.rb` the `redis` gem is required and you need a [Redis](http://redis.io/) server running on `localhost:6379` 1. create a `Gemfile` @@ -319,7 +319,7 @@ For `examples/simple/redis_word_count_topology.rb` the `redis` gem is required a 4. submit the cluster topology jar file to the cluster, assuming you have the Storm distribution installed and the Storm `bin/` directory in your path: ``` sh - $ redstorm cluster examples/simple/redis_word_count_topology.rb + $ redstorm cluster examples/dsl/redis_word_count_topology.rb ``` Using `redis-cli` push words into the `test` list and watch Storm pick them up diff --git a/examples/dsl/exclamation_bolt.rb b/examples/dsl/exclamation_bolt.rb new file mode 100644 index 0000000..ed9d7ae --- /dev/null +++ b/examples/dsl/exclamation_bolt.rb @@ -0,0 +1,10 @@ +require 'red_storm' + +module RedStorm + module Examples + class ExclamationBolt < DSL::Bolt + output_fields :word + on_receive(:ack => true, :anchor => true) {|tuple| tuple[0] + "!!!"} # tuple[:word] or tuple["word"] are also valid + end + end +end diff --git a/examples/simple/exclamation_topology.rb b/examples/dsl/exclamation_topology.rb similarity index 86% rename from examples/simple/exclamation_topology.rb rename to examples/dsl/exclamation_topology.rb index 6365717..22f637e 100644 --- a/examples/simple/exclamation_topology.rb +++ b/examples/dsl/exclamation_topology.rb @@ -1,21 +1,21 @@ java_import 'backtype.storm.testing.TestWordSpout' require 'red_storm' -require 'examples/simple/exclamation_bolt' +require 'examples/dsl/exclamation_bolt' # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt module RedStorm module Examples - class ExclamationTopology < SimpleTopology + class ExclamationTopology < DSL::Topology spout TestWordSpout, :parallelism => 2 do debug true end - + bolt ExclamationBolt, :parallelism => 2 do source TestWordSpout, :shuffle end - + bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do source ExclamationBolt, :shuffle debug true @@ -32,7 +32,7 @@ module RedStorm on_submit do |env| if env == :local - sleep(5) + sleep(10) cluster.shutdown end end diff --git a/examples/simple/exclamation_topology2.rb b/examples/dsl/exclamation_topology2.rb similarity index 86% rename from examples/simple/exclamation_topology2.rb rename to examples/dsl/exclamation_topology2.rb index 9d25bb1..4f23197 100644 --- a/examples/simple/exclamation_topology2.rb +++ b/examples/dsl/exclamation_topology2.rb @@ -7,18 +7,18 @@ require 'red_storm' module RedStorm module Examples - class ExclamationBolt < RedStorm::SimpleBolt + class ExclamationBolt < DSL::Bolt output_fields :word - on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"} + on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple[0]}!"} # tuple[:word] or tuple["word"] are also valid end - class ExclamationTopology2 < RedStorm::SimpleTopology + class ExclamationTopology2 < DSL::Topology spout TestWordSpout, :parallelism => 2 - + bolt ExclamationBolt, :parallelism => 2 do source TestWordSpout, :shuffle end - + bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do source ExclamationBolt, :shuffle end diff --git a/examples/simple/hello_world_topology.rb b/examples/dsl/hello_world_topology.rb similarity index 57% rename from examples/simple/hello_world_topology.rb rename to examples/dsl/hello_world_topology.rb index 63a1b2d..9c7ce97 100644 --- a/examples/simple/hello_world_topology.rb +++ b/examples/dsl/hello_world_topology.rb @@ -1,17 +1,17 @@ require 'red_storm' -class HelloWorldSpout < RedStorm::SimpleSpout +class HelloWorldSpout < RedStorm::DSL::Spout on_init {@words = ["hello", "world"]} on_send {@words.shift unless @words.empty?} end -class HelloWorldBolt < RedStorm::SimpleBolt +class HelloWorldBolt < RedStorm::DSL::Bolt on_receive :emit => false do |tuple| - log.info(tuple.getString(0)) + log.info(tuple[0]) # tuple[:word] or tuple["word"] are also valid end end -class HelloWorldTopology < RedStorm::SimpleTopology +class HelloWorldTopology < RedStorm::DSL::Topology spout HelloWorldSpout do output_fields :word end diff --git a/examples/simple/kafka_topology.rb b/examples/dsl/kafka_topology.rb similarity index 91% rename from examples/simple/kafka_topology.rb rename to examples/dsl/kafka_topology.rb index 679f0f4..404c9c8 100644 --- a/examples/simple/kafka_topology.rb +++ b/examples/dsl/kafka_topology.rb @@ -22,7 +22,7 @@ require 'red_storm' # # -class KafkaTopology < RedStorm::SimpleTopology +class KafkaTopology < RedStorm::DSL::Topology spout_config = SpoutConfig.new( KafkaConfig::ZkHosts.new("localhost:2181", "/brokers"), "words", # topic to read from @@ -31,8 +31,8 @@ class KafkaTopology < RedStorm::SimpleTopology ) spout_config.scheme = StringScheme.new - class SplitStringBolt < RedStorm::SimpleBolt - on_receive {|tuple| tuple.getString(0).split.map{|w| [w]}} + class SplitStringBolt < RedStorm::DSL::Bolt + on_receive {|tuple| tuple[0].split.map{|w| [w]}} end spout KafkaSpout, [spout_config] diff --git a/examples/simple/random_sentence_spout.rb b/examples/dsl/random_sentence_spout.rb similarity index 88% rename from examples/simple/random_sentence_spout.rb rename to examples/dsl/random_sentence_spout.rb index 84bff0e..bc37bae 100644 --- a/examples/simple/random_sentence_spout.rb +++ b/examples/dsl/random_sentence_spout.rb @@ -2,7 +2,7 @@ require 'red_storm' module RedStorm module Examples - class RandomSentenceSpout < RedStorm::SimpleSpout + class RandomSentenceSpout < DSL::Spout output_fields :word on_send {@sentences[rand(@sentences.length)]} diff --git a/examples/simple/redis_word_count_topology.rb b/examples/dsl/redis_word_count_topology.rb similarity index 81% rename from examples/simple/redis_word_count_topology.rb rename to examples/dsl/redis_word_count_topology.rb index 6bedb18..9323584 100644 --- a/examples/simple/redis_word_count_topology.rb +++ b/examples/dsl/redis_word_count_topology.rb @@ -1,15 +1,15 @@ require 'red_storm' -require 'examples/simple/word_count_bolt' +require 'examples/dsl/word_count_bolt' require 'redis' require 'thread' module RedStorm module Examples - # RedisWordSpout reads the Redis queue "test" on localhost:6379 + # RedisWordSpout reads the Redis queue "test" on localhost:6379 # and emits each word items pop'ed from the queue. - class RedisWordSpout < RedStorm::SimpleSpout + class RedisWordSpout < DSL::Spout output_fields :word on_send {@q.pop.to_s if @q.size > 0} @@ -18,7 +18,7 @@ module RedStorm @q = Queue.new @redis_reader = detach_redis_reader end - + private def detach_redis_reader @@ -35,16 +35,15 @@ module RedStorm end end - class RedisWordCountTopology < RedStorm::SimpleTopology + class RedisWordCountTopology < DSL::Topology spout RedisWordSpout - + bolt WordCountBolt, :parallelism => 3 do source RedisWordSpout, :fields => ["word"] end configure do |env| debug true - # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" case env when :local max_task_parallelism 3 diff --git a/examples/simple/ruby_version_topology.rb b/examples/dsl/ruby_version_topology.rb similarity index 60% rename from examples/simple/ruby_version_topology.rb rename to examples/dsl/ruby_version_topology.rb index d8cbfb1..a0d8916 100644 --- a/examples/simple/ruby_version_topology.rb +++ b/examples/dsl/ruby_version_topology.rb @@ -4,27 +4,23 @@ require 'red_storm' module RedStorm module Examples - class VersionSpout < RedStorm::SimpleSpout + class VersionSpout < DSL::Spout output_fields :dummy on_init do + log.info("***************** REDSTORM VERSION=#{VERSION}") log.info("***************** RUBY_VERSION=#{RUBY_VERSION}") log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}") - log.info("***************** VERSION=#{VERSION}") log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}") log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}") end on_send {} end - class RubyVersionTopology < RedStorm::SimpleTopology + class RubyVersionTopology < DSL::Topology spout VersionSpout - - configure do |env| - debug true - # force the JRuby version property for this topology. this will only affect remote cluster execution - # for local execution use the --1.8|--1.9 switch when launching - # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" + configure do |env| + debug false end on_submit do |env| diff --git a/examples/simple/split_sentence_bolt.rb b/examples/dsl/split_sentence_bolt.rb similarity index 60% rename from examples/simple/split_sentence_bolt.rb rename to examples/dsl/split_sentence_bolt.rb index 13136fc..0ef330f 100644 --- a/examples/simple/split_sentence_bolt.rb +++ b/examples/dsl/split_sentence_bolt.rb @@ -2,31 +2,31 @@ require 'red_storm' module RedStorm module Examples - class SplitSentenceBolt < RedStorm::SimpleBolt + class SplitSentenceBolt < DSL::Bolt output_fields :word # block declaration style using auto-emit (default) # - on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}} + on_receive {|tuple| tuple[0].split(' ').map{|w| [w]}} # block declaration style no auto-emit # - # on_receive :emit => false do |tuple| - # tuple.getString(0).split(' ').each{|w| unanchored_emit(w)} + # on_receive :emit => false do |tuple| + # tuple[0].split(' ').each{|w| unanchored_emit(w)} # end # alternate declaration style using on_receive method # # on_receive :emit => true # def on_receive(tuple) - # tuple.getString(0).split(' ').map{|w| [w]} + # tuple[0].split(' ').map{|w| [w]} # end # alternate declaration style using any specific method # # on_receive :my_method, :emit => true # def my_method(tuple) - # tuple.getString(0).split(' ').map{|w| [w]} + # tuple[0].split(' ').map{|w| [w]} # end end end diff --git a/examples/simple/word_count_bolt.rb b/examples/dsl/word_count_bolt.rb similarity index 79% rename from examples/simple/word_count_bolt.rb rename to examples/dsl/word_count_bolt.rb index 0b5a858..7593549 100644 --- a/examples/simple/word_count_bolt.rb +++ b/examples/dsl/word_count_bolt.rb @@ -2,14 +2,14 @@ require 'red_storm' module RedStorm module Examples - class WordCountBolt < RedStorm::SimpleBolt + class WordCountBolt < DSL::Bolt output_fields :word, :count on_init {@counts = Hash.new{|h, k| h[k] = 0}} # block declaration style using auto-emit (default) # on_receive do |tuple| - word = tuple.getValue(0).to_s + word = tuple[0].to_s @counts[word] += 1 [word, @counts[word]] diff --git a/examples/simple/word_count_topology.rb b/examples/dsl/word_count_topology.rb similarity index 76% rename from examples/simple/word_count_topology.rb rename to examples/dsl/word_count_topology.rb index 48abf78..ff2fef9 100644 --- a/examples/simple/word_count_topology.rb +++ b/examples/dsl/word_count_topology.rb @@ -1,17 +1,17 @@ require 'red_storm' -require 'examples/simple/random_sentence_spout' -require 'examples/simple/split_sentence_bolt' -require 'examples/simple/word_count_bolt' +require 'examples/dsl/random_sentence_spout' +require 'examples/dsl/split_sentence_bolt' +require 'examples/dsl/word_count_bolt' module RedStorm module Examples - class WordCountTopology < SimpleTopology + class WordCountTopology < DSL::Topology spout RandomSentenceSpout, :parallelism => 2 - + bolt SplitSentenceBolt, :parallelism => 2 do source RandomSentenceSpout, :shuffle end - + bolt WordCountBolt, :parallelism => 2 do source SplitSentenceBolt, :fields => ["word"] end diff --git a/examples/shell/shell_topology.rb b/examples/shell/shell_topology.rb index 108b51c..d7ff3f2 100644 --- a/examples/shell/shell_topology.rb +++ b/examples/shell/shell_topology.rb @@ -3,7 +3,7 @@ require 'thread' java_import 'redstorm.storm.jruby.JRubyShellBolt' -class SimpleSpout < RedStorm::SimpleSpout +class SimpleSpout < RedStorm::DSL::Spout on_init do @q = Queue.new @q << "the quick red fox" @@ -16,7 +16,7 @@ class SimpleSpout < RedStorm::SimpleSpout end end -class ShellTopology < RedStorm::SimpleTopology +class ShellTopology < RedStorm::DSL::Topology spout SimpleSpout do output_fields :string end diff --git a/examples/simple/exclamation_bolt.rb b/examples/simple/exclamation_bolt.rb deleted file mode 100644 index f24035f..0000000 --- a/examples/simple/exclamation_bolt.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'red_storm' - -module RedStorm - module Examples - class ExclamationBolt < RedStorm::SimpleBolt - output_fields :word - on_receive(:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"} - end - end -end diff --git a/test/topology/basic_topology.rb b/test/topology/basic_topology.rb index 693aa34..590a071 100644 --- a/test/topology/basic_topology.rb +++ b/test/topology/basic_topology.rb @@ -2,7 +2,7 @@ require 'red_storm' require 'thread' require 'redis' -class SingleTupleSpout < RedStorm::SimpleSpout +class SingleTupleSpout < RedStorm::DSL::Spout output_fields :string on_init do @@ -17,16 +17,16 @@ class SingleTupleSpout < RedStorm::SimpleSpout end end -class RedisPushBolt < RedStorm::SimpleBolt +class RedisPushBolt < RedStorm::DSL::Bolt on_init {@redis = Redis.new(:host => "localhost", :port => 6379)} on_receive :emit => false do |tuple| - data = tuple.getValue(0).to_s + data = tuple[0].to_s @redis.lpush(File.basename(__FILE__), data) end end -class BasicTopology < RedStorm::SimpleTopology +class BasicTopology < RedStorm::DSL::Topology spout SingleTupleSpout, :parallelism => 1 bolt RedisPushBolt, :parallelism => 1 do diff --git a/test/topology/reliable_topology.rb b/test/topology/reliable_topology.rb index ab412b5..4140cdf 100644 --- a/test/topology/reliable_topology.rb +++ b/test/topology/reliable_topology.rb @@ -2,7 +2,7 @@ require 'red_storm' require 'thread' require 'redis' -class ReliableSpout < RedStorm::SimpleSpout +class ReliableSpout < RedStorm::DSL::Spout output_fields :string on_init do @@ -36,30 +36,30 @@ class ReliableSpout < RedStorm::SimpleSpout end end -class AckBolt < RedStorm::SimpleBolt +class AckBolt < RedStorm::DSL::Bolt on_receive :emit => false do |tuple| ack(tuple) end end -class ImplicitPassthruBolt < RedStorm::SimpleBolt +class ImplicitPassthruBolt < RedStorm::DSL::Bolt output_fields :string on_receive :emit => true, :ack => true, :anchor => true do |tuple| - tuple.getString(0) + tuple[0] end end -class ExplicitPassthruBolt < RedStorm::SimpleBolt +class ExplicitPassthruBolt < RedStorm::DSL::Bolt output_fields :string on_receive :emit => false do |tuple| - anchored_emit(tuple, tuple.getString(0)) + anchored_emit(tuple, tuple[0]) ack(tuple) end end -class ReliableTopology < RedStorm::SimpleTopology +class ReliableTopology < RedStorm::DSL::Topology spout ReliableSpout, :parallelism => 1 bolt ImplicitPassthruBolt, :parallelism => 1 do