jruby-gradle
/
redstorm
mirror of https://github.com/jruby-gradle/redstorm
1
0
Fork 0
Code Issues Releases Wiki Activity

all examples in RedStorm::Examples namespace and descriptive ids

This commit is contained in:
Colin Surprenant 2011-12-12 17:16:49 -05:00
parent f482f3124a
commit 0128e29279
17 changed files with 396 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
examples/native/cluster_word_count_topology.rb
View File

@ -1,20 +1,25 @@
require 'red_storm'
require 'examples/native/random_sentence_spout'
require 'examples/native/split_sentence_bolt'
require 'examples/native/word_count_bolt'
module RedStorm
module Examples
class ClusterWordCountTopology
RedStorm::Configuration.topology_class = self
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('1', JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
builder.setBolt('2', JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 4).shuffleGrouping('1')
builder.setBolt('3', JRubyBolt.new(base_class_path, "WordCountBolt"), 4).fieldsGrouping('2', Fields.new("word"))
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 4).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Config.new
conf.setDebug(true)
conf.setNumWorkers(20);
conf.setMaxSpoutPending(1000);
StormSubmitter.submitTopology("word-count", conf, builder.createTopology);
StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
end
end
end
end

4
examples/native/exclamation_bolt.rb
View File

@ -1,3 +1,5 @@
module RedStorm
module Examples
class ExclamationBolt
def prepare(conf, context, collector)
@collector = collector
@ -12,3 +14,5 @@ class ExclamationBolt
declarer.declare(Fields.new("word"))
end
end
end
end

16
examples/native/local_exclamation_topology.rb
View File

@ -1,25 +1,31 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm'
require 'examples/native/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
module RedStorm
module Examples
class LocalExclamationTopology
RedStorm::Configuration.topology_class = self
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('1', TestWordSpout.new, 10)
builder.setBolt('2', JRubyBolt.new(base_class_path, "ExclamationBolt"), 3).shuffleGrouping('1')
builder.setBolt('3', JRubyBolt.new(base_class_path, "ExclamationBolt"), 2).shuffleGrouping('2')
builder.setSpout('TestWordSpout', TestWordSpout.new, 10)
builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('ExclamationBolt1')
conf = Config.new
conf.setDebug(true)
cluster = LocalCluster.new
cluster.submitTopology("test", conf, builder.createTopology)
cluster.submitTopology("exclamation", conf, builder.createTopology)
sleep(5)
cluster.killTopology("test")
cluster.killTopology("exclamation")
cluster.shutdown
end
end
end
end

18
examples/native/local_exclamation_topology2.rb
View File

@ -1,12 +1,16 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm'
module RedStorm
module Examples
class ExclamationBolt2
def prepare(conf, context, collector)
@collector = collector
end
def execute(tuple)
@collector.emit(tuple, Values.new(tuple.getString(0) + "!!!"))
@collector.emit(tuple, Values.new("!#{tuple.getString(0)}!"))
@collector.ack(tuple)
end
@ -23,17 +27,19 @@ class LocalExclamationTopology2
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('1', TestWordSpout.new, 10)
builder.setBolt('2', JRubyBolt.new(base_class_path, "ExclamationBolt2"), 3).shuffleGrouping('1')
builder.setBolt('3', JRubyBolt.new(base_class_path, "ExclamationBolt2"), 2).shuffleGrouping('2')
builder.setSpout('TestWordSpout', TestWordSpout.new, 10)
builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 3).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 2).shuffleGrouping('ExclamationBolt21')
conf = Config.new
conf.setDebug(true)
cluster = LocalCluster.new
cluster.submitTopology("test", conf, builder.createTopology)
cluster.submitTopology("exclamation", conf, builder.createTopology)
sleep(5)
cluster.killTopology("test")
cluster.killTopology("exclamation")
cluster.shutdown
end
end
end
end

11
examples/native/local_redis_word_count_topology.rb
View File

@ -1,7 +1,10 @@
require 'redis'
require 'thread'
require 'lib/red_storm'
require 'examples/native/word_count_bolt'
module RedStorm
module Examples
# RedisWordSpout reads the Redis queue "test" on localhost:6379
# and emits each word items pop'ed from the queue.
class RedisWordSpout
@ -45,16 +48,18 @@ class LocalRedisWordCountTopology
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('1', JRubySpout.new(base_class_path, "RedisWordSpout"), 1)
builder.setBolt('2', JRubyBolt.new(base_class_path, "WordCountBolt"), 3).fieldsGrouping('1', Fields.new("word"))
builder.setSpout('RedisWordSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RedisWordSpout"), 1)
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 3).fieldsGrouping('RedisWordSpout', Fields.new("word"))
conf = Config.new
conf.setDebug(true)
conf.setMaxTaskParallelism(3)
cluster = LocalCluster.new
cluster.submitTopology("redis-word-count", conf, builder.createTopology)
cluster.submitTopology("redis_word_count", conf, builder.createTopology)
sleep(600)
cluster.shutdown
end
end
end
end

9
examples/native/local_word_count_topology.rb
View File

@ -1,3 +1,4 @@
require 'lib/red_storm'
require 'examples/native/random_sentence_spout'
require 'examples/native/split_sentence_bolt'
require 'examples/native/word_count_bolt'
@ -9,16 +10,16 @@ module Examples
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('1', JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
builder.setBolt('2', JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 8).shuffleGrouping('1')
builder.setBolt('3', JRubyBolt.new(base_class_path, "WordCountBolt"), 12).fieldsGrouping('2', Fields.new("word"))
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 8).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Config.new
conf.setDebug(true)
conf.setMaxTaskParallelism(3)
cluster = LocalCluster.new
cluster.submitTopology("word-count", conf, builder.createTopology)
cluster.submitTopology("word_count", conf, builder.createTopology)
sleep(5)
cluster.shutdown
end

4
examples/native/random_sentence_spout.rb
View File

@ -1,3 +1,5 @@
module RedStorm
module Examples
class RandomSentenceSpout
attr_reader :is_distributed
@ -24,3 +26,5 @@ class RandomSentenceSpout
declarer.declare(Fields.new("word"))
end
end
end
end

4
examples/native/split_sentence_bolt.rb
View File

@ -1,3 +1,5 @@
module RedStorm
module Examples
class SplitSentenceBolt
def prepare(conf, context, collector)
@collector = collector
@ -11,3 +13,5 @@ class SplitSentenceBolt
declarer.declare(Fields.new("word"))
end
end
end
end

4
examples/native/word_count_bolt.rb
View File

@ -1,3 +1,5 @@
module RedStorm
module Examples
class WordCountBolt
def initialize
@counts = Hash.new{|h, k| h[k] = 0}
@ -17,3 +19,5 @@ class WordCountBolt
declarer.declare(Fields.new("word", "count"))
end
end
end
end

4
examples/simple/exclamation_bolt.rb
View File

@ -1,6 +1,10 @@
require 'red_storm'
module RedStorm
module Examples
class ExclamationBolt < RedStorm::SimpleBolt
output_fields :word
on_receive(:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
end
end
end

9
examples/simple/exclamation_topology.rb
View File

@ -4,6 +4,8 @@ require 'examples/simple/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
module RedStorm
module Examples
class ExclamationTopology < RedStorm::SimpleTopology
spout TestWordSpout, :parallelism => 10
@ -11,17 +13,16 @@ class ExclamationTopology < RedStorm::SimpleTopology
source TestWordSpout, :shuffle
end
bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
source ExclamationBolt, :shuffle
end
configure do |env|
debug true
case env
when :local
debug true
max_task_parallelism 3
when :cluster
debug true
num_workers 20
max_spout_pending(1000);
end
@ -34,3 +35,5 @@ class ExclamationTopology < RedStorm::SimpleTopology
end
end
end
end
end

9
examples/simple/exclamation_topology2.rb
View File

@ -4,6 +4,8 @@ require 'red_storm'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
# and a locally defined ExclamationBolt
module RedStorm
module Examples
class ExclamationBolt < RedStorm::SimpleBolt
output_fields :word
on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"}
@ -16,17 +18,16 @@ class ExclamationTopology2 < RedStorm::SimpleTopology
source TestWordSpout, :shuffle
end
bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
source ExclamationBolt, :shuffle
end
configure do |env|
debug true
case env
when :local
debug true
max_task_parallelism 3
when :cluster
debug true
num_workers 20
max_spout_pending(1000);
end
@ -39,3 +40,5 @@ class ExclamationTopology2 < RedStorm::SimpleTopology
end
end
end
end
end

4
examples/simple/random_sentence_spout.rb
View File

@ -1,5 +1,7 @@
require 'red_storm'
module RedStorm
module Examples
class RandomSentenceSpout < RedStorm::SimpleSpout
set :is_distributed => true
output_fields :word
@ -16,3 +18,5 @@ class RandomSentenceSpout < RedStorm::SimpleSpout
]
end
end
end
end

8
examples/simple/redis_word_count_topology.rb
View File

@ -5,6 +5,9 @@ require 'red_storm'
require 'examples/simple/word_count_bolt'
module RedStorm
module Examples
# RedisWordSpout reads the Redis queue "test" on localhost:6379
# and emits each word items pop'ed from the queue.
@ -42,14 +45,15 @@ class RedisWordCountTopology < RedStorm::SimpleTopology
end
configure do |env|
debug true
case env
when :local
debug true
max_task_parallelism 3
when :cluster
debug true
num_workers 20
max_spout_pending(1000);
end
end
end
end
end

4
examples/simple/split_sentence_bolt.rb
View File

@ -1,5 +1,7 @@
require 'red_storm'
module RedStorm
module Examples
class SplitSentenceBolt < RedStorm::SimpleBolt
output_fields :word
@ -27,3 +29,5 @@ class SplitSentenceBolt < RedStorm::SimpleBolt
# tuple.getString(0).split(' ').map{|w| [w]}
# end
end
end
end

4
examples/simple/word_count_bolt.rb
View File

@ -1,5 +1,7 @@
require 'red_storm'
module RedStorm
module Examples
class WordCountBolt < RedStorm::SimpleBolt
output_fields :word, :count
on_init {@counts = Hash.new{|h, k| h[k] = 0}}
@ -13,3 +15,5 @@ class WordCountBolt < RedStorm::SimpleBolt
[word, @counts[word]]
end
end
end
end

5
examples/simple/word_count_topology.rb
View File

@ -3,6 +3,7 @@ require 'examples/simple/split_sentence_bolt'
require 'examples/simple/word_count_bolt'
module RedStorm
module Examples
class WordCountTopology < SimpleTopology
spout RandomSentenceSpout, :parallelism => 5
@ -15,12 +16,11 @@ module RedStorm
end
configure :word_count do |env|
debug true
case env
when :local
debug true
max_task_parallelism 3
when :cluster
debug true
num_workers 20
max_spout_pending(1000);
end
@ -34,3 +34,4 @@ module RedStorm
end
end
end
end