jruby-gradle
/
redstorm
mirror of https://github.com/jruby-gradle/redstorm
1
0
Fork 0
Code Issues Releases Wiki Activity

examples cleanup

This commit is contained in:
Colin Surprenant 2013-05-14 16:32:58 -04:00
parent 39cb20289c
commit 78f0e06c9b
12 changed files with 54 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/native/Gemfile
View File

@ -1,2 +0,0 @@
source :rubygems
gem 'redis'

8
examples/native/cluster_word_count_topology.rb
View File

@ -10,13 +10,13 @@ module RedStorm
def start(base_class_path, env) def start(base_class_path, env)
builder = TopologyBuilder.new builder = TopologyBuilder.new
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5) builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 4).shuffleGrouping('RandomSentenceSpout') builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Backtype::Config.new conf = Backtype::Config.new
conf.setDebug(true) conf.setDebug(true)
conf.setNumWorkers(20); conf.setNumWorkers(4);
conf.setMaxSpoutPending(1000); conf.setMaxSpoutPending(1000);
StormSubmitter.submitTopology("word_count", conf, builder.createTopology); StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
end end

8
examples/native/local_exclamation_topology.rb
View File

@ -1,6 +1,6 @@
java_import 'backtype.storm.testing.TestWordSpout' java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm' require 'red_storm'
require 'examples/native/exclamation_bolt' require 'examples/native/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
@ -13,9 +13,9 @@ module RedStorm
def start(base_class_path, env) def start(base_class_path, env)
builder = TopologyBuilder.new builder = TopologyBuilder.new
builder.setSpout('TestWordSpout', TestWordSpout.new, 10) builder.setSpout('TestWordSpout', TestWordSpout.new, 1)
builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('TestWordSpout') builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('ExclamationBolt1') builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('ExclamationBolt1')
conf = Backtype::Config.new conf = Backtype::Config.new
conf.setDebug(true) conf.setDebug(true)

6
examples/native/local_exclamation_topology2.rb
View File

@ -1,6 +1,6 @@
java_import 'backtype.storm.testing.TestWordSpout' java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm' require 'red_storm'
module RedStorm module RedStorm
module Examples module Examples
@ -30,8 +30,8 @@ module RedStorm
def start(base_class_path, env) def start(base_class_path, env)
builder = TopologyBuilder.new builder = TopologyBuilder.new
builder.setSpout('TestWordSpout', TestWordSpout.new, 10) builder.setSpout('TestWordSpout', TestWordSpout.new, 2)
builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 3).shuffleGrouping('TestWordSpout') builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('ExclamationBolt21') builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('ExclamationBolt21')
conf = Backtype::Config.new conf = Backtype::Config.new

5
examples/native/local_redis_word_count_topology.rb
View File

@ -1,8 +1,7 @@
require 'bundler/setup' require 'red_storm'
require 'examples/native/word_count_bolt'
require 'redis' require 'redis'
require 'thread' require 'thread'
require 'lib/red_storm'
require 'examples/native/word_count_bolt'
module RedStorm module RedStorm
module Examples module Examples

8
examples/native/local_word_count_topology.rb
View File

@ -1,4 +1,4 @@
require 'lib/red_storm' require 'red_storm'
require 'examples/native/random_sentence_spout' require 'examples/native/random_sentence_spout'
require 'examples/native/split_sentence_bolt' require 'examples/native/split_sentence_bolt'
require 'examples/native/word_count_bolt' require 'examples/native/word_count_bolt'
@ -10,9 +10,9 @@ module Examples
def start(base_class_path, env) def start(base_class_path, env)
builder = TopologyBuilder.new builder = TopologyBuilder.new
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5) builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 8).shuffleGrouping('RandomSentenceSpout') builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word")) builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Backtype::Config.new conf = Backtype::Config.new
conf.setDebug(true) conf.setDebug(true)

18
examples/simple/exclamation_topology.rb
View File

@ -1,36 +1,32 @@
java_import 'backtype.storm.testing.TestWordSpout' java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
require 'examples/simple/exclamation_bolt' require 'examples/simple/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
module RedStorm module RedStorm
module Examples module Examples
class ExclamationTopology < RedStorm::SimpleTopology class ExclamationTopology < SimpleTopology
spout TestWordSpout, :parallelism => 5 do spout TestWordSpout, :parallelism => 2 do
debug true debug true
end end
bolt ExclamationBolt, :parallelism => 2 do bolt ExclamationBolt, :parallelism => 2 do
source TestWordSpout, :shuffle source TestWordSpout, :shuffle
# max_task_parallelism 1
end end
bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
source ExclamationBolt, :shuffle source ExclamationBolt, :shuffle
# max_task_parallelism 1
debug true debug true
end end
configure do |env| configure do |env|
debug false debug false
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" max_task_parallelism 4
case env if env == :cluster
when :local num_workers 4
max_task_parallelism 40 max_spout_pending(1000)
when :cluster
num_workers 20
max_spout_pending(1000);
end end
end end

22
examples/simple/exclamation_topology2.rb
View File

@ -1,9 +1,10 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
# and a locally defined ExclamationBolt # and a locally defined ExclamationBolt
java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
module RedStorm module RedStorm
module Examples module Examples
class ExclamationBolt < RedStorm::SimpleBolt class ExclamationBolt < RedStorm::SimpleBolt
@ -12,9 +13,9 @@ module RedStorm
end end
class ExclamationTopology2 < RedStorm::SimpleTopology class ExclamationTopology2 < RedStorm::SimpleTopology
spout TestWordSpout, :parallelism => 10 spout TestWordSpout, :parallelism => 2
bolt ExclamationBolt, :parallelism => 3 do bolt ExclamationBolt, :parallelism => 2 do
source TestWordSpout, :shuffle source TestWordSpout, :shuffle
end end
@ -24,13 +25,10 @@ module RedStorm
configure do |env| configure do |env|
debug true debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" max_task_parallelism 4
case env if env == :cluster
when :local num_workers 4
max_task_parallelism 3 max_spout_pending(1000)
when :cluster
num_workers 20
max_spout_pending(1000);
end end
end end

3
examples/simple/kafka_topology.rb
View File

@ -1,9 +1,10 @@
require 'red_storm'
java_import 'storm.kafka.KafkaConfig' java_import 'storm.kafka.KafkaConfig'
java_import 'storm.kafka.SpoutConfig' java_import 'storm.kafka.SpoutConfig'
java_import 'storm.kafka.StringScheme' java_import 'storm.kafka.StringScheme'
java_import 'storm.kafka.KafkaSpout' java_import 'storm.kafka.KafkaSpout'
require 'red_storm'
# the KafkaTopology obviously requires a Kafka server running, you can ajust the # the KafkaTopology obviously requires a Kafka server running, you can ajust the
# host and port below. # host and port below.
# #

8
examples/simple/redis_word_count_topology.rb
View File

@ -1,9 +1,7 @@
require 'rubygems'
require 'red_storm' require 'red_storm'
require 'examples/simple/word_count_bolt'
require 'redis' require 'redis'
require 'thread' require 'thread'
require 'examples/simple/word_count_bolt'
module RedStorm module RedStorm
module Examples module Examples
@ -46,14 +44,14 @@ module RedStorm
configure do |env| configure do |env|
debug true debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
case env case env
when :local when :local
max_task_parallelism 3 max_task_parallelism 3
when :cluster when :cluster
max_task_parallelism 5 max_task_parallelism 5
num_workers 20 num_workers 20
max_spout_pending(1000); max_spout_pending(1000)
end end
end end
end end

8
examples/simple/ruby_version_topology.rb
View File

@ -6,7 +6,13 @@ module RedStorm
module Examples module Examples
class VersionSpout < RedStorm::SimpleSpout class VersionSpout < RedStorm::SimpleSpout
output_fields :dummy output_fields :dummy
on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")} on_init do
log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")
log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}")
log.info("***************** VERSION=#{VERSION}")
log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}")
log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}")
end
on_send {} on_send {}
end end

18
examples/simple/word_count_topology.rb
View File

@ -1,3 +1,4 @@
require 'red_storm'
require 'examples/simple/random_sentence_spout' require 'examples/simple/random_sentence_spout'
require 'examples/simple/split_sentence_bolt' require 'examples/simple/split_sentence_bolt'
require 'examples/simple/word_count_bolt' require 'examples/simple/word_count_bolt'
@ -5,25 +6,22 @@ require 'examples/simple/word_count_bolt'
module RedStorm module RedStorm
module Examples module Examples
class WordCountTopology < SimpleTopology class WordCountTopology < SimpleTopology
spout RandomSentenceSpout, :parallelism => 5 spout RandomSentenceSpout, :parallelism => 2
bolt SplitSentenceBolt, :parallelism => 8 do bolt SplitSentenceBolt, :parallelism => 2 do
source RandomSentenceSpout, :shuffle source RandomSentenceSpout, :shuffle
end end
bolt WordCountBolt, :parallelism => 12 do bolt WordCountBolt, :parallelism => 2 do
source SplitSentenceBolt, :fields => ["word"] source SplitSentenceBolt, :fields => ["word"]
end end
configure :word_count do |env| configure :word_count do |env|
debug true debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9" max_task_parallelism 4
case env if env == :cluster
when :local num_workers 6
max_task_parallelism 3 max_spout_pending(1000)
when :cluster
num_workers 20
max_spout_pending(1000);
end end
end end