jruby-gradle
/
redstorm
mirror of https://github.com/jruby-gradle/redstorm
1
0
Fork 0
Code Issues Releases Wiki Activity

examples cleanup

This commit is contained in:
Colin Surprenant 2013-05-14 16:32:58 -04:00
parent 39cb20289c
commit 78f0e06c9b
12 changed files with 54 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/native/Gemfile
View File

@ -1,2 +0,0 @@
source :rubygems
gem 'redis'

8
examples/native/cluster_word_count_topology.rb
View File

@ -10,13 +10,13 @@ module RedStorm
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 4).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Backtype::Config.new
conf.setDebug(true)
conf.setNumWorkers(20);
conf.setNumWorkers(4);
conf.setMaxSpoutPending(1000);
StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
end

8
examples/native/local_exclamation_topology.rb
View File

@ -1,6 +1,6 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm'
require 'red_storm'
require 'examples/native/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
@ -13,9 +13,9 @@ module RedStorm
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('TestWordSpout', TestWordSpout.new, 10)
builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 3).shuffleGrouping('ExclamationBolt1')
builder.setSpout('TestWordSpout', TestWordSpout.new, 1)
builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('ExclamationBolt1')
conf = Backtype::Config.new
conf.setDebug(true)

6
examples/native/local_exclamation_topology2.rb
View File

@ -1,6 +1,6 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'lib/red_storm'
require 'red_storm'
module RedStorm
module Examples
@ -30,8 +30,8 @@ module RedStorm
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('TestWordSpout', TestWordSpout.new, 10)
builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 3).shuffleGrouping('TestWordSpout')
builder.setSpout('TestWordSpout', TestWordSpout.new, 2)
builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('TestWordSpout')
builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('ExclamationBolt21')
conf = Backtype::Config.new

5
examples/native/local_redis_word_count_topology.rb
View File

@ -1,8 +1,7 @@
require 'bundler/setup'
require 'red_storm'
require 'examples/native/word_count_bolt'
require 'redis'
require 'thread'
require 'lib/red_storm'
require 'examples/native/word_count_bolt'
module RedStorm
module Examples

8
examples/native/local_word_count_topology.rb
View File

@ -1,4 +1,4 @@
require 'lib/red_storm'
require 'red_storm'
require 'examples/native/random_sentence_spout'
require 'examples/native/split_sentence_bolt'
require 'examples/native/word_count_bolt'
@ -10,9 +10,9 @@ module Examples
def start(base_class_path, env)
builder = TopologyBuilder.new
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 5)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 8).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
conf = Backtype::Config.new
conf.setDebug(true)

18
examples/simple/exclamation_topology.rb
View File

@ -1,36 +1,32 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
require 'examples/simple/exclamation_bolt'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
module RedStorm
module Examples
class ExclamationTopology < RedStorm::SimpleTopology
spout TestWordSpout, :parallelism => 5 do
class ExclamationTopology < SimpleTopology
spout TestWordSpout, :parallelism => 2 do
debug true
end
bolt ExclamationBolt, :parallelism => 2 do
source TestWordSpout, :shuffle
# max_task_parallelism 1
end
bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
source ExclamationBolt, :shuffle
# max_task_parallelism 1
debug true
end
configure do |env|
debug false
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
case env
when :local
max_task_parallelism 40
when :cluster
num_workers 20
max_spout_pending(1000);
max_task_parallelism 4
if env == :cluster
num_workers 4
max_spout_pending(1000)
end
end

22
examples/simple/exclamation_topology2.rb
View File

@ -1,9 +1,10 @@
java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
# and a locally defined ExclamationBolt
java_import 'backtype.storm.testing.TestWordSpout'
require 'red_storm'
module RedStorm
module Examples
class ExclamationBolt < RedStorm::SimpleBolt
@ -12,9 +13,9 @@ module RedStorm
end
class ExclamationTopology2 < RedStorm::SimpleTopology
spout TestWordSpout, :parallelism => 10
spout TestWordSpout, :parallelism => 2
bolt ExclamationBolt, :parallelism => 3 do
bolt ExclamationBolt, :parallelism => 2 do
source TestWordSpout, :shuffle
end
@ -24,13 +25,10 @@ module RedStorm
configure do |env|
debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
case env
when :local
max_task_parallelism 3
when :cluster
num_workers 20
max_spout_pending(1000);
max_task_parallelism 4
if env == :cluster
num_workers 4
max_spout_pending(1000)
end
end

3
examples/simple/kafka_topology.rb
View File

@ -1,9 +1,10 @@
require 'red_storm'
java_import 'storm.kafka.KafkaConfig'
java_import 'storm.kafka.SpoutConfig'
java_import 'storm.kafka.StringScheme'
java_import 'storm.kafka.KafkaSpout'
require 'red_storm'
# the KafkaTopology obviously requires a Kafka server running, you can ajust the
# host and port below.
#

8
examples/simple/redis_word_count_topology.rb
View File

@ -1,9 +1,7 @@
require 'rubygems'
require 'red_storm'
require 'examples/simple/word_count_bolt'
require 'redis'
require 'thread'
require 'examples/simple/word_count_bolt'
module RedStorm
module Examples
@ -46,14 +44,14 @@ module RedStorm
configure do |env|
debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
# set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
case env
when :local
max_task_parallelism 3
when :cluster
max_task_parallelism 5
num_workers 20
max_spout_pending(1000);
max_spout_pending(1000)
end
end
end

8
examples/simple/ruby_version_topology.rb
View File

@ -6,7 +6,13 @@ module RedStorm
module Examples
class VersionSpout < RedStorm::SimpleSpout
output_fields :dummy
on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")}
on_init do
log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")
log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}")
log.info("***************** VERSION=#{VERSION}")
log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}")
log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}")
end
on_send {}
end

18
examples/simple/word_count_topology.rb
View File

@ -1,3 +1,4 @@
require 'red_storm'
require 'examples/simple/random_sentence_spout'
require 'examples/simple/split_sentence_bolt'
require 'examples/simple/word_count_bolt'
@ -5,25 +6,22 @@ require 'examples/simple/word_count_bolt'
module RedStorm
module Examples
class WordCountTopology < SimpleTopology
spout RandomSentenceSpout, :parallelism => 5
spout RandomSentenceSpout, :parallelism => 2
bolt SplitSentenceBolt, :parallelism => 8 do
bolt SplitSentenceBolt, :parallelism => 2 do
source RandomSentenceSpout, :shuffle
end
bolt WordCountBolt, :parallelism => 12 do
bolt WordCountBolt, :parallelism => 2 do
source SplitSentenceBolt, :fields => ["word"]
end
configure :word_count do |env|
debug true
set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
case env
when :local
max_task_parallelism 3
when :cluster
num_workers 20
max_spout_pending(1000);
max_task_parallelism 4
if env == :cluster
num_workers 6
max_spout_pending(1000)
end
end