reworked env and path settings, debug traces
This commit is contained in:
Colin Surprenant
parent
4a9d543c64
commit
9c0b41af43
|
|
@ -1,45 +1,46 @@
|
|||
# this is the entry point for these two contexts:
|
||||
# - running red_storm.rake
|
||||
# - at remote cluster topology execution. Once topology_launcher.rb has submitted the topology
|
||||
# the spouts and bolts classes will be instanciated and will require red_storm.rb
|
||||
|
||||
# we depends on rubygems begings loaded at this point for setting up gem/bundle environments
|
||||
# explicitely requiring rubygems is required in remote cluster environment
|
||||
require 'rubygems'
|
||||
|
||||
|
||||
module RedStorm
|
||||
LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
|
||||
JAR_CONTEXT = !!(LAUNCH_PATH =~ /\.jar!$/)
|
||||
TOPOLOGY_LAUNCHED = defined?(LAUNCH_PATH)
|
||||
|
||||
if JAR_CONTEXT
|
||||
REDSTORM_HOME = LAUNCH_PATH
|
||||
TARGET_PATH = LAUNCH_PATH
|
||||
BUNDLE_GEMFILE = "#{TARGET_PATH}/bundler/Gemfile"
|
||||
BUNDLE_PATH = "#{TARGET_PATH}/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
|
||||
GEM_PATH = "#{TARGET_PATH}/gems/"
|
||||
unless TOPOLOGY_LAUNCHED
|
||||
LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
|
||||
JAR_CONTEXT = !!(LAUNCH_PATH =~ /\.jar!$/)
|
||||
|
||||
ENV['BUNDLE_GEMFILE'] = RedStorm::BUNDLE_GEMFILE
|
||||
ENV['BUNDLE_PATH'] = RedStorm::BUNDLE_PATH
|
||||
ENV["GEM_PATH"] = RedStorm::GEM_PATH
|
||||
ENV['BUNDLE_DISABLE_SHARED_GEMS'] = "1"
|
||||
else
|
||||
REDSTORM_HOME = File.expand_path(LAUNCH_PATH + '/..')
|
||||
TARGET_PATH = Dir.pwd
|
||||
BUNDLE_GEMFILE = "#{TARGET_PATH}/target/gems/bundler/Gemfile"
|
||||
BUNDLE_PATH = "#{TARGET_PATH}/target/gems/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
|
||||
GEM_PATH = "#{TARGET_PATH}/target/gems/gems"
|
||||
end
|
||||
|
||||
def setup_gems
|
||||
unless JAR_CONTEXT
|
||||
ENV['BUNDLE_GEMFILE'] = RedStorm::BUNDLE_GEMFILE
|
||||
ENV['BUNDLE_PATH'] = RedStorm::BUNDLE_PATH
|
||||
ENV["GEM_PATH"] = RedStorm::GEM_PATH
|
||||
ENV['BUNDLE_DISABLE_SHARED_GEMS'] = "1"
|
||||
if JAR_CONTEXT
|
||||
BASE_PATH = LAUNCH_PATH
|
||||
else
|
||||
BASE_PATH = Dir.pwd
|
||||
end
|
||||
end
|
||||
|
||||
module_function :setup_gems
|
||||
end
|
||||
|
||||
$:.unshift RedStorm::TARGET_PATH
|
||||
puts("**** red_storm ** PRE PWD=#{Dir.pwd}")
|
||||
puts("**** red_storm ** PRE RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("**** red_storm ** PRE RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("**** red_storm ** PRE RedStorm::BASE_PATH=#{RedStorm::BASE_PATH}")
|
||||
|
||||
|
||||
unless RedStorm::JAR_CONTEXT
|
||||
puts("red_storm UNSHIFTING #{RedStorm::BASE_PATH}/lib")
|
||||
$:.unshift "#{RedStorm::BASE_PATH}/lib"
|
||||
end
|
||||
|
||||
unless RedStorm::TOPOLOGY_LAUNCHED
|
||||
require 'red_storm/environment'
|
||||
# setup gems env only in JAR context otherwise it has already been setup
|
||||
# in topology_launcher.rb
|
||||
RedStorm.setup_gems if RedStorm::JAR_CONTEXT
|
||||
end
|
||||
|
||||
require 'red_storm/version'
|
||||
require 'red_storm/configuration'
|
||||
require 'red_storm/application'
|
||||
|
|
@ -47,12 +48,12 @@ require 'red_storm/simple_bolt'
|
|||
require 'red_storm/simple_spout'
|
||||
require 'red_storm/simple_topology'
|
||||
|
||||
puts("************************ PWD=#{Dir.pwd}")
|
||||
puts("************************ RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("************************ RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("************************ RedStorm::REDSTORM_HOME=#{RedStorm::REDSTORM_HOME}")
|
||||
puts("************************ RedStorm::TARGET_PATH=#{RedStorm::TARGET_PATH}")
|
||||
puts("************************ RedStorm::GEM_PATH=#{RedStorm::GEM_PATH}")
|
||||
puts("************************ ENV['BUNDLE_GEMFILE']=#{ENV['BUNDLE_GEMFILE']}")
|
||||
puts("************************ ENV['BUNDLE_PATH']=#{ENV['BUNDLE_PATH']}")
|
||||
puts("************************ ENV['GEM_PATH']=#{ENV['GEM_PATH']}")
|
||||
puts("**** red_storm ** POST PWD=#{Dir.pwd}")
|
||||
puts("**** red_storm ** POST edStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("**** red_storm ** POST RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("**** red_storm ** POST RedStorm::REDSTORM_HOME=#{RedStorm::REDSTORM_HOME}")
|
||||
puts("**** red_storm ** POST RedStorm::TARGET_PATH=#{RedStorm::TARGET_PATH}")
|
||||
puts("**** red_storm ** POST RedStorm::GEM_PATH=#{RedStorm::GEM_PATH}")
|
||||
puts("**** red_storm ** POST ENV['BUNDLE_GEMFILE']=#{ENV['BUNDLE_GEMFILE']}")
|
||||
puts("**** red_storm ** POST ENV['BUNDLE_PATH']=#{ENV['BUNDLE_PATH']}")
|
||||
puts("**** red_storm ** POST ENV['GEM_PATH']=#{ENV['GEM_PATH']}")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
module RedStorm
|
||||
# LAUNCH_PATH, BASE_PATH and JAR_CONTEXT must be set before requiring this module
|
||||
# typically, they must be set in either red_storm.rb and topology_launcher.rd
|
||||
# which are the 2 entry points
|
||||
|
||||
if JAR_CONTEXT
|
||||
REDSTORM_HOME = LAUNCH_PATH
|
||||
TARGET_PATH = LAUNCH_PATH
|
||||
BUNDLE_GEMFILE = "#{TARGET_PATH}/bundler/Gemfile"
|
||||
BUNDLE_PATH = "#{TARGET_PATH}/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
|
||||
GEM_PATH = "#{TARGET_PATH}/gems/"
|
||||
else
|
||||
REDSTORM_HOME = File.expand_path(LAUNCH_PATH + '/..')
|
||||
TARGET_PATH = "#{BASE_PATH}/target"
|
||||
BUNDLE_GEMFILE = "#{TARGET_PATH}/gems/bundler/Gemfile"
|
||||
BUNDLE_PATH = "#{TARGET_PATH}/gems/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
|
||||
GEM_PATH = "#{TARGET_PATH}/gems/gems"
|
||||
end
|
||||
|
||||
def setup_gems
|
||||
ENV['BUNDLE_GEMFILE'] = RedStorm::BUNDLE_GEMFILE
|
||||
ENV['BUNDLE_PATH'] = RedStorm::BUNDLE_PATH
|
||||
ENV["GEM_PATH"] = RedStorm::GEM_PATH
|
||||
ENV['BUNDLE_DISABLE_SHARED_GEMS'] = "1"
|
||||
end
|
||||
|
||||
module_function :setup_gems
|
||||
end
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
require 'java'
|
||||
require 'rubygems'
|
||||
# require 'rubygems'
|
||||
|
||||
begin
|
||||
# will work from gem, since lib dir is in gem require_paths
|
||||
require 'red_storm'
|
||||
rescue LoadError
|
||||
# will work within RedStorm dev project
|
||||
$:.unshift './lib'
|
||||
require 'red_storm'
|
||||
end
|
||||
# begin
|
||||
# # will work from gem, since lib dir is in gem require_paths
|
||||
# require 'red_storm'
|
||||
# rescue LoadError
|
||||
# # will work within RedStorm dev project
|
||||
# $:.unshift './lib'
|
||||
# require 'red_storm'
|
||||
# end
|
||||
|
||||
# see https://github.com/colinsurprenant/redstorm/issues/7
|
||||
module Backtype
|
||||
|
|
@ -27,34 +27,64 @@ java_import 'redstorm.storm.jruby.JRubySpout'
|
|||
|
||||
java_package 'redstorm'
|
||||
|
||||
module RedStorm
|
||||
LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
|
||||
JAR_CONTEXT = !!(LAUNCH_PATH =~ /\.jar!$/)
|
||||
|
||||
if JAR_CONTEXT
|
||||
BASE_PATH = LAUNCH_PATH
|
||||
LIB_PATH = "#{BASE_PATH}/lib"
|
||||
else
|
||||
BASE_PATH = Dir.pwd
|
||||
LIB_PATH = "#{BASE_PATH}/target/lib"
|
||||
end
|
||||
end
|
||||
|
||||
puts("****TOPOLOGY LAUNCHER PRE ** PWD=#{Dir.pwd}")
|
||||
puts("****TOPOLOGY LAUNCHER PRE ** RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("****TOPOLOGY LAUNCHER PRE ** RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER PRE ** RedStorm::BASE_PATH=#{RedStorm::BASE_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER PRE ** RedStorm::LIB_PATH=#{RedStorm::LIB_PATH}")
|
||||
|
||||
# TopologyLauncher is the application entry point when launching a topology. Basically it will
|
||||
# call require on the specified Ruby topology class file path and call its start method
|
||||
class TopologyLauncher
|
||||
|
||||
java_signature 'void main(String[])'
|
||||
def self.main(args)
|
||||
unless args.size > 1
|
||||
# this is the entry point for these two contexts:
|
||||
# - runnig a topology in local mode. the current Ruby env will stay the same at topology execution
|
||||
# - submitting a topology in cluster mode. the current Ruby env will be valid only at topology submission. At topology execution
|
||||
# in the cluster, the new entry point will be the red_storm.rb, topology_launcher will not be called
|
||||
|
||||
unless args.size > 1
|
||||
puts("Usage: redstorm local|cluster topology_class_file_name")
|
||||
exit(1)
|
||||
end
|
||||
|
||||
puts("****TOPOLOGY LAUNCHER ** PWD=#{Dir.pwd}")
|
||||
puts("****TOPOLOGY LAUNCHER ** RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("****TOPOLOGY LAUNCHER ** RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER ** RedStorm::REDSTORM_HOME=#{RedStorm::REDSTORM_HOME}")
|
||||
puts("****TOPOLOGY LAUNCHER ** RedStorm::TARGET_PATH=#{RedStorm::TARGET_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER ** RedStorm::GEM_PATH=#{RedStorm::GEM_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER ** ENV['BUNDLE_GEMFILE']=#{ENV['BUNDLE_GEMFILE']}")
|
||||
puts("****TOPOLOGY LAUNCHER ** ENV['BUNDLE_PATH']=#{ENV['BUNDLE_PATH']}")
|
||||
puts("****TOPOLOGY LAUNCHER ** ENV['GEM_PATH']=#{ENV['GEM_PATH']}")
|
||||
|
||||
RedStorm.setup_gems
|
||||
|
||||
env = args[0].to_sym
|
||||
class_path = args[1]
|
||||
|
||||
$:.unshift "#{RedStorm::BASE_PATH}"
|
||||
$:.unshift "#{RedStorm::LIB_PATH}"
|
||||
|
||||
require "./#{class_path}" # ./ for 1.9 compatibility
|
||||
require 'red_storm/environment'
|
||||
RedStorm.setup_gems
|
||||
|
||||
puts("****TOPOLOGY LAUNCHER POST ** PWD=#{Dir.pwd}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::REDSTORM_HOME=#{RedStorm::REDSTORM_HOME}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::TARGET_PATH=#{RedStorm::TARGET_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::GEM_PATH=#{RedStorm::GEM_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::BUNDLER_PATH=#{RedStorm::BUNDLE_PATH}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** RedStorm::BUNDLE_GEMFILE=#{RedStorm::BUNDLE_GEMFILE}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** ENV['BUNDLE_GEMFILE']=#{ENV['BUNDLE_GEMFILE']}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** ENV['BUNDLE_PATH']=#{ENV['BUNDLE_PATH']}")
|
||||
puts("****TOPOLOGY LAUNCHER POST ** ENV['GEM_PATH']=#{ENV['GEM_PATH']}")
|
||||
|
||||
|
||||
require "#{class_path}"
|
||||
|
||||
topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
|
||||
puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ DEFAULT_GEMFILE = "Gemfile"
|
|||
|
||||
CWD = Dir.pwd
|
||||
TARGET_DIR = "#{CWD}/target"
|
||||
TARGET_LIB_DIR = "#{TARGET_DIR}/lib"
|
||||
TARGET_SRC_DIR = "#{TARGET_DIR}/src"
|
||||
TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"
|
||||
TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency"
|
||||
|
|
@ -23,8 +24,8 @@ TARGET_MARKERS_DIR = "#{TARGET_DIR}/dependency-markers"
|
|||
TARGET_GEMS_DIR = "#{TARGET_DIR}/gems"
|
||||
TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar"
|
||||
|
||||
JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main"
|
||||
JRUBY_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/lib"
|
||||
REDSTORM_JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main"
|
||||
REDSTORM_LIB_DIR = "#{RedStorm::REDSTORM_HOME}/lib"
|
||||
|
||||
SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples"
|
||||
DST_EXAMPLES = "#{CWD}/examples"
|
||||
|
|
@ -74,6 +75,7 @@ task :unpack do
|
|||
end
|
||||
|
||||
task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
|
||||
puts("\n--> Generating JAR file #{TARGET_CLUSTER_JAR}")
|
||||
ant.jar :destfile => TARGET_CLUSTER_JAR do
|
||||
fileset :dir => TARGET_CLASSES_DIR
|
||||
fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
|
||||
|
|
@ -81,7 +83,9 @@ task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
|
|||
# remove bundler config dir to avoid setting BUNDLE_PATH
|
||||
exclude :name => "bundler/.bundle/**"
|
||||
end
|
||||
fileset :dir => JRUBY_SRC_DIR do
|
||||
# red_storm.rb and red_storm/* must be in root of jar so that "require 'red_storm'"
|
||||
# in bolt works in jar context
|
||||
fileset :dir => TARGET_LIB_DIR do
|
||||
exclude :name => "tasks/**"
|
||||
end
|
||||
fileset :dir => CWD do
|
||||
|
|
@ -91,7 +95,7 @@ task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
|
|||
attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
|
||||
end
|
||||
end
|
||||
puts("\nRedStorm generated jar file #{TARGET_CLUSTER_JAR}")
|
||||
puts("\nRedStorm generated JAR file #{TARGET_CLUSTER_JAR}")
|
||||
end
|
||||
|
||||
task :examples do
|
||||
|
|
@ -104,34 +108,37 @@ task :examples do
|
|||
exit(1)
|
||||
end
|
||||
|
||||
puts("copying examples into #{DST_EXAMPLES}")
|
||||
puts("\n--> Installing examples into #{DST_EXAMPLES}")
|
||||
system("mkdir #{DST_EXAMPLES}")
|
||||
system("cp -r #{SRC_EXAMPLES}/* #{DST_EXAMPLES}")
|
||||
puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}")
|
||||
end
|
||||
|
||||
task :deps => :setup do
|
||||
puts("\n--> Installing dependencies")
|
||||
# install maven dependencies in target
|
||||
system("rmvn dependency:copy-dependencies \
|
||||
-f #{RedStorm::REDSTORM_HOME}/pom.xml \
|
||||
-DoutputDirectory=#{TARGET_DEPENDENCY_DIR} \
|
||||
-DmarkersDirectory=#{TARGET_MARKERS_DIR} \
|
||||
-Dstorm-storm.version=#{INSTALL_STORM_VERSION} \
|
||||
-Dorg.jruby-jruby-complete.version=#{INSTALL_JRUBY_VERSION}")
|
||||
# copy RedStorm lib dir in target
|
||||
puts("cp -r #{REDSTORM_LIB_DIR} #{TARGET_DIR}")
|
||||
system("cp -r #{REDSTORM_LIB_DIR} #{TARGET_DIR}")
|
||||
end
|
||||
|
||||
task :build => :setup do
|
||||
# compile the JRuby proxy classes to Java
|
||||
build_jruby("#{JRUBY_SRC_DIR}/red_storm/proxy")
|
||||
build_jruby("#{REDSTORM_LIB_DIR}/red_storm/proxy")
|
||||
|
||||
# compile the generated Java proxy classes
|
||||
build_java_dir("#{TARGET_SRC_DIR}")
|
||||
|
||||
# generate the JRuby topology launcher
|
||||
build_jruby("#{JRUBY_SRC_DIR}/red_storm/topology_launcher.rb")
|
||||
build_jruby("#{REDSTORM_LIB_DIR}/red_storm/topology_launcher.rb")
|
||||
|
||||
# compile the JRuby proxy classes
|
||||
build_java_dir("#{JAVA_SRC_DIR}")
|
||||
build_java_dir("#{REDSTORM_JAVA_SRC_DIR}")
|
||||
|
||||
# compile the JRuby proxy classes
|
||||
build_java_dir("#{TARGET_SRC_DIR}")
|
||||
|
|
@ -143,17 +150,25 @@ task :bundle, [:bundler_options] => :gems do |t, args|
|
|||
# basically copy specified Gemfile to target/gems/bundler and install into this dir
|
||||
gemfile = bundler_options =~ /--gemfile\s+([^\s]+)/ ? $1 : DEFAULT_GEMFILE
|
||||
if bundler_options =~ /--gemfile\s+[^\s]+/
|
||||
bundler_options.gsub!(/--gemfile\s+[^\s]+/, "--gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile")
|
||||
bundler_options.gsub!(/--gemfile\s+[^\s]+/, "--gemfile ./Gemfile ")
|
||||
else
|
||||
bundler_options = bundler_options + " --gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile"
|
||||
bundler_options = bundler_options + " --gemfile ./Gemfile"
|
||||
end
|
||||
bundler_options = bundler_options + " --path ./"
|
||||
|
||||
if File.exist?(gemfile)
|
||||
puts("\n--> Bundling gems in #{TARGET_GEMS_DIR}/bundler using #{gemfile}")
|
||||
system("cp #{gemfile} #{TARGET_GEMS_DIR}/bundler/Gemfile")
|
||||
system("export BUNDLE_GEMFILE=; \
|
||||
export GEM_PATH=#{RedStorm::GEM_PATH}; \
|
||||
jruby #{RedStorm::RUNTIME['RUBY_VERSION']} -S bundle install #{bundler_options} --path #{TARGET_GEMS_DIR}/bundler/")
|
||||
cmd = "(cd #{TARGET_GEMS_DIR}/bundler; " + \
|
||||
"unset BUNDLE_GEMFILE; " + \
|
||||
"unset BUNDLE_PATH; " + \
|
||||
# "unset BUNDLE_FROZEN; " + \
|
||||
# "unset BUNDLE_DISABLE_SHARED_GEMS; " + \
|
||||
"unset RUBYOPT; " + \
|
||||
"export GEM_PATH=#{RedStorm::GEM_PATH}; " + \
|
||||
"export GEM_HOME=#{RedStorm::GEM_PATH}; " + \
|
||||
"jruby #{RedStorm::RUNTIME['RUBY_VERSION']} -S bundle install #{bundler_options})"
|
||||
system(cmd)
|
||||
elsif gemfile != DEFAULT_GEMFILE
|
||||
puts("WARNING: #{gemfile} not found, cannot bundle gems")
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue