Skip to content

Programmatically build the storm.yaml file #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eherot wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Programmatically build the storm.yaml file #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
183a0d5
Move attributes for supervisor, zookeeper, and nimbus to their own files
eherot May 22, 2014
cf03338
Build the storm yaml file programmatically rather than hard-coding a …
eherot May 22, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 31 additions & 59 deletions attributes/default.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,73 +12,45 @@
default['storm']['cluster_role'] = ""

# general storm attributes
default['storm']['java_lib_path'] = "/usr/local/lib:/opt/local/lib:/usr/lib"
default['storm']['local_dir'] = "/mnt/storm"
default['storm']['local_mode_zmq'] = "false"
default['storm']['cluster_mode'] = "distributed"

# zookeeper attributes
default['storm']['zookeeper']['port'] = 2181
default['storm']['zookeeper']['root'] = "/storm"
default['storm']['zookeeper']['session_timeout'] = 30000
default['storm']['zookeeper']['retry_times'] = 60
default['storm']['zookeeper']['retry_interval'] = 5000

# supervisor attributes
default['storm']['supervisor']['workers'] = 4
default['storm']['supervisor']['childopts'] = "-Xmx1024m"
default['storm']['supervisor']['worker_start_timeout'] = 120
default['storm']['supervisor']['worker_timeout_secs'] = 30
default['storm']['supervisor']['monitor_frequecy_secs'] = 3
default['storm']['supervisor']['heartbeat_frequency_secs'] = 5
default['storm']['supervisor']['enable'] = true
default['storm']['yaml']['java.library.path'] = '/usr/local/lib:/opt/local/lib:/usr/lib'
default['storm']['yaml']['storm.local.dir'] = '/mnt/storm'
default['storm']['yaml']['storm.cluster.mode'] = 'distributed'

# worker attributes
default['storm']['worker']['childopts'] = "-Xmx1280m -XX:+UseConcMarkSweepGC -Dcom.sun.management.jmxremote"
default['storm']['worker']['heartbeat_frequency_secs'] = 1
default['storm']['task']['heartbeat_frequency_secs'] = 3
default['storm']['task']['refresh_poll_secs'] = 10
default['storm']['zmq']['threads'] = 1
default['storm']['zmq']['longer_millis'] = 5000
default['storm']['yaml']['worker.childopts'] = '-Xmx1280m -XX:+UseConcMarkSweepGC ' \
'-Dcom.sun.management.jmxremote'
default['storm']['yaml']['worker.heartbeat.frequency.secs'] = 1

# nimbus attributes
default['storm']['nimbus']['host'] = ""
default['storm']['nimbus']['thrift_port'] = 6627
default['storm']['nimbus']['childopts'] = "-Xmx1024m"
default['storm']['nimbus']['task_timeout_secs'] = 30
default['storm']['nimbus']['supervisor_timeout_secs'] = 60
default['storm']['nimbus']['monitor_freq_secs'] = 10
default['storm']['nimbus']['cleanup_inbox_freq_secs'] = 600
default['storm']['nimbus']['inbox_jar_expiration_secs'] = 3600
default['storm']['nimbus']['task_launch_secs'] = 120
default['storm']['nimbus']['reassign'] = true
default['storm']['nimbus']['file_copy_expiration_secs'] = 600
default['storm']['yaml']['task.heartbeat.frequency.secs'] = 3
default['storm']['yaml']['task.refresh.poll.secs'] = 10
default['storm']['yaml']['zmq.threads'] = 1
default['storm']['yaml']['zmq.longer.millis'] = 5000

# ui attributes
default['storm']['ui']['port'] = 8080
default['storm']['ui']['childopts'] = "-Xmx768m"
default['storm']['yaml']['ui.port'] = 8080
default['storm']['yaml']['ui.childopts'] = '-Xmx768m'

# drpc attributes
default['storm']['drpc']['port'] = 3772
default['storm']['drpc']['invocations_port'] = 3773
default['storm']['drpc']['request_timeout_secs'] = 600
default['storm']['yaml']['drpc.port'] = 3772
default['storm']['yaml']['drpc.invocations.port'] = 3773
default['storm']['yaml']['drpc.request.timeout.secs'] = 600

# transactional attributes
default['storm']['transactional']['zookeeper']['root'] = "/storm-transactional"
default['storm']['transactional']['zookeeper']['port'] = 2181
default['storm']['yaml']['transactional.zookeeper.root'] = '/storm-transactional'
default['storm']['yaml']['transactional.zookeeper.port'] = 2181

# topology attributes
default['storm']['topology']['debug'] = false
default['storm']['topology']['optimize'] = true
default['storm']['topology']['workers'] = 1
default['storm']['topology']['acker_executors'] = 1
default['storm']['topology']['acker_tasks'] = "null"
default['storm']['topology']['tasks'] = "null"
default['storm']['topology']['message_timeout_secs'] = 30
default['storm']['topology']['skip_missing_kryo_registrations'] = false
default['storm']['topology']['max_task_parallelism'] = "null"
default['storm']['topology']['max_spout_pending'] = "null"
default['storm']['topology']['state_synchronization_timeout_secs'] = 60
default['storm']['topology']['stats_sample_rate'] = 0.05
default['storm']['topology']['fall_back_on_java_serialization'] = true
default['storm']['topology']['worker_childopts'] = "null"
default['storm']['yaml']['topology.debug'] = false
default['storm']['yaml']['topology.optimize'] = true
default['storm']['yaml']['topology.workers'] = 1
default['storm']['yaml']['topology.acker.executors'] = 1
default['storm']['yaml']['topology.acker.tasks'] = 'null'
default['storm']['yaml']['topology.tasks'] = 'null'
default['storm']['yaml']['topology.message.timeout.secs'] = 30
default['storm']['yaml']['topology.skip.missing.kryo.registrations'] = false
default['storm']['yaml']['topology.max.task.parallelism'] = 'null'
default['storm']['yaml']['topology.max.spout.pending'] = 'null'
default['storm']['yaml']['topology.state.synchronization.timeout.secs'] = 60
default['storm']['yaml']['topology.stats.sample.rate'] = 0.05
default['storm']['yaml']['topology.fall.back.on.java.serialization'] = true
default['storm']['yaml']['topology.worker.childopts'] = 'null'
10 changes: 10 additions & 0 deletions attributes/nimbus.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
default['storm']['yaml']['nimbus.thrift.port'] = 6627
default['storm']['yaml']['nimbus.childopts'] = '-Xmx1024m'
default['storm']['yaml']['nimbus.task.timeout.secs'] = 30
default['storm']['yaml']['nimbus.supervisor.timeout.secs'] = 60
default['storm']['yaml']['nimbus.monitor.freq.secs'] = 10
default['storm']['yaml']['nimbus.cleanup.inbox.freq.secs'] = 600
default['storm']['yaml']['nimbus.inbox.jar.expiration.secs'] = 3600
default['storm']['yaml']['nimbus.task.launch.secs'] = 120
default['storm']['yaml']['nimbus.reassign'] = true
default['storm']['yaml']['nimbus.file.copy.expiration.secs'] = 600
8 changes: 8 additions & 0 deletions attributes/supervisor.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
default['storm']['supervisor']['workers'] = 4

default['storm']['yaml']['supervisor.childopts'] = '-Xmx1024m'
default['storm']['yaml']['supervisor.worker.start.timeout.secs'] = 120
default['storm']['yaml']['supervisor.worker.timeout.secs'] = 30
default['storm']['yaml']['supervisor.monitor.frequency.secs'] = 3
default['storm']['yaml']['supervisor.heartbeat.frequency.secs'] = 5
default['storm']['yaml']['supervisor.enable'] = true
5 changes: 5 additions & 0 deletions attributes/zookeeper.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
default['storm']['yaml']['storm.zookeeper.port'] = 2181
default['storm']['yaml']['storm.zookeeper.root'] = '/storm'
default['storm']['yaml']['storm.zookeeper.session.timeout'] = 30_000
default['storm']['yaml']['storm.zookeeper.retry.times'] = 60
default['storm']['yaml']['storm.zookeeper.retry.interval'] = 5000
21 changes: 16 additions & 5 deletions recipes/default.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@

#locate the nimbus for this storm cluster
if node.recipes.include?("storm::nimbus")
nimbus_host = node
node.set['storm']['yaml']['nimbus.host'] = node[:fqdn]
else
nimbus_host = search(:node, "role:storm_nimbus AND role:#{node['storm']['cluster_role']} AND chef_environment:#{node.chef_environment}").first
node.set['storm']['yaml']['nimbus.host'] = search(:node, "role:storm_nimbus AND role:#{node['storm']['cluster_role']} AND chef_environment:#{node.chef_environment}").first
end

# search for zookeeper servers
Expand All @@ -65,7 +65,12 @@
end

# setup directories
%w{conf_dir local_dir log_dir install_dir bin_dir}.each do |name|
%w(
conf_dir
log_dir
install_dir
bin_dir
).each do |name|
directory node['storm'][name] do
owner "storm"
group "storm"
Expand All @@ -74,6 +79,13 @@
end
end

directory node['storm']['yaml']['storm.local.dir'] do
owner 'storm'
group 'storm'
action :create
recursive true
end

# download storm
remote_file "#{Chef::Config[:file_cache_path]}/storm-#{node[:storm][:version]}.tar.gz" do
source "#{node['storm']['download_url']}/storm-#{node['storm']['version']}.tar.gz"
Expand Down Expand Up @@ -102,7 +114,6 @@
source "storm.yaml.erb"
mode 00644
variables(
:nimbus => nimbus_host,
:zookeeper_quorum => zookeeper_quorum
)
end
Expand All @@ -126,4 +137,4 @@
variables({
:log_dir => node['storm']['log_dir']
})
end
end
94 changes: 12 additions & 82 deletions templates/default/storm.yaml.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,101 +1,31 @@
### base
java.library.path: "<%= node[:storm][:java_lib_path] %>"
storm.local.dir: "<%= node[:storm][:local_dir] %>"
storm.local.mode.zmq: <%= node[:storm][:local_mode_zmq] %>
storm.cluster.mode: "<%= node[:storm][:cluster_mode] %>"



### zookeeper.*
storm.zookeeper.servers:
<% @zookeeper_quorum.each do |host| -%>
- "<%= host %>"
<% end -%>
storm.zookeeper.port: <%= node[:storm][:zookeeper][:port] %>
storm.zookeeper.root: "<%= node[:storm][:zookeeper][:root] %>"
storm.zookeeper.session.timeout: <%= node[:storm][:zookeeper][:session_timeout] %>
storm.zookeeper.retry.times: <%= node[:storm][:zookeeper][:retry_times] %>
storm.zookeeper.retry.interval: <%= node[:storm][:zookeeper][:retry_interval] %>



### supervisor.* configs are for node supervisors
supervisor.slots.ports:
<% for i in 0..(node[:storm][:supervisor][:workers]-1) -%>
- <%= 6700+i -%>

<% end -%>
supervisor.childopts: "<%= node[:storm][:supervisor][:childopts] %>"
supervisor.worker.start.timeout.secs: <%= node[:storm][:supervisor][:worker_start_timeout] %>
supervisor.worker.timeout.secs: <%= node[:storm][:supervisor][:worker_timeout_secs] %>
supervisor.monitor.frequency.secs: <%= node[:storm][:supervisor][:monitor_frequecy_secs] %>
supervisor.heartbeat.frequency.secs: <%= node[:storm][:supervisor][:heartbeat_frequency_secs] %>
supervisor.enable: <%= node[:storm][:supervisor][:enable] %>



### worker.* configs are for task workers
worker.childopts: "<%= node[:storm][:worker][:childopts] %>"
worker.heartbeat.frequency.secs: <%= node[:storm][:worker][:heartbeat_frequency_secs] %>
task.heartbeat.frequency.secs: <%= node[:storm][:task][:heartbeat_frequency_secs] %>
task.refresh.poll.secs: <%= node[:storm][:task][:refresh_poll_secs] %>
zmq.threads: <%= node[:storm][:zmq][:threads] %>
zmq.linger.millis: <%= node[:storm][:zmq][:longer_millis] %>



### nimbus.* configs are for the master
nimbus.host: "<%= @nimbus[:fqdn] %>"
nimbus.thrift.port: <%= node[:storm][:nimbus][:thrift_port] %>
nimbus.childopts: "<%= node[:storm][:nimbus][:childopts] %>"
nimbus.task.timeout.secs: <%= node[:storm][:nimbus][:task_timeout_secs] %>
nimbus.supervisor.timeout.secs: <%= node[:storm][:nimbus][:supervisor_timeout_secs] %>
nimbus.monitor.freq.secs: <%= node[:storm][:nimbus][:monitor_freq_secs] %>
nimbus.cleanup.inbox.freq.secs: <%= node[:storm][:nimbus][:cleanup_inbox_freq_secs] %>
nimbus.inbox.jar.expiration.secs: <%= node[:storm][:nimbus][:inbox_jar_expiration_secs] %>
nimbus.task.launch.secs: <%= node[:storm][:nimbus][:task_launch_secs] %>
nimbus.reassign: <%= node[:storm][:nimbus][:reassign] %>
nimbus.file.copy.expiration.secs: <%= node[:storm][:nimbus][:file_copy_expiration_secs] %>



### ui.* configs are for the master
ui.port: <%= node[:storm][:ui][:port] %>
ui.childopts: "<%= node[:storm][:ui][:childopts] %>"



### drpc.* configs
drpc.port: <%= node[:storm][:drpc][:port] %>
drpc.invocations.port: <%= node[:storm][:drpc][:invocations_port] %>
drpc.request.timeout.secs: <%= node[:storm][:drpc][:request_timeout_secs] %>



### transactional.* configs
transactional.zookeeper.servers:
<% @zookeeper_quorum.each do |host| -%>
- "<%= host %>"
<% end -%>
transactional.zookeeper.root: "<%= node[:storm][:transactional][:zookeeper][:root] %>"
transactional.zookeeper.port: <%= node[:storm][:transactional][:zookeeper][:port] %>



### topology.* configs are for specific executing storms
topology.debug: <%= node[:storm][:topology][:debug] %>
topology.optimize: <%= node[:storm][:topology][:optimize] %>
topology.workers: <%= node[:storm][:topology][:workers] %>
topology.acker.executors: <%= node[:storm][:topology][:acker_executors] %>
topology.acker.tasks: <%= node[:storm][:topology][:acker_tasks] %>
topology.tasks: <%= node[:storm][:topology][:tasks] %>
topology.message.timeout.secs: <%= node[:storm][:topology][:message_timeout_secs] %>
topology.skip.missing.kryo.registrations: <%= node[:storm][:topology][:skip_missing_kryo_registrations] %>
topology.max.task.parallelism: <%= node[:storm][:topology][:max_task_parallelism] %>
topology.max.spout.pending: <%= node[:storm][:topology][:max_spout_pending] %>
topology.state.synchronization.timeout.secs: <%= node[:storm][:topology][:state_synchronization_timeout_secs] %>
topology.stats.sample.rate: <%= node[:storm][:topology][:stats_sample_rate] %>
topology.fall.back.on.java.serialization: <%= node[:storm][:topology][:fall_back_on_java_serialization] %>
topology.worker.childopts: <%= node[:storm][:topology][:worker_childopts] %>


<% node['storm']['yaml'].each do |k, v| -%>
<%= k %>: <%=
case
when v.class == String && v.downcase == 'null'
'null'
when v.class == String
"\"#{v}\""
else
v
end
%>
<% end -%>