diff --git a/attributes/default.rb b/attributes/default.rb index 0e7571e..39017ab 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -12,73 +12,45 @@ default['storm']['cluster_role'] = "" # general storm attributes -default['storm']['java_lib_path'] = "/usr/local/lib:/opt/local/lib:/usr/lib" -default['storm']['local_dir'] = "/mnt/storm" -default['storm']['local_mode_zmq'] = "false" -default['storm']['cluster_mode'] = "distributed" - -# zookeeper attributes -default['storm']['zookeeper']['port'] = 2181 -default['storm']['zookeeper']['root'] = "/storm" -default['storm']['zookeeper']['session_timeout'] = 30000 -default['storm']['zookeeper']['retry_times'] = 60 -default['storm']['zookeeper']['retry_interval'] = 5000 - -# supervisor attributes -default['storm']['supervisor']['workers'] = 4 -default['storm']['supervisor']['childopts'] = "-Xmx1024m" -default['storm']['supervisor']['worker_start_timeout'] = 120 -default['storm']['supervisor']['worker_timeout_secs'] = 30 -default['storm']['supervisor']['monitor_frequecy_secs'] = 3 -default['storm']['supervisor']['heartbeat_frequency_secs'] = 5 -default['storm']['supervisor']['enable'] = true +default['storm']['yaml']['java.library.path'] = '/usr/local/lib:/opt/local/lib:/usr/lib' +default['storm']['yaml']['storm.local.dir'] = '/mnt/storm' +default['storm']['yaml']['storm.cluster.mode'] = 'distributed' # worker attributes -default['storm']['worker']['childopts'] = "-Xmx1280m -XX:+UseConcMarkSweepGC -Dcom.sun.management.jmxremote" -default['storm']['worker']['heartbeat_frequency_secs'] = 1 -default['storm']['task']['heartbeat_frequency_secs'] = 3 -default['storm']['task']['refresh_poll_secs'] = 10 -default['storm']['zmq']['threads'] = 1 -default['storm']['zmq']['longer_millis'] = 5000 +default['storm']['yaml']['worker.childopts'] = '-Xmx1280m -XX:+UseConcMarkSweepGC ' \ + '-Dcom.sun.management.jmxremote' +default['storm']['yaml']['worker.heartbeat.frequency.secs'] = 1 -# nimbus attributes -default['storm']['nimbus']['host'] = "" -default['storm']['nimbus']['thrift_port'] = 6627 -default['storm']['nimbus']['childopts'] = "-Xmx1024m" -default['storm']['nimbus']['task_timeout_secs'] = 30 -default['storm']['nimbus']['supervisor_timeout_secs'] = 60 -default['storm']['nimbus']['monitor_freq_secs'] = 10 -default['storm']['nimbus']['cleanup_inbox_freq_secs'] = 600 -default['storm']['nimbus']['inbox_jar_expiration_secs'] = 3600 -default['storm']['nimbus']['task_launch_secs'] = 120 -default['storm']['nimbus']['reassign'] = true -default['storm']['nimbus']['file_copy_expiration_secs'] = 600 +default['storm']['yaml']['task.heartbeat.frequency.secs'] = 3 +default['storm']['yaml']['task.refresh.poll.secs'] = 10 +default['storm']['yaml']['zmq.threads'] = 1 +default['storm']['yaml']['zmq.longer.millis'] = 5000 # ui attributes -default['storm']['ui']['port'] = 8080 -default['storm']['ui']['childopts'] = "-Xmx768m" +default['storm']['yaml']['ui.port'] = 8080 +default['storm']['yaml']['ui.childopts'] = '-Xmx768m' # drpc attributes -default['storm']['drpc']['port'] = 3772 -default['storm']['drpc']['invocations_port'] = 3773 -default['storm']['drpc']['request_timeout_secs'] = 600 +default['storm']['yaml']['drpc.port'] = 3772 +default['storm']['yaml']['drpc.invocations.port'] = 3773 +default['storm']['yaml']['drpc.request.timeout.secs'] = 600 # transactional attributes -default['storm']['transactional']['zookeeper']['root'] = "/storm-transactional" -default['storm']['transactional']['zookeeper']['port'] = 2181 +default['storm']['yaml']['transactional.zookeeper.root'] = '/storm-transactional' +default['storm']['yaml']['transactional.zookeeper.port'] = 2181 # topology attributes -default['storm']['topology']['debug'] = false -default['storm']['topology']['optimize'] = true -default['storm']['topology']['workers'] = 1 -default['storm']['topology']['acker_executors'] = 1 -default['storm']['topology']['acker_tasks'] = "null" -default['storm']['topology']['tasks'] = "null" -default['storm']['topology']['message_timeout_secs'] = 30 -default['storm']['topology']['skip_missing_kryo_registrations'] = false -default['storm']['topology']['max_task_parallelism'] = "null" -default['storm']['topology']['max_spout_pending'] = "null" -default['storm']['topology']['state_synchronization_timeout_secs'] = 60 -default['storm']['topology']['stats_sample_rate'] = 0.05 -default['storm']['topology']['fall_back_on_java_serialization'] = true -default['storm']['topology']['worker_childopts'] = "null" +default['storm']['yaml']['topology.debug'] = false +default['storm']['yaml']['topology.optimize'] = true +default['storm']['yaml']['topology.workers'] = 1 +default['storm']['yaml']['topology.acker.executors'] = 1 +default['storm']['yaml']['topology.acker.tasks'] = 'null' +default['storm']['yaml']['topology.tasks'] = 'null' +default['storm']['yaml']['topology.message.timeout.secs'] = 30 +default['storm']['yaml']['topology.skip.missing.kryo.registrations'] = false +default['storm']['yaml']['topology.max.task.parallelism'] = 'null' +default['storm']['yaml']['topology.max.spout.pending'] = 'null' +default['storm']['yaml']['topology.state.synchronization.timeout.secs'] = 60 +default['storm']['yaml']['topology.stats.sample.rate'] = 0.05 +default['storm']['yaml']['topology.fall.back.on.java.serialization'] = true +default['storm']['yaml']['topology.worker.childopts'] = 'null' diff --git a/attributes/nimbus.rb b/attributes/nimbus.rb new file mode 100644 index 0000000..5845434 --- /dev/null +++ b/attributes/nimbus.rb @@ -0,0 +1,10 @@ +default['storm']['yaml']['nimbus.thrift.port'] = 6627 +default['storm']['yaml']['nimbus.childopts'] = '-Xmx1024m' +default['storm']['yaml']['nimbus.task.timeout.secs'] = 30 +default['storm']['yaml']['nimbus.supervisor.timeout.secs'] = 60 +default['storm']['yaml']['nimbus.monitor.freq.secs'] = 10 +default['storm']['yaml']['nimbus.cleanup.inbox.freq.secs'] = 600 +default['storm']['yaml']['nimbus.inbox.jar.expiration.secs'] = 3600 +default['storm']['yaml']['nimbus.task.launch.secs'] = 120 +default['storm']['yaml']['nimbus.reassign'] = true +default['storm']['yaml']['nimbus.file.copy.expiration.secs'] = 600 diff --git a/attributes/supervisor.rb b/attributes/supervisor.rb new file mode 100644 index 0000000..ea4118e --- /dev/null +++ b/attributes/supervisor.rb @@ -0,0 +1,8 @@ +default['storm']['supervisor']['workers'] = 4 + +default['storm']['yaml']['supervisor.childopts'] = '-Xmx1024m' +default['storm']['yaml']['supervisor.worker.start.timeout.secs'] = 120 +default['storm']['yaml']['supervisor.worker.timeout.secs'] = 30 +default['storm']['yaml']['supervisor.monitor.frequency.secs'] = 3 +default['storm']['yaml']['supervisor.heartbeat.frequency.secs'] = 5 +default['storm']['yaml']['supervisor.enable'] = true diff --git a/attributes/zookeeper.rb b/attributes/zookeeper.rb new file mode 100644 index 0000000..61c244f --- /dev/null +++ b/attributes/zookeeper.rb @@ -0,0 +1,5 @@ +default['storm']['yaml']['storm.zookeeper.port'] = 2181 +default['storm']['yaml']['storm.zookeeper.root'] = '/storm' +default['storm']['yaml']['storm.zookeeper.session.timeout'] = 30_000 +default['storm']['yaml']['storm.zookeeper.retry.times'] = 60 +default['storm']['yaml']['storm.zookeeper.retry.interval'] = 5000 diff --git a/recipes/default.rb b/recipes/default.rb index c087aa0..70364bd 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -36,9 +36,9 @@ #locate the nimbus for this storm cluster if node.recipes.include?("storm::nimbus") - nimbus_host = node + node.set['storm']['yaml']['nimbus.host'] = node[:fqdn] else - nimbus_host = search(:node, "role:storm_nimbus AND role:#{node['storm']['cluster_role']} AND chef_environment:#{node.chef_environment}").first + node.set['storm']['yaml']['nimbus.host'] = search(:node, "role:storm_nimbus AND role:#{node['storm']['cluster_role']} AND chef_environment:#{node.chef_environment}").first end # search for zookeeper servers @@ -65,7 +65,12 @@ end # setup directories -%w{conf_dir local_dir log_dir install_dir bin_dir}.each do |name| +%w( + conf_dir + log_dir + install_dir + bin_dir +).each do |name| directory node['storm'][name] do owner "storm" group "storm" @@ -74,6 +79,13 @@ end end +directory node['storm']['yaml']['storm.local.dir'] do + owner 'storm' + group 'storm' + action :create + recursive true +end + # download storm remote_file "#{Chef::Config[:file_cache_path]}/storm-#{node[:storm][:version]}.tar.gz" do source "#{node['storm']['download_url']}/storm-#{node['storm']['version']}.tar.gz" @@ -102,7 +114,6 @@ source "storm.yaml.erb" mode 00644 variables( - :nimbus => nimbus_host, :zookeeper_quorum => zookeeper_quorum ) end @@ -126,4 +137,4 @@ variables({ :log_dir => node['storm']['log_dir'] }) -end \ No newline at end of file +end diff --git a/templates/default/storm.yaml.erb b/templates/default/storm.yaml.erb index 0a87ac8..3e296f0 100644 --- a/templates/default/storm.yaml.erb +++ b/templates/default/storm.yaml.erb @@ -1,23 +1,8 @@ -### base -java.library.path: "<%= node[:storm][:java_lib_path] %>" -storm.local.dir: "<%= node[:storm][:local_dir] %>" -storm.local.mode.zmq: <%= node[:storm][:local_mode_zmq] %> -storm.cluster.mode: "<%= node[:storm][:cluster_mode] %>" - - - ### zookeeper.* storm.zookeeper.servers: <% @zookeeper_quorum.each do |host| -%> - "<%= host %>" <% end -%> -storm.zookeeper.port: <%= node[:storm][:zookeeper][:port] %> -storm.zookeeper.root: "<%= node[:storm][:zookeeper][:root] %>" -storm.zookeeper.session.timeout: <%= node[:storm][:zookeeper][:session_timeout] %> -storm.zookeeper.retry.times: <%= node[:storm][:zookeeper][:retry_times] %> -storm.zookeeper.retry.interval: <%= node[:storm][:zookeeper][:retry_interval] %> - - ### supervisor.* configs are for node supervisors supervisor.slots.ports: @@ -25,77 +10,22 @@ supervisor.slots.ports: - <%= 6700+i -%> <% end -%> -supervisor.childopts: "<%= node[:storm][:supervisor][:childopts] %>" -supervisor.worker.start.timeout.secs: <%= node[:storm][:supervisor][:worker_start_timeout] %> -supervisor.worker.timeout.secs: <%= node[:storm][:supervisor][:worker_timeout_secs] %> -supervisor.monitor.frequency.secs: <%= node[:storm][:supervisor][:monitor_frequecy_secs] %> -supervisor.heartbeat.frequency.secs: <%= node[:storm][:supervisor][:heartbeat_frequency_secs] %> -supervisor.enable: <%= node[:storm][:supervisor][:enable] %> - - - -### worker.* configs are for task workers -worker.childopts: "<%= node[:storm][:worker][:childopts] %>" -worker.heartbeat.frequency.secs: <%= node[:storm][:worker][:heartbeat_frequency_secs] %> -task.heartbeat.frequency.secs: <%= node[:storm][:task][:heartbeat_frequency_secs] %> -task.refresh.poll.secs: <%= node[:storm][:task][:refresh_poll_secs] %> -zmq.threads: <%= node[:storm][:zmq][:threads] %> -zmq.linger.millis: <%= node[:storm][:zmq][:longer_millis] %> - - - -### nimbus.* configs are for the master -nimbus.host: "<%= @nimbus[:fqdn] %>" -nimbus.thrift.port: <%= node[:storm][:nimbus][:thrift_port] %> -nimbus.childopts: "<%= node[:storm][:nimbus][:childopts] %>" -nimbus.task.timeout.secs: <%= node[:storm][:nimbus][:task_timeout_secs] %> -nimbus.supervisor.timeout.secs: <%= node[:storm][:nimbus][:supervisor_timeout_secs] %> -nimbus.monitor.freq.secs: <%= node[:storm][:nimbus][:monitor_freq_secs] %> -nimbus.cleanup.inbox.freq.secs: <%= node[:storm][:nimbus][:cleanup_inbox_freq_secs] %> -nimbus.inbox.jar.expiration.secs: <%= node[:storm][:nimbus][:inbox_jar_expiration_secs] %> -nimbus.task.launch.secs: <%= node[:storm][:nimbus][:task_launch_secs] %> -nimbus.reassign: <%= node[:storm][:nimbus][:reassign] %> -nimbus.file.copy.expiration.secs: <%= node[:storm][:nimbus][:file_copy_expiration_secs] %> - - - -### ui.* configs are for the master -ui.port: <%= node[:storm][:ui][:port] %> -ui.childopts: "<%= node[:storm][:ui][:childopts] %>" - - - -### drpc.* configs -drpc.port: <%= node[:storm][:drpc][:port] %> -drpc.invocations.port: <%= node[:storm][:drpc][:invocations_port] %> -drpc.request.timeout.secs: <%= node[:storm][:drpc][:request_timeout_secs] %> - - ### transactional.* configs transactional.zookeeper.servers: <% @zookeeper_quorum.each do |host| -%> - "<%= host %>" <% end -%> -transactional.zookeeper.root: "<%= node[:storm][:transactional][:zookeeper][:root] %>" -transactional.zookeeper.port: <%= node[:storm][:transactional][:zookeeper][:port] %> - - - -### topology.* configs are for specific executing storms -topology.debug: <%= node[:storm][:topology][:debug] %> -topology.optimize: <%= node[:storm][:topology][:optimize] %> -topology.workers: <%= node[:storm][:topology][:workers] %> -topology.acker.executors: <%= node[:storm][:topology][:acker_executors] %> -topology.acker.tasks: <%= node[:storm][:topology][:acker_tasks] %> -topology.tasks: <%= node[:storm][:topology][:tasks] %> -topology.message.timeout.secs: <%= node[:storm][:topology][:message_timeout_secs] %> -topology.skip.missing.kryo.registrations: <%= node[:storm][:topology][:skip_missing_kryo_registrations] %> -topology.max.task.parallelism: <%= node[:storm][:topology][:max_task_parallelism] %> -topology.max.spout.pending: <%= node[:storm][:topology][:max_spout_pending] %> -topology.state.synchronization.timeout.secs: <%= node[:storm][:topology][:state_synchronization_timeout_secs] %> -topology.stats.sample.rate: <%= node[:storm][:topology][:stats_sample_rate] %> -topology.fall.back.on.java.serialization: <%= node[:storm][:topology][:fall_back_on_java_serialization] %> -topology.worker.childopts: <%= node[:storm][:topology][:worker_childopts] %> - +<% node['storm']['yaml'].each do |k, v| -%> +<%= k %>: <%= +case +when v.class == String && v.downcase == 'null' + 'null' +when v.class == String + "\"#{v}\"" +else + v +end +%> +<% end -%>