Spark JMX Metrics

Every 10 seconds, the collectd service gathers the following Spark JMX metrics on each node in the cluster. Descriptions for the Spark metrics are not currently available.

  • mapr.spark.driver_block_manager_disk_space_used_mb
  • mapr.spark.driver_block_manager_memory_max_mem_mbb
  • mapr.spark.driver_block_manager_memory_max_off_heap_mem_mb
  • mapr.spark.driver_block_manager_memory_max_on_heap_mem_mb
  • mapr.spark.driver_block_manager_memory_mem_used_mb
  • mapr.spark.driver_block_manager_memory_off_heap_mem_used_mb
  • mapr.spark.driver_block_manager_memory_on_heap_mem_used_mb
  • mapr.spark.driver_block_manager_memory_remaining_mem_mb
  • mapr.spark.driver_block_manager_memory_remaining_off_heap_mem_mb
  • mapr.spark.driver_block_manager_memory_remaining_on_heap_mem_mb
  • mapr.spark.driver_dag_scheduler_job_active_jobs
  • mapr.spark.driver_dag_scheduler_job_all_jobs
  • mapr.spark.driver_dag_scheduler_stage_failed_stages
  • mapr.spark.driver_dag_scheduler_stage_running_stages
  • mapr.spark.driver_dag_scheduler_stage_waiting_stages
  • mapr.spark.driver_live_listener_bus_queue_app_status_size
  • mapr.spark.driver_live_listener_bus_queue_event_log_size
  • mapr.spark.driver_live_listener_bus_queue_executor_management_size
  • mapr.spark.driver_hive_external_catalog_file_cache_hits
  • mapr.spark.driver_hive_external_catalog_files_discovered
  • mapr.spark.driver_hive_external_catalog_hive_client_calls
  • mapr.spark.driver_hive_external_catalog_parallel_listing_job_count
  • mapr.spark.driver_hive_external_catalog_partitions_fetched
  • mapr.spark.driver_live_listener_bus_num_events_posted
  • mapr.spark.driver_live_listener_bus_queue_app_status_num_dropped_events
  • mapr.spark.driver_live_listener_bus_queue_event_log_num_dropped_events
  • mapr.spark.driver_live_listener_bus_queue_executor_management_num_dropped_events
  • mapr.spark.driver_code_generator_compilation_time_count
  • mapr.spark.driver_code_generator_compilation_time_max
  • mapr.spark.driver_code_generator_compilation_time_mean
  • mapr.spark.driver_code_generator_compilation_time_min
  • mapr.spark.driver_code_generator_compilation_time_p50
  • mapr.spark.driver_code_generator_compilation_time_p75
  • mapr.spark.driver_code_generator_compilation_time_p95
  • mapr.spark.driver_code_generator_compilation_time_p98
  • mapr.spark.driver_code_generator_compilation_time_p99
  • mapr.spark.driver_code_generator_compilation_time_p999
  • mapr.spark.driver_code_generator_compilation_time_stddev
  • mapr.spark.driver_code_generator_generated_class_size_count
  • mapr.spark.driver_code_generator_generated_class_size_max
  • mapr.spark.driver_code_generator_generated_class_size_mean
  • mapr.spark.driver_code_generator_generated_class_size_min
  • mapr.spark.driver_code_generator_generated_class_size_p50
  • mapr.spark.driver_code_generator_generated_class_size_p75
  • mapr.spark.driver_code_generator_generated_class_size_p95
  • mapr.spark.driver_code_generator_generated_class_size_p98
  • mapr.spark.driver_code_generator_generated_class_size_p99
  • mapr.spark.driver_code_generator_generated_class_size_p999
  • mapr.spark.driver_code_generator_generated_class_size_stddev
  • mapr.spark.driver_code_generator_generated_method_size_count
  • mapr.spark.driver_code_generator_generated_method_size_max
  • mapr.spark.driver_code_generator_generated_method_size_mean
  • mapr.spark.driver_code_generator_generated_method_size_min
  • mapr.spark.driver_code_generator_generated_method_size_p50
  • mapr.spark.driver_code_generator_generated_method_size_p75
  • mapr.spark.driver_code_generator_generated_method_size_p95
  • mapr.spark.driver_code_generator_generated_method_size_p98
  • mapr.spark.driver_code_generator_generated_method_size_p99
  • mapr.spark.driver_code_generator_generated_method_size_p999
  • mapr.spark.driver_code_generator_generated_method_size_stddev
  • mapr.spark.driver_code_generator_source_code_size_count
  • mapr.spark.driver_code_generator_source_code_size_max
  • mapr.spark.driver_code_generator_source_code_size_mean
  • mapr.spark.driver_code_generator_source_code_size_min
  • mapr.spark.driver_code_generator_source_code_size_p50
  • mapr.spark.driver_code_generator_source_code_size_p75
  • mapr.spark.driver_code_generator_source_code_size_p95
  • mapr.spark.driver_code_generator_source_code_size_p98
  • mapr.spark.driver_code_generator_source_code_size_p99
  • mapr.spark.driver_code_generator_source_code_size_p999
  • mapr.spark.driver_code_generator_source_code_size_stddev
  • mapr.spark.driver_dag_scheduler_message_processing_time_count
  • mapr.spark.driver_dag_scheduler_message_processing_time_max
  • mapr.spark.driver_dag_scheduler_message_processing_time_mean
  • mapr.spark.driver_dag_scheduler_message_processing_time_min
  • mapr.spark.driver_dag_scheduler_message_processing_time_p50
  • mapr.spark.driver_dag_scheduler_message_processing_time_p75
  • mapr.spark.driver_dag_scheduler_message_processing_time_p95
  • mapr.spark.driver_dag_scheduler_message_processing_time_p98
  • mapr.spark.driver_dag_scheduler_message_processing_time_p99
  • mapr.spark.driver_dag_scheduler_message_processing_time_p999
  • mapr.spark.driver_dag_scheduler_message_processing_time_stddev
  • mapr.spark.driver_dag_scheduler_message_processing_time_m15_rate
  • mapr.spark.driver_dag_scheduler_message_processing_time_m1_rate
  • mapr.spark.driver_dag_scheduler_message_processing_time_m5_rate
  • mapr.spark.driver_dag_scheduler_message_processing_time_mean_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_count
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_max
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_min
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p50
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p75
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p95
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p98
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p99
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p999
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_stddev
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m15_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m1_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m5_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_count
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_max
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_min
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p50
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p75
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p95
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p98
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p99
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p999
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_stddev
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m15_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m1_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m5_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_count
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_max
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_min
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p50
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p75
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p95
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p98
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p99
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p999
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_stddev
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m15_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m1_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m5_rate
  • mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean_rate
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_count
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_max
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_min
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p50
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p75
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p95
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p98
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p99
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p999
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_stddev
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m15_rat
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m1_rate
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m5_rate
  • mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean_rate
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_count
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_max
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_min
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p50
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p75
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p95
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p98
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p99
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p999
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_stddev
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m15_rate
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m1_rate
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m5_rate
  • mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean_rate
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_count
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_max
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_min
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p50
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p75
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p95
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p98
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p99
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p990
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_stddev
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m15_rate
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m1_rate
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m5_rate
  • mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean_rate
  • mapr.spark.driver_jvm_ps_mark_sweep_count
  • mapr.spark.driver_jvm_ps_mark_sweep_time
  • mapr.spark.driver_jvm_ps_scavenge_count
  • mapr.spark.driver_jvm_ps_scavenge_time
  • mapr.spark.driver_jvm_direct_capacity
  • mapr.spark.driver_jvm_direct_count
  • mapr.spark.driver_jvm_direct_used
  • mapr.spark.driver_jvm_direct_heap_committed
  • mapr.spark.driver_jvm_direct_heap_init
  • mapr.spark.driver_jvm_direct_heap_max
  • mapr.spark.driver_jvm_direct_heap_usage
  • mapr.spark.driver_jvm_direct_heap_used
  • mapr.spark.driver_jvm_direct_mapped_capacity
  • mapr.spark.driver_jvm_direct_mapped_count
  • mapr.spark.driver_jvm_direct_mapped_used
  • mapr.spark.driver_jvm_direct_non_heap_committed
  • mapr.spark.driver_jvm_direct_non_heap_init
  • mapr.spark.driver_jvm_direct_non_heap_max
  • mapr.spark.driver_jvm_direct_non_heap_usage
  • mapr.spark.driver_jvm_direct_non_heap_used
  • mapr.spark.driver_jvm_pools_code_cache_committed
  • mapr.spark.driver_jvm_pools_code_cache_init
  • mapr.spark.driver_jvm_pools_code_cache_max
  • mapr.spark.driver_jvm_pools_code_cache_usage
  • mapr.spark.driver_jvm_pools_code_cache_used
  • mapr.spark.driver_jvm_pools_compressed_class_space_committed
  • mapr.spark.driver_jvm_pools_compressed_class_space_init
  • mapr.spark.driver_jvm_pools_compressed_class_space_max
  • mapr.spark.driver_jvm_pools_compressed_class_space_usage
  • mapr.spark.driver_jvm_pools_compressed_class_space_used
  • mapr.spark.driver_jvm_pools_metaspace_committed
  • mapr.spark.driver_jvm_pools_metaspace_init
  • mapr.spark.driver_jvm_pools_metaspace_max
  • mapr.spark.driver_jvm_pools_metaspace_usage
  • mapr.spark.driver_jvm_pools_metaspace_used
  • mapr.spark.driver_jvm_pools_ps_eden_space_committed
  • mapr.spark.driver_jvm_pools_ps_eden_space_init
  • mapr.spark.driver_jvm_pools_ps_eden_space_max
  • mapr.spark.driver_jvm_pools_ps_eden_space_usage
  • mapr.spark.driver_jvm_pools_ps_eden_space_used
  • mapr.spark.driver_jvm_pools_ps_old_gen_committed
  • mapr.spark.driver_jvm_pools_ps_old_gen_init
  • mapr.spark.driver_jvm_pools_ps_old_gen_max
  • mapr.spark.driver_jvm_pools_ps_old_gen_usage
  • mapr.spark.driver_jvm_pools_ps_old_gen_used
  • mapr.spark.driver_jvm_pools_ps_survivor_space_committed
  • mapr.spark.driver_jvm_pools_ps_survivor_space_init
  • mapr.spark.driver_jvm_pools_ps_survivor_space_max
  • mapr.spark.driver_jvm_pools_ps_survivor_space_usage
  • mapr.spark.driver_jvm_pools_ps_survivor_space_used
  • mapr.spark.driver_jvm_total_committed
  • mapr.spark.driver_jvm_total_init
  • mapr.spark.driver_jvm_total_max
  • mapr.spark.driver_jvm_total_used