# HELP python_gc_objects_collected_total Objects collected during gc # TYPE python_gc_objects_collected_total counter python_gc_objects_collected_total{generation="0"} 3738.0 python_gc_objects_collected_total{generation="1"} 2386.0 python_gc_objects_collected_total{generation="2"} 3346.0 # HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC # TYPE python_gc_objects_uncollectable_total counter python_gc_objects_uncollectable_total{generation="0"} 0.0 python_gc_objects_uncollectable_total{generation="1"} 0.0 python_gc_objects_uncollectable_total{generation="2"} 0.0 # HELP python_gc_collections_total Number of times this generation was collected # TYPE python_gc_collections_total counter python_gc_collections_total{generation="0"} 5662.0 python_gc_collections_total{generation="1"} 514.0 python_gc_collections_total{generation="2"} 41.0 # HELP python_info Python platform information # TYPE python_info gauge python_info{implementation="CPython",major="3",minor="11",patchlevel="17",version="3.11.17"} 1.0 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 1.846910976e+09 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 2.97115648e+08 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1.79266524714e+09 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 15125.439999999999 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds 23.0 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP ai_coach_requests_total Total number of API requests # TYPE ai_coach_requests_total counter # HELP ai_coach_request_duration_seconds API request duration seconds # TYPE ai_coach_request_duration_seconds histogram # HELP ai_workflow_transition_total Workflow state transitions (enqueue, start, success, failure) # TYPE ai_workflow_transition_total counter ai_workflow_transition_total{from_state="queued",reason="ok",result="success",scene="airp_101",to_state="running"} 13.0 ai_workflow_transition_total{from_state="running",reason="fetch_error",result="system_error",scene="airp_101",to_state="failed"} 39.0 # HELP ai_workflow_transition_created Workflow state transitions (enqueue, start, success, failure) # TYPE ai_workflow_transition_created gauge ai_workflow_transition_created{from_state="queued",reason="ok",result="success",scene="airp_101",to_state="running"} 1.7926762924168384e+09 ai_workflow_transition_created{from_state="running",reason="fetch_error",result="system_error",scene="airp_101",to_state="failed"} 1.7926762925517716e+09 # HELP ai_workflow_queue_wait_seconds Seconds from enqueue to worker start # TYPE ai_workflow_queue_wait_seconds histogram ai_workflow_queue_wait_seconds_bucket{le="0.005",queue="ai_coach_tasks",scene="airp_101"} 0.0 ai_workflow_queue_wait_seconds_bucket{le="0.01",queue="ai_coach_tasks",scene="airp_101"} 0.0 ai_workflow_queue_wait_seconds_bucket{le="0.025",queue="ai_coach_tasks",scene="airp_101"} 9.0 ai_workflow_queue_wait_seconds_bucket{le="0.05",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="0.075",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="0.1",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="0.25",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="0.5",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="0.75",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="1.0",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="2.5",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="5.0",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="7.5",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="10.0",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_bucket{le="+Inf",queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_count{queue="ai_coach_tasks",scene="airp_101"} 13.0 ai_workflow_queue_wait_seconds_sum{queue="ai_coach_tasks",scene="airp_101"} 0.2857701778411865 # HELP ai_workflow_queue_wait_seconds_created Seconds from enqueue to worker start # TYPE ai_workflow_queue_wait_seconds_created gauge ai_workflow_queue_wait_seconds_created{queue="ai_coach_tasks",scene="airp_101"} 1.7926762924167502e+09 # HELP ai_workflow_inflight Workflows currently executing a stage # TYPE ai_workflow_inflight gauge ai_workflow_inflight{scene="airp_101",stage="fetch"} 0.0 # HELP ai_workflow_stage_duration_seconds Workflow stage duration in seconds # TYPE ai_workflow_stage_duration_seconds histogram ai_workflow_stage_duration_seconds_bucket{le="0.005",result="success",scene="airp_101",stage="queue_wait"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.01",result="success",scene="airp_101",stage="queue_wait"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.025",result="success",scene="airp_101",stage="queue_wait"} 9.0 ai_workflow_stage_duration_seconds_bucket{le="0.05",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="0.075",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="0.1",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="0.25",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="0.5",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="0.75",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="1.0",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="2.5",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="5.0",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="7.5",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="10.0",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_bucket{le="+Inf",result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_count{result="success",scene="airp_101",stage="queue_wait"} 13.0 ai_workflow_stage_duration_seconds_sum{result="success",scene="airp_101",stage="queue_wait"} 0.2857701778411865 ai_workflow_stage_duration_seconds_bucket{le="0.005",result="system_error",scene="airp_101",stage="fetch"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.01",result="system_error",scene="airp_101",stage="fetch"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.025",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.05",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.075",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.1",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.25",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.5",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.75",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="1.0",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="2.5",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="5.0",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="7.5",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="10.0",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="+Inf",result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_count{result="system_error",scene="airp_101",stage="fetch"} 39.0 ai_workflow_stage_duration_seconds_sum{result="system_error",scene="airp_101",stage="fetch"} 0.5478904210031033 ai_workflow_stage_duration_seconds_bucket{le="0.005",result="system_error",scene="airp_101",stage="e2e"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.01",result="system_error",scene="airp_101",stage="e2e"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.025",result="system_error",scene="airp_101",stage="e2e"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.05",result="system_error",scene="airp_101",stage="e2e"} 0.0 ai_workflow_stage_duration_seconds_bucket{le="0.075",result="system_error",scene="airp_101",stage="e2e"} 30.0 ai_workflow_stage_duration_seconds_bucket{le="0.1",result="system_error",scene="airp_101",stage="e2e"} 38.0 ai_workflow_stage_duration_seconds_bucket{le="0.25",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.5",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="0.75",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="1.0",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="2.5",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="5.0",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="7.5",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="10.0",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_bucket{le="+Inf",result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_count{result="system_error",scene="airp_101",stage="e2e"} 39.0 ai_workflow_stage_duration_seconds_sum{result="system_error",scene="airp_101",stage="e2e"} 2.6869688741862774 # HELP ai_workflow_stage_duration_seconds_created Workflow stage duration in seconds # TYPE ai_workflow_stage_duration_seconds_created gauge ai_workflow_stage_duration_seconds_created{result="success",scene="airp_101",stage="queue_wait"} 1.7926762924167995e+09 ai_workflow_stage_duration_seconds_created{result="system_error",scene="airp_101",stage="fetch"} 1.7926762924995556e+09 ai_workflow_stage_duration_seconds_created{result="system_error",scene="airp_101",stage="e2e"} 1.7926762925518003e+09 # HELP ai_workflow_retry_total Workflow, LLM, or callback retries # TYPE ai_workflow_retry_total counter ai_workflow_retry_total{reason="internal_error",scene="airp_101",stage="e2e"} 26.0 ai_workflow_retry_total{reason="callback_http_error",scene="airp_101",stage="callback"} 26.0 # HELP ai_workflow_retry_created Workflow, LLM, or callback retries # TYPE ai_workflow_retry_created gauge ai_workflow_retry_created{reason="internal_error",scene="airp_101",stage="e2e"} 1.7926762925518816e+09 ai_workflow_retry_created{reason="callback_http_error",scene="airp_101",stage="callback"} 1.7926762956782777e+09 # HELP ai_workflow_retry_scheduled_total Task-level retries parked in Redis until not_before # TYPE ai_workflow_retry_scheduled_total counter # HELP ai_workflow_retry_replayed_total Deferred retries republished onto the main Kafka topic # TYPE ai_workflow_retry_replayed_total counter # HELP ai_workflow_retry_dispatch_failure_total Deferred retry republish or dispatch-ack failures # TYPE ai_workflow_retry_dispatch_failure_total counter # HELP ai_workflow_retry_backlog Deferred retries waiting in Redis for not_before # TYPE ai_workflow_retry_backlog gauge ai_workflow_retry_backlog 0.0 # HELP ai_workflow_retry_backoff_seconds Seconds from scheduling a deferred retry until the next attempt starts # TYPE ai_workflow_retry_backoff_seconds histogram # HELP ai_workflow_dead_letter_total Tasks that exhausted retries and will not run again # TYPE ai_workflow_dead_letter_total counter ai_workflow_dead_letter_total{queue="ai_coach_tasks",reason="fetch_error",scene="airp_101"} 13.0 # HELP ai_workflow_dead_letter_created Tasks that exhausted retries and will not run again # TYPE ai_workflow_dead_letter_created gauge ai_workflow_dead_letter_created{queue="ai_coach_tasks",reason="fetch_error",scene="airp_101"} 1.7926762986979592e+09 # HELP ai_llm_request_total LLM calls by result # TYPE ai_llm_request_total counter # HELP ai_llm_duration_seconds LLM call duration from request to full response in seconds # TYPE ai_llm_duration_seconds histogram # HELP ai_llm_tokens_total LLM token consumption # TYPE ai_llm_tokens_total counter # HELP ai_llm_inflight LLM calls currently waiting for a full response # TYPE ai_llm_inflight gauge # HELP ai_callback_total Algorithm callbacks to Server # TYPE ai_callback_total counter ai_callback_total{reason="callback_http_error",result="system_error",scene="airp_101"} 39.0 # HELP ai_callback_created Algorithm callbacks to Server # TYPE ai_callback_created gauge ai_callback_created{reason="callback_http_error",result="system_error",scene="airp_101"} 1.7926762956782072e+09 # HELP ai_callback_duration_seconds Callback HTTP duration in seconds # TYPE ai_callback_duration_seconds histogram ai_callback_duration_seconds_bucket{le="0.005",result="system_error",scene="airp_101"} 35.0 ai_callback_duration_seconds_bucket{le="0.01",result="system_error",scene="airp_101"} 36.0 ai_callback_duration_seconds_bucket{le="0.025",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.05",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.075",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.1",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.25",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.5",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="0.75",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="1.0",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="2.5",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="5.0",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="7.5",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="10.0",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_bucket{le="+Inf",result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_count{result="system_error",scene="airp_101"} 39.0 ai_callback_duration_seconds_sum{result="system_error",scene="airp_101"} 0.18514013290405273 # HELP ai_callback_duration_seconds_created Callback HTTP duration in seconds # TYPE ai_callback_duration_seconds_created gauge ai_callback_duration_seconds_created{result="system_error",scene="airp_101"} 1.7926762956782289e+09 # HELP ai_worker_pool_slots Shared worker asyncio pool. idle and running are slot counts; waiting is 1 while Kafka fetch is paused because the pool is full, else 0 # TYPE ai_worker_pool_slots gauge ai_worker_pool_slots{state="idle"} 100.0 ai_worker_pool_slots{state="running"} 0.0 ai_worker_pool_slots{state="waiting"} 0.0 # HELP ai_worker_draining 1 while the worker is draining in-flight tasks for shutdown, else 0 # TYPE ai_worker_draining gauge ai_worker_draining 0.0 # HELP ai_worker_assigned_partitions Kafka partitions currently assigned to this worker # TYPE ai_worker_assigned_partitions gauge ai_worker_assigned_partitions 8.0 # HELP ai_tool_request_total Outbound tool calls, currently Java ai-dialog # TYPE ai_tool_request_total counter ai_tool_request_total{result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_request_total{result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 # HELP ai_tool_request_created Outbound tool calls, currently Java ai-dialog # TYPE ai_tool_request_created gauge ai_tool_request_created{result="system_error",scene="unknown",tool="v2/multilingual/query"} 1.79266525596574e+09 ai_tool_request_created{result="success",scene="unknown",tool="v2/multilingual/query"} 1.7937020568917925e+09 # HELP ai_tool_duration_seconds Outbound tool call duration in seconds # TYPE ai_tool_duration_seconds histogram ai_tool_duration_seconds_bucket{le="0.005",result="system_error",scene="unknown",tool="v2/multilingual/query"} 0.0 ai_tool_duration_seconds_bucket{le="0.01",result="system_error",scene="unknown",tool="v2/multilingual/query"} 0.0 ai_tool_duration_seconds_bucket{le="0.025",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.05",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.075",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.1",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.25",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.5",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="0.75",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="1.0",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="2.5",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="5.0",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="7.5",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="10.0",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_bucket{le="+Inf",result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_count{result="system_error",scene="unknown",tool="v2/multilingual/query"} 48.0 ai_tool_duration_seconds_sum{result="system_error",scene="unknown",tool="v2/multilingual/query"} 0.5252813845872879 ai_tool_duration_seconds_bucket{le="0.005",result="success",scene="unknown",tool="v2/multilingual/query"} 0.0 ai_tool_duration_seconds_bucket{le="0.01",result="success",scene="unknown",tool="v2/multilingual/query"} 0.0 ai_tool_duration_seconds_bucket{le="0.025",result="success",scene="unknown",tool="v2/multilingual/query"} 0.0 ai_tool_duration_seconds_bucket{le="0.05",result="success",scene="unknown",tool="v2/multilingual/query"} 3.0 ai_tool_duration_seconds_bucket{le="0.075",result="success",scene="unknown",tool="v2/multilingual/query"} 3.0 ai_tool_duration_seconds_bucket{le="0.1",result="success",scene="unknown",tool="v2/multilingual/query"} 3.0 ai_tool_duration_seconds_bucket{le="0.25",result="success",scene="unknown",tool="v2/multilingual/query"} 3.0 ai_tool_duration_seconds_bucket{le="0.5",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="0.75",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="1.0",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="2.5",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="5.0",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="7.5",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="10.0",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_bucket{le="+Inf",result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_count{result="success",scene="unknown",tool="v2/multilingual/query"} 5.0 ai_tool_duration_seconds_sum{result="success",scene="unknown",tool="v2/multilingual/query"} 0.6959001906216145 # HELP ai_tool_duration_seconds_created Outbound tool call duration in seconds # TYPE ai_tool_duration_seconds_created gauge ai_tool_duration_seconds_created{result="system_error",scene="unknown",tool="v2/multilingual/query"} 1.7926652559657793e+09 ai_tool_duration_seconds_created{result="success",scene="unknown",tool="v2/multilingual/query"} 1.7937020568918202e+09 # HELP ai_worker_task_timeout_total Worker attempt or whole-task hard timeouts # TYPE ai_worker_task_timeout_total counter # HELP ai_worker_active_tasks Worker tasks currently registered # TYPE ai_worker_active_tasks gauge ai_worker_active_tasks 0.0 # HELP ai_worker_oldest_task_age_seconds Age in seconds of the oldest registered worker task # TYPE ai_worker_oldest_task_age_seconds gauge ai_worker_oldest_task_age_seconds 0.0