arangodb · mchacki · May 6, 2025 · May 6, 2025 · May 6, 2025 · Apr 4, 2025
@@ -0,0 +1,14 @@
+name: arangodb_monitoring_tasks_existing
+introducedIn: "3.12.5"
+help: |
+  Number of currently existing monitoring tasks.
+unit: number
+type: gauge
+category: Statistics
+complexity: advanced
+exposedBy:
+  - dbserver
+  - coordinator
+  - agent
+description: |
+  Number of currently existing monitoring tasks.
@@ -0,0 +1,14 @@
+name: arangodb_monitoring_tasks_existing_thread_registries
+introducedIn: "3.12.5"
+help: |
+  Number of threads that started currently existing monitoring tasks.
+unit: number
+type: gauge
+category: Statistics
+complexity: advanced
+exposedBy:
+  - dbserver
+  - coordinator
+  - agent
+description: |
+  Number of threads that started currently existing monitoring tasks. The thread itself does not need to exist any more. This number also includes still running threads that have started monitoring tasks that do not exist any more.
@@ -0,0 +1,14 @@
+name: arangodb_monitoring_tasks_ready_for_deletion
+introducedIn: "3.12.5"
+help: |
+  Number of currently existing monitoring tasks that wait for their garbage collection.
+unit: number
+type: gauge
+category: Statistics
+complexity: advanced
+exposedBy:
+  - dbserver
+  - coordinator
+  - agent
+description: |
+  Number of currently existing monitoring tasks that wait for their garbage collection.
@@ -0,0 +1,14 @@
+name: arangodb_monitoring_tasks_thread_registries_total
+introducedIn: "3.12.5"
+help: |
+  Total number of threads that started monitoring tasks since database creation.
+unit: number
+type: counter
+category: Statistics
+complexity: advanced
+exposedBy:
+  - dbserver
+  - coordinator
+  - agent
+description: |
+  Total number of threads that started monitoring tasks since database creation.
@@ -0,0 +1,14 @@
+name: arangodb_monitoring_tasks_total
+introducedIn: "3.12.5"
+help: |
+  Total number of created monitoring tasks since database creation.
+unit: number
+type: counter
+category: Statistics
+complexity: advanced
+exposedBy:
+  - dbserver
+  - coordinator
+  - agent
+description: |
+  Total number of created monitoring tasks since database creation.
diff --git a/arangod/AsyncRegistryServer/PrettyPrinter/.gdbinit b/arangod/AsyncRegistryServer/PrettyPrinter/.gdbinit
diff --git a/arangod/AsyncRegistryServer/Stacktrace/CMakeLists.txt b/arangod/AsyncRegistryServer/Stacktrace/CMakeLists.txt
diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt
@@ -103,7 +103,6 @@ get_target_property(IRESEARCH_INCLUDE
 include(arangoserver.cmake)
 add_subdirectory(Agency)
 add_subdirectory(Aql)
-add_subdirectory(AsyncRegistryServer)
 add_subdirectory(Cache)
 add_subdirectory(Cluster)
 add_subdirectory(ClusterEngine)
@@ -118,6 +117,7 @@ add_subdirectory(RestHandler)
 add_subdirectory(RestServer)
 add_subdirectory(RocksDBEngine)
 add_subdirectory(StorageEngine)
+add_subdirectory(SystemMonitor)
 add_subdirectory(Utils)
 if (USE_V8)
   add_subdirectory(V8Server)

diff --git a/arangod/Cluster/Maintenance.cpp b/arangod/Cluster/Maintenance.cpp
@@ -34,7 +34,7 @@
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/FollowerInfo.h"
-#include "Cluster/ResignShardLeadership.h"
+#include "Cluster/MaintenanceActions/ResignShardLeadership.h"
 #include "Indexes/Index.h"
 #include "Inspection/VPack.h"
 #include "IResearch/IResearchCommon.h"

diff --git a/arangod/Cluster/Action.cpp → ...god/Cluster/MaintenanceActions/Action.cpp b/arangod/Cluster/Action.cpp → ...god/Cluster/MaintenanceActions/Action.cpp
@@ -25,18 +25,18 @@
 #include "Action.h"
 
 #include "Basics/Exceptions.h"
-#include "Cluster/CreateCollection.h"
-#include "Cluster/CreateDatabase.h"
-#include "Cluster/DropCollection.h"
-#include "Cluster/DropDatabase.h"
-#include "Cluster/DropIndex.h"
-#include "Cluster/EnsureIndex.h"
+#include "Cluster/MaintenanceActions/CreateCollection.h"
+#include "Cluster/MaintenanceActions/CreateDatabase.h"
+#include "Cluster/MaintenanceActions/DropCollection.h"
+#include "Cluster/MaintenanceActions/DropDatabase.h"
+#include "Cluster/MaintenanceActions/DropIndex.h"
+#include "Cluster/MaintenanceActions/EnsureIndex.h"
 #include "Cluster/MaintenanceStrings.h"
-#include "Cluster/ResignShardLeadership.h"
-#include "Cluster/SynchronizeShard.h"
-#include "Cluster/TakeoverShardLeadership.h"
-#include "Cluster/UpdateCollection.h"
-#include "Cluster/UpdateReplicatedLogAction.h"
+#include "Cluster/MaintenanceActions/ResignShardLeadership.h"
+#include "Cluster/MaintenanceActions/SynchronizeShard.h"
+#include "Cluster/MaintenanceActions/TakeoverShardLeadership.h"
+#include "Cluster/MaintenanceActions/UpdateCollection.h"
+#include "Cluster/MaintenanceActions/UpdateReplicatedLogAction.h"
 
 using namespace arangodb;
 using namespace arangodb::maintenance;

diff --git a/arangod/Cluster/Action.h → arangod/Cluster/MaintenanceActions/Action.h b/arangod/Cluster/Action.h → arangod/Cluster/MaintenanceActions/Action.h
diff --git a/arangod/Cluster/ActionBase.cpp → ...Cluster/MaintenanceActions/ActionBase.cpp b/arangod/Cluster/ActionBase.cpp → ...Cluster/MaintenanceActions/ActionBase.cpp
@@ -22,7 +22,7 @@
 /// @author Matthew Von-Maszewski
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "Cluster/ActionBase.h"
+#include "Cluster/MaintenanceActions/ActionBase.h"
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/TimeString.h"

diff --git a/arangod/Cluster/ActionBase.h → ...d/Cluster/MaintenanceActions/ActionBase.h b/arangod/Cluster/ActionBase.h → ...d/Cluster/MaintenanceActions/ActionBase.h
diff --git a/arangod/Cluster/ActionDescription.cpp → .../MaintenanceActions/ActionDescription.cpp b/arangod/Cluster/ActionDescription.cpp → .../MaintenanceActions/ActionDescription.cpp
diff --git a/arangod/Cluster/ActionDescription.h → ...er/MaintenanceActions/ActionDescription.h b/arangod/Cluster/ActionDescription.h → ...er/MaintenanceActions/ActionDescription.h
diff --git a/arangod/Cluster/CreateCollection.cpp → ...r/MaintenanceActions/CreateCollection.cpp b/arangod/Cluster/CreateCollection.cpp → ...r/MaintenanceActions/CreateCollection.cpp
@@ -44,6 +44,7 @@
 #include "Replication2/ReplicatedState/ReplicatedState.h"
 #include "Replication2/StateMachines/Document/DocumentFollowerState.h"
 #include "Replication2/StateMachines/Document/DocumentLeaderState.h"
+#include "TaskMonitoring/task.h"
 
 #include <velocypack/Compare.h>
 #include <velocypack/Iterator.h>
@@ -112,6 +113,11 @@ bool CreateCollection::first() {
   auto const& leader = _description.get(THE_LEADER);
   auto const& props = properties();
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"CreateCollection for DB: '" + database +
+                                    "', Collection: '" + collection +
+                                    "', Shard: '" + shard + "'"};
+
   std::string from;
   _description.get("from", from);
 

diff --git a/arangod/Cluster/CreateCollection.h → ...ter/MaintenanceActions/CreateCollection.h b/arangod/Cluster/CreateCollection.h → ...ter/MaintenanceActions/CreateCollection.h
diff --git a/arangod/Cluster/CreateDatabase.cpp → ...ter/MaintenanceActions/CreateDatabase.cpp b/arangod/Cluster/CreateDatabase.cpp → ...ter/MaintenanceActions/CreateDatabase.cpp
@@ -37,6 +37,7 @@
 #include "Utils/DatabaseGuard.h"
 #include "Utils/OperationOptions.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;
@@ -69,6 +70,10 @@ bool CreateDatabase::first() {
   VPackSlice users;
   auto database = _description.get(DATABASE);
 
+  // Add task monitoring
+  auto task =
+      task_monitoring::Task{"CreateDatabase for DB: '" + database + "'"};
+
   LOG_TOPIC("953b1", DEBUG, Logger::MAINTENANCE)
       << "CreateDatabase: creating database " << database;
 

diff --git a/arangod/Cluster/CreateDatabase.h → ...uster/MaintenanceActions/CreateDatabase.h b/arangod/Cluster/CreateDatabase.h → ...uster/MaintenanceActions/CreateDatabase.h
diff --git a/arangod/Cluster/DropCollection.cpp → ...ter/MaintenanceActions/DropCollection.cpp b/arangod/Cluster/DropCollection.cpp → ...ter/MaintenanceActions/DropCollection.cpp
@@ -40,6 +40,7 @@
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
 #include "VocBase/vocbase.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;
@@ -69,6 +70,10 @@ bool DropCollection::first() {
   auto const& database = getDatabase();
   auto const& shard = getShard();
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"DropCollection for DB: '" + database +
+                                    "', Shard: '" + shard + "'"};
+
   LOG_TOPIC("a2961", DEBUG, Logger::MAINTENANCE)
       << "DropCollection: dropping local shard '" << database << "/" << shard;
 

diff --git a/arangod/Cluster/DropCollection.h → ...uster/MaintenanceActions/DropCollection.h b/arangod/Cluster/DropCollection.h → ...uster/MaintenanceActions/DropCollection.h
diff --git a/arangod/Cluster/DropDatabase.cpp → ...uster/MaintenanceActions/DropDatabase.cpp b/arangod/Cluster/DropDatabase.cpp → ...uster/MaintenanceActions/DropDatabase.cpp
@@ -36,6 +36,7 @@
 #include "Utils/ExecContext.h"
 #include "Utils/OperationOptions.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb::application_features;
 using namespace arangodb::methods;
@@ -69,6 +70,9 @@ bool DropDatabase::first() {
   LOG_TOPIC("22779", DEBUG, Logger::MAINTENANCE)
       << "DropDatabase: dropping " << database;
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"DropDatabase for DB: '" + database + "'"};
+
   try {
     auto& df = _feature.server().getFeature<DatabaseFeature>();
     DatabaseGuard guard(df, StaticStrings::SystemDatabase);

diff --git a/arangod/Cluster/DropDatabase.h → ...Cluster/MaintenanceActions/DropDatabase.h b/arangod/Cluster/DropDatabase.h → ...Cluster/MaintenanceActions/DropDatabase.h
diff --git a/arangod/Cluster/DropIndex.cpp → .../Cluster/MaintenanceActions/DropIndex.cpp b/arangod/Cluster/DropIndex.cpp → .../Cluster/MaintenanceActions/DropIndex.cpp
@@ -39,6 +39,7 @@
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
 #include "VocBase/Methods/Indexes.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb::application_features;
 using namespace arangodb::maintenance;
@@ -90,6 +91,11 @@ bool DropIndex::first() {
   auto const& shard = _description.get(SHARD);
   auto const& id = _description.get(INDEX);
 
+  // Add task monitoring
+  auto task =
+      task_monitoring::Task{"DropIndex for DB: '" + database + "', Shard: '" +
+                            shard + "', Index: '" + id + "'"};
+
   VPackBuilder index;
   index.add(VPackValue(_description.get(INDEX)));
 

diff --git a/arangod/Cluster/DropIndex.h → ...od/Cluster/MaintenanceActions/DropIndex.h b/arangod/Cluster/DropIndex.h → ...od/Cluster/MaintenanceActions/DropIndex.h
diff --git a/arangod/Cluster/EnsureIndex.cpp → ...luster/MaintenanceActions/EnsureIndex.cpp b/arangod/Cluster/EnsureIndex.cpp → ...luster/MaintenanceActions/EnsureIndex.cpp
@@ -40,6 +40,7 @@
 #include "Utils/DatabaseGuard.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;
@@ -114,6 +115,11 @@ bool EnsureIndex::first() {
   auto const& shard = _description.get(SHARD);
   auto const& id = properties().get(ID).copyString();
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"EnsureIndex for DB: '" + database +
+                                    "', Collection: '" + collection +
+                                    "', Shard: '" + shard + "'"};
+
   try {  // now try to guard the database
     auto& df = _feature.server().getFeature<DatabaseFeature>();
     DatabaseGuard guard(df, database);

diff --git a/arangod/Cluster/EnsureIndex.h → .../Cluster/MaintenanceActions/EnsureIndex.h b/arangod/Cluster/EnsureIndex.h → .../Cluster/MaintenanceActions/EnsureIndex.h
diff --git a/arangod/Cluster/ResignShardLeadership.cpp → ...ntenanceActions/ResignShardLeadership.cpp b/arangod/Cluster/ResignShardLeadership.cpp → ...ntenanceActions/ResignShardLeadership.cpp
@@ -41,6 +41,7 @@
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 #include <velocypack/Compare.h>
 #include <velocypack/Iterator.h>
@@ -80,6 +81,11 @@ bool ResignShardLeadership::first() {
   std::string const& database = getDatabase();
   std::string const& collection = getShard();
 
+  // Add task monitoring
+  auto task =
+      task_monitoring::Task{"ResignShardLeadership for DB: '" + database +
+                            "', Shard: '" + collection + "'"};
+
   LOG_TOPIC("14f43", DEBUG, Logger::MAINTENANCE)
       << "trying to withdraw as leader of shard '" << database << "/"
       << collection;

diff --git a/arangod/Cluster/ResignShardLeadership.h → ...aintenanceActions/ResignShardLeadership.h b/arangod/Cluster/ResignShardLeadership.h → ...aintenanceActions/ResignShardLeadership.h
diff --git a/arangod/Cluster/SynchronizeShard.cpp → ...r/MaintenanceActions/SynchronizeShard.cpp b/arangod/Cluster/SynchronizeShard.cpp → ...r/MaintenanceActions/SynchronizeShard.cpp
@@ -33,15 +33,15 @@
 #include "Basics/TimeString.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/debugging.h"
-#include "Cluster/ActionDescription.h"
+#include "Cluster/MaintenanceActions/ActionDescription.h"
 #include "Cluster/AgencyCache.h"
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/CollectionInfoCurrent.h"
 #include "Cluster/FollowerInfo.h"
 #include "Cluster/Maintenance.h"
 #include "Cluster/MaintenanceFeature.h"
-#include "Cluster/ResignShardLeadership.h"
+#include "Cluster/MaintenanceActions/ResignShardLeadership.h"
 #include "Cluster/ReplicationTimeoutFeature.h"
 #include "Cluster/ServerState.h"
 #include "GeneralServer/AuthenticationFeature.h"
@@ -66,6 +66,7 @@
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 #include <absl/strings/str_cat.h>
 #include <velocypack/Compare.h>

diff --git a/arangod/Cluster/SynchronizeShard.h → ...ter/MaintenanceActions/SynchronizeShard.h b/arangod/Cluster/SynchronizeShard.h → ...ter/MaintenanceActions/SynchronizeShard.h
@@ -25,8 +25,8 @@
 #pragma once
 
 #include "Basics/ResultT.h"
-#include "Cluster/ActionBase.h"
-#include "Cluster/ActionDescription.h"
+#include "Cluster/MaintenanceActions/ActionBase.h"
+#include "Cluster/MaintenanceActions/ActionDescription.h"
 #include "Replication/utilities.h"
 #include "VocBase/voc-types.h"
 

diff --git a/arangod/Cluster/TakeoverShardLeadership.cpp → ...enanceActions/TakeoverShardLeadership.cpp b/arangod/Cluster/TakeoverShardLeadership.cpp → ...enanceActions/TakeoverShardLeadership.cpp
@@ -51,6 +51,7 @@
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 #include <velocypack/Compare.h>
 #include <velocypack/Iterator.h>
@@ -267,6 +268,11 @@ bool TakeoverShardLeadership::first() {
   uint64_t planIndex = basics::StringUtils::uint64(planRaftIndex);
   Result res;
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"TakeoverShardLeadership for DB: '" +
+                                    database + "', Collection: '" + collection +
+                                    "', Shard: '" + shard + "'"};
+
   try {
     auto& df = _feature.server().getFeature<DatabaseFeature>();
     DatabaseGuard guard(df, database);

diff --git a/arangod/Cluster/TakeoverShardLeadership.h → ...ntenanceActions/TakeoverShardLeadership.h b/arangod/Cluster/TakeoverShardLeadership.h → ...ntenanceActions/TakeoverShardLeadership.h
diff --git a/arangod/Cluster/UpdateCollection.cpp → ...r/MaintenanceActions/UpdateCollection.cpp b/arangod/Cluster/UpdateCollection.cpp → ...r/MaintenanceActions/UpdateCollection.cpp
@@ -43,6 +43,7 @@
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Methods/Collections.h"
 #include "VocBase/Methods/Databases.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;
@@ -89,6 +90,11 @@ bool UpdateCollection::first() {
   auto const& props = properties();
   Result res;
 
+  // Add task monitoring
+  auto task = task_monitoring::Task{"UpdateCollection for DB: '" + database +
+                                    "', Collection: '" + collection +
+                                    "', Shard: '" + shard + "'"};
+
   std::string from;
   _description.get("from", from);
 

diff --git a/arangod/Cluster/UpdateCollection.h → ...ter/MaintenanceActions/UpdateCollection.h b/arangod/Cluster/UpdateCollection.h → ...ter/MaintenanceActions/UpdateCollection.h
diff --git a/...god/Cluster/UpdateReplicatedLogAction.cpp → ...anceActions/UpdateReplicatedLogAction.cpp b/...god/Cluster/UpdateReplicatedLogAction.cpp → ...anceActions/UpdateReplicatedLogAction.cpp
@@ -41,6 +41,7 @@
 #include "UpdateReplicatedLogAction.h"
 #include "Utils/DatabaseGuard.h"
 #include "VocBase/vocbase.h"
+#include "TaskMonitoring/task.h"
 
 using namespace arangodb;
 using namespace arangodb::basics;
@@ -63,6 +64,11 @@ bool arangodb::maintenance::UpdateReplicatedLogAction::first() {
   auto const& database = _description.get(DATABASE);
   auto& df = _feature.server().getFeature<DatabaseFeature>();
 
+  // Add task monitoring
+  auto task =
+      task_monitoring::Task{"UpdateReplicatedLogAction for DB: '" + database +
+                            "', LogId: '" + std::to_string(logId.id()) + "'"};
+
   auto result = basics::catchToResult([&] {
     DatabaseGuard guard(df, database);
 

diff --git a/arangod/Cluster/UpdateReplicatedLogAction.h → ...enanceActions/UpdateReplicatedLogAction.h b/arangod/Cluster/UpdateReplicatedLogAction.h → ...enanceActions/UpdateReplicatedLogAction.h
@@ -23,7 +23,7 @@
 
 #pragma once
 
-#include "Cluster/ActionBase.h"
+#include "Cluster/MaintenanceActions/ActionBase.h"
 
 namespace arangodb::maintenance {