dynamic_modules: add scheduler & HTTP callback API to bootstrap ABI

Signed-off-by: Rohit Agrawal <[email protected]>

Author: agrawroh

source/extensions/access_loggers/dynamic_modules/BUILD

@@ -36,6 +36,7 @@ envoy_cc_library(
         "//source/common/http:header_utility_lib",
         "//source/common/http:utility_lib",
         "//source/extensions/access_loggers/common:access_log_base",
+        "//source/extensions/dynamic_modules:abi_impl",
         "//source/extensions/dynamic_modules:dynamic_modules_lib",
     ],
 )

source/extensions/bootstrap/dynamic_modules/BUILD

@@ -14,6 +14,7 @@ envoy_cc_library(
     srcs = ["extension_config.cc"],
     hdrs = ["extension_config.h"],
     deps = [
+        "//envoy/event:dispatcher_interface",
         "//source/common/common:assert_lib",
         "//source/extensions/dynamic_modules:dynamic_modules_lib",
     ],
@@ -32,11 +33,21 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "abi_impl",
+    srcs = ["abi_impl.cc"],
+    deps = [
+        ":extension_config_lib",
+        "//source/extensions/dynamic_modules:dynamic_modules_lib",
+    ],
+)
+
 envoy_cc_extension(
     name = "config",
     srcs = ["factory.cc"],
     hdrs = ["factory.h"],
     deps = [
+        ":abi_impl",
         ":extension_lib",
         "//envoy/registry",
         "//envoy/server:bootstrap_extension_config_interface",

source/extensions/bootstrap/dynamic_modules/abi_impl.cc

@@ -0,0 +1,35 @@
+// NOLINT(namespace-envoy)
+
+// This file provides host-side implementations for ABI callbacks specific to bootstrap extensions.
+
+#include "source/extensions/bootstrap/dynamic_modules/extension_config.h"
+#include "source/extensions/dynamic_modules/abi.h"
+
+using Envoy::Extensions::Bootstrap::DynamicModules::DynamicModuleBootstrapExtensionConfig;
+using Envoy::Extensions::Bootstrap::DynamicModules::DynamicModuleBootstrapExtensionConfigScheduler;
+
+extern "C" {
+
+envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr
+envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new(
+    envoy_dynamic_module_type_bootstrap_extension_config_envoy_ptr extension_config_envoy_ptr) {
+  auto* config = static_cast<DynamicModuleBootstrapExtensionConfig*>(extension_config_envoy_ptr);
+  return new DynamicModuleBootstrapExtensionConfigScheduler(config->weak_from_this(),
+                                                            config->main_thread_dispatcher_);
+}
+
+void envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete(
+    envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr
+        scheduler_module_ptr) {
+  delete static_cast<DynamicModuleBootstrapExtensionConfigScheduler*>(scheduler_module_ptr);
+}
+
+void envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_commit(
+    envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr scheduler_module_ptr,
+    uint64_t event_id) {
+  auto* scheduler =
+      static_cast<DynamicModuleBootstrapExtensionConfigScheduler*>(scheduler_module_ptr);
+  scheduler->commit(event_id);
+}
+
+} // extern "C"

source/extensions/bootstrap/dynamic_modules/extension_config.cc

@@ -9,8 +9,9 @@ namespace DynamicModules {
 
 DynamicModuleBootstrapExtensionConfig::DynamicModuleBootstrapExtensionConfig(
     const absl::string_view extension_name, const absl::string_view extension_config,
-    Extensions::DynamicModules::DynamicModulePtr dynamic_module)
-    : dynamic_module_(std::move(dynamic_module)) {
+    Extensions::DynamicModules::DynamicModulePtr dynamic_module,
+    Event::Dispatcher& main_thread_dispatcher)
+    : dynamic_module_(std::move(dynamic_module)), main_thread_dispatcher_(main_thread_dispatcher) {
   ASSERT(dynamic_module_ != nullptr);
   ASSERT(extension_name.data() != nullptr);
   ASSERT(extension_config.data() != nullptr);
@@ -22,10 +23,17 @@ DynamicModuleBootstrapExtensionConfig::~DynamicModuleBootstrapExtensionConfig()
   }
 }
 
+void DynamicModuleBootstrapExtensionConfig::onScheduled(uint64_t event_id) {
+  if (in_module_config_ != nullptr && on_bootstrap_extension_config_scheduled_ != nullptr) {
+    on_bootstrap_extension_config_scheduled_(thisAsVoidPtr(), in_module_config_, event_id);
+  }
+}
+
 absl::StatusOr<DynamicModuleBootstrapExtensionConfigSharedPtr>
 newDynamicModuleBootstrapExtensionConfig(
     const absl::string_view extension_name, const absl::string_view extension_config,
-    Extensions::DynamicModules::DynamicModulePtr dynamic_module) {
+    Extensions::DynamicModules::DynamicModulePtr dynamic_module,
+    Event::Dispatcher& main_thread_dispatcher) {
 
   // Resolve the required symbols from the dynamic module.
   auto constructor =
@@ -69,8 +77,15 @@ newDynamicModuleBootstrapExtensionConfig(
     return on_extension_destroy.status();
   }
 
+  auto on_config_scheduled =
+      dynamic_module->getFunctionPointer<OnBootstrapExtensionConfigScheduledType>(
+          "envoy_dynamic_module_on_bootstrap_extension_config_scheduled");
+  if (!on_config_scheduled.ok()) {
+    return on_config_scheduled.status();
+  }
+
   auto config = std::make_shared<DynamicModuleBootstrapExtensionConfig>(
-      extension_name, extension_config, std::move(dynamic_module));
+      extension_name, extension_config, std::move(dynamic_module), main_thread_dispatcher);
 
   const void* extension_config_module_ptr = (*constructor.value())(
       static_cast<void*>(config.get()), {extension_name.data(), extension_name.size()},
@@ -85,6 +100,7 @@ newDynamicModuleBootstrapExtensionConfig(
   config->on_bootstrap_extension_server_initialized_ = on_server_initialized.value();
   config->on_bootstrap_extension_worker_thread_initialized_ = on_worker_thread_initialized.value();
   config->on_bootstrap_extension_destroy_ = on_extension_destroy.value();
+  config->on_bootstrap_extension_config_scheduled_ = on_config_scheduled.value();
 
   return config;
 }

source/extensions/bootstrap/dynamic_modules/extension_config.h

@@ -2,6 +2,8 @@
 
 #include <string>
 
+#include "envoy/event/dispatcher.h"
+
 #include "source/extensions/dynamic_modules/abi.h"
 #include "source/extensions/dynamic_modules/dynamic_modules.h"
 
@@ -19,27 +21,43 @@ using OnBootstrapExtensionWorkerThreadInitializedType =
     decltype(&envoy_dynamic_module_on_bootstrap_extension_worker_thread_initialized);
 using OnBootstrapExtensionDestroyType =
     decltype(&envoy_dynamic_module_on_bootstrap_extension_destroy);
+using OnBootstrapExtensionConfigScheduledType =
+    decltype(&envoy_dynamic_module_on_bootstrap_extension_config_scheduled);
 
 class DynamicModuleBootstrapExtension;
 
 /**
  * A config to create bootstrap extensions based on a dynamic module. This will be owned by the
  * bootstrap extension. This resolves and holds the symbols used for the bootstrap extension.
  */
-class DynamicModuleBootstrapExtensionConfig {
+class DynamicModuleBootstrapExtensionConfig
+    : public std::enable_shared_from_this<DynamicModuleBootstrapExtensionConfig> {
 public:
   /**
    * Constructor for the config.
    * @param extension_name the name of the extension.
    * @param extension_config the configuration for the module.
    * @param dynamic_module the dynamic module to use.
+   * @param main_thread_dispatcher the main thread dispatcher.
    */
-  DynamicModuleBootstrapExtensionConfig(
-      const absl::string_view extension_name, const absl::string_view extension_config,
-      Extensions::DynamicModules::DynamicModulePtr dynamic_module);
+  DynamicModuleBootstrapExtensionConfig(const absl::string_view extension_name,
+                                        const absl::string_view extension_config,
+                                        Extensions::DynamicModules::DynamicModulePtr dynamic_module,
+                                        Event::Dispatcher& main_thread_dispatcher);
 
   ~DynamicModuleBootstrapExtensionConfig();
 
+  /**
+   * This is called when an event is scheduled via
+   * DynamicModuleBootstrapExtensionConfigScheduler::commit.
+   */
+  void onScheduled(uint64_t event_id);
+
+  /**
+   * Helper to get the `this` pointer as a void pointer.
+   */
+  void* thisAsVoidPtr() { return static_cast<void*>(this); }
+
   // The corresponding in-module configuration.
   envoy_dynamic_module_type_bootstrap_extension_config_module_ptr in_module_config_ = nullptr;
 
@@ -52,26 +70,60 @@ class DynamicModuleBootstrapExtensionConfig {
   OnBootstrapExtensionWorkerThreadInitializedType
       on_bootstrap_extension_worker_thread_initialized_ = nullptr;
   OnBootstrapExtensionDestroyType on_bootstrap_extension_destroy_ = nullptr;
+  OnBootstrapExtensionConfigScheduledType on_bootstrap_extension_config_scheduled_ = nullptr;
 
   // The dynamic module.
   Extensions::DynamicModules::DynamicModulePtr dynamic_module_;
+
+  // The main thread dispatcher.
+  Event::Dispatcher& main_thread_dispatcher_;
 };
 
 using DynamicModuleBootstrapExtensionConfigSharedPtr =
     std::shared_ptr<DynamicModuleBootstrapExtensionConfig>;
 
+/**
+ * This class is used to schedule a bootstrap extension config event hook from a different thread
+ * than the main thread. This is created via
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new and deleted via
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete.
+ */
+class DynamicModuleBootstrapExtensionConfigScheduler {
+public:
+  DynamicModuleBootstrapExtensionConfigScheduler(
+      std::weak_ptr<DynamicModuleBootstrapExtensionConfig> config, Event::Dispatcher& dispatcher)
+      : config_(std::move(config)), dispatcher_(dispatcher) {}
+
+  void commit(uint64_t event_id) {
+    dispatcher_.post([config = config_, event_id]() {
+      if (std::shared_ptr<DynamicModuleBootstrapExtensionConfig> config_shared = config.lock()) {
+        config_shared->onScheduled(event_id);
+      }
+    });
+  }
+
+private:
+  // The config that this scheduler is associated with. Using a weak pointer to avoid unnecessarily
+  // extending the lifetime of the config.
+  std::weak_ptr<DynamicModuleBootstrapExtensionConfig> config_;
+  // The dispatcher is used to post the event to the main thread.
+  Event::Dispatcher& dispatcher_;
+};
+
 /**
  * Creates a new DynamicModuleBootstrapExtensionConfig from the given module and configuration.
  * @param extension_name the name of the extension.
  * @param extension_config the configuration for the module.
  * @param dynamic_module the dynamic module to use.
+ * @param main_thread_dispatcher the main thread dispatcher.
  * @return an error status if the module could not be loaded or the configuration could not be
  * created, or a shared pointer to the config.
  */
 absl::StatusOr<DynamicModuleBootstrapExtensionConfigSharedPtr>
 newDynamicModuleBootstrapExtensionConfig(
     const absl::string_view extension_name, const absl::string_view extension_config,
-    Extensions::DynamicModules::DynamicModulePtr dynamic_module);
+    Extensions::DynamicModules::DynamicModulePtr dynamic_module,
+    Event::Dispatcher& main_thread_dispatcher);
 
 } // namespace DynamicModules
 } // namespace Bootstrap

source/extensions/bootstrap/dynamic_modules/factory.cc

@@ -39,7 +39,8 @@ Server::BootstrapExtensionPtr DynamicModuleBootstrapExtensionFactory::createBoot
   }
 
   auto extension_config = newDynamicModuleBootstrapExtensionConfig(
-      proto_config.extension_name(), extension_config_str, std::move(dynamic_module.value()));
+      proto_config.extension_name(), extension_config_str, std::move(dynamic_module.value()),
+      context.mainThreadDispatcher());
 
   if (!extension_config.ok()) {
     throwEnvoyExceptionOrPanic("Failed to create extension config: " +

source/extensions/dynamic_modules/BUILD

@@ -27,6 +27,7 @@ envoy_cc_library(
     srcs = ["abi_impl.cc"],
     deps = [
         ":dynamic_modules_lib",
+        "//source/common/common:assert_lib",
         "//source/common/common:logger_lib",
     ],
 )

source/extensions/dynamic_modules/abi.h

@@ -4979,6 +4979,20 @@ typedef void* envoy_dynamic_module_type_bootstrap_extension_envoy_ptr;
  */
 typedef const void* envoy_dynamic_module_type_bootstrap_extension_module_ptr;
 
+/**
+ * envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr is a raw pointer to
+ * the DynamicModuleBootstrapExtensionConfigScheduler class in Envoy.
+ *
+ * OWNERSHIP: The allocation is done by Envoy but the module is responsible for managing the
+ * lifetime of the pointer. Notably, it must be explicitly destroyed by the module
+ * when scheduling the bootstrap extension config event is done. The creation of this pointer is
+ * done by envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new and the scheduling
+ * and destruction is done by
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete. Since its lifecycle is
+ * owned/managed by the module, this has _module_ptr suffix.
+ */
+typedef void* envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr;
+
 // =============================================================================
 // Bootstrap Extension Event Hooks
 // =============================================================================
@@ -5060,6 +5074,76 @@ void envoy_dynamic_module_on_bootstrap_extension_worker_thread_initialized(
 void envoy_dynamic_module_on_bootstrap_extension_destroy(
     envoy_dynamic_module_type_bootstrap_extension_module_ptr extension_module_ptr);
 
+/**
+ * envoy_dynamic_module_on_bootstrap_extension_config_scheduled is called when the bootstrap
+ * extension configuration is scheduled to be executed on the main thread with
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_commit callback.
+ *
+ * @param extension_config_envoy_ptr is the pointer to the DynamicModuleBootstrapExtensionConfig
+ * object.
+ * @param extension_config_ptr is the pointer to the in-module bootstrap extension configuration
+ * created by envoy_dynamic_module_on_bootstrap_extension_config_new.
+ * @param event_id is the ID of the event passed to
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_commit.
+ */
+void envoy_dynamic_module_on_bootstrap_extension_config_scheduled(
+    envoy_dynamic_module_type_bootstrap_extension_config_envoy_ptr extension_config_envoy_ptr,
+    envoy_dynamic_module_type_bootstrap_extension_config_module_ptr extension_config_ptr,
+    uint64_t event_id);
+
+// =============================================================================
+// Bootstrap Extension Callbacks
+// =============================================================================
+
+/**
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new is called by the module
+ * to create a new bootstrap extension configuration scheduler. The scheduler is used to dispatch
+ * bootstrap extension configuration operations to the main thread from any thread including the
+ * ones managed by the module.
+ *
+ * @param extension_config_envoy_ptr is the pointer to the DynamicModuleBootstrapExtensionConfig
+ * object.
+ * @return envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr is the pointer
+ * to the created bootstrap extension configuration scheduler.
+ *
+ * NOTE: it is caller's responsibility to delete the scheduler using
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete when it is no longer
+ * needed. See the comment on
+ * envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr.
+ */
+envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr
+envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new(
+    envoy_dynamic_module_type_bootstrap_extension_config_envoy_ptr extension_config_envoy_ptr);
+
+/**
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete is called by the
+ * module to delete the bootstrap extension configuration scheduler created by
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new.
+ *
+ * @param scheduler_module_ptr is the pointer to the bootstrap extension configuration scheduler
+ * created by envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new.
+ */
+void envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_delete(
+    envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr scheduler_module_ptr);
+
+/**
+ * envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_commit is called by the
+ * module to schedule a generic event to the bootstrap extension configuration on the main thread.
+ *
+ * This will eventually end up invoking envoy_dynamic_module_on_bootstrap_extension_config_scheduled
+ * event hook on the main thread.
+ *
+ * This can be called multiple times to schedule multiple events to the same configuration.
+ *
+ * @param scheduler_module_ptr is the pointer to the bootstrap extension configuration scheduler
+ * created by envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_new.
+ * @param event_id is the ID of the event. This can be used to differentiate between multiple
+ * events scheduled to the same configuration. It can be any module-defined value.
+ */
+void envoy_dynamic_module_callback_bootstrap_extension_config_scheduler_commit(
+    envoy_dynamic_module_type_bootstrap_extension_config_scheduler_module_ptr scheduler_module_ptr,
+    uint64_t event_id);
+
 #ifdef __cplusplus
 }
 #endif