Merge pull request #447 from clEsperanto/fix-macos-leak

StRigaud · web-flow · commit 39e6556ab3db · 2025-08-21T15:56:07.000+02:00
minimize clBuildProgram calls
diff --git a/clic/include/device.hpp b/clic/include/device.hpp
@@ -4,13 +4,72 @@
 #include "clic.hpp"
 
 #include <iostream>
+#include <list>
 #include <memory>
 #include <sstream>
 #include <unordered_map>
 
 namespace cle
 {
 
+struct Cache
+{
+  static constexpr size_t MAX_PROGRAM_CACHE_SIZE = 64;
+
+  // Store program and its position in LRU list
+  struct Entry
+  {
+    std::shared_ptr<void>            program;
+    std::list<std::string>::iterator lru_it;
+  };
+
+  std::unordered_map<std::string, Entry> program_cache;
+  std::list<std::string>                 program_lru;
+
+  Cache() { program_cache.reserve(MAX_PROGRAM_CACHE_SIZE); }
+
+  ~Cache() = default;
+
+  auto
+  cacheProgram(const std::string & key, const std::shared_ptr<void> & program) -> void
+  {
+    auto it = program_cache.find(key);
+    if (it != program_cache.end())
+    {
+      // Move key to back (most recently used)
+      program_lru.erase(it->second.lru_it);
+      program_lru.push_back(key);
+      it->second.lru_it = std::prev(program_lru.end());
+      it->second.program = program;
+      return;
+    }
+    if (program_cache.size() >= MAX_PROGRAM_CACHE_SIZE)
+    {
+      // Remove oldest
+      auto oldest = program_lru.front();
+      program_lru.pop_front();
+      program_cache.erase(oldest);
+    }
+    program_lru.push_back(key);
+    program_cache[key] = { program, std::prev(program_lru.end()) };
+  }
+
+  auto
+  getCachedProgram(const std::string & key) -> std::shared_ptr<void>
+  {
+    auto it = program_cache.find(key);
+    if (it != program_cache.end())
+    {
+      // Move accessed key to back (most recently used)
+      program_lru.erase(it->second.lru_it);
+      program_lru.push_back(key);
+      it->second.lru_it = std::prev(program_lru.end());
+      return it->second.program;
+    }
+    return nullptr;
+  }
+};
+
 /**
  * @brief Device class
  *        This class is used to manage devices
@@ -145,6 +204,12 @@ class Device
   [[nodiscard]] virtual auto
   getLocalMemorySize() const -> size_t = 0;
 
+  [[nodiscard]] virtual auto
+  getProgramFromCache(const std::string & key) const -> std::shared_ptr<void> = 0;
+
+  virtual auto
+  addProgramToCache(const std::string & key, std::shared_ptr<void> program) -> void = 0;
+
   /**
    * @brief operator << for Device::Type
    */
@@ -412,10 +477,18 @@ class OpenCLDevice : public Device
   [[nodiscard]] auto
   getLocalMemorySize() const -> size_t override;
 
+
+  [[nodiscard]] auto
+  getProgramFromCache(const std::string & key) const -> std::shared_ptr<void> override;
+
+  auto
+  addProgramToCache(const std::string & key, std::shared_ptr<void> program) -> void override;
+
 private:
   std::shared_ptr<Ressources>   clRessources = nullptr;
   std::shared_ptr<Context>      clContext = nullptr;
   std::shared_ptr<CommandQueue> clCommandQueue = nullptr;
+  std::shared_ptr<Cache>        cache = std::make_shared<Cache>();
   bool                          initialized = false;
   bool                          waitFinish = false;
 };
@@ -582,13 +655,20 @@ class CUDADevice : public Device
   [[nodiscard]] auto
   getLocalMemorySize() const -> size_t override;
 
+  [[nodiscard]] auto
+  getProgramFromCache(const std::string & key) const -> std::shared_ptr<void> override;
+
+  auto
+  addProgramToCache(const std::string & key, std::shared_ptr<void> program) -> void override;
+
 private:
-  int       cudaDeviceIndex;
-  CUdevice  cudaDevice;
-  CUcontext cudaContext;
-  CUstream  cudaStream;
-  bool      initialized = false;
-  bool      waitFinish = false;
+  int              cudaDeviceIndex;
+  CUdevice         cudaDevice;
+  CUcontext        cudaContext;
+  CUstream         cudaStream;
+  shared_ptr<void> cache = std::make_shared<Cache>();
+  bool             initialized = false;
+  bool             waitFinish = false;
 };
 #endif // USE_CUDA
 
diff --git a/clic/src/cudadevice.cpp b/clic/src/cudadevice.cpp
@@ -243,6 +243,19 @@ CUDADevice::getInfoExtended() const -> std::string
   return getInfo();
 }
 
+auto
+CUDADevice::getProgramFromCache(const std::string & key) const -> std::shared_ptr<void>
+{
+  return cache->getCachedProgram(key);
+}
+
+auto
+CUDADevice::addProgramToCache(const std::string & key, std::shared_ptr<void> program) -> void
+{
+  cache->cacheProgram(key, program);
+}
+
+
 #endif // USE_CUDA
 
 } // namespace cle
diff --git a/clic/src/openclbackend.cpp b/clic/src/openclbackend.cpp
@@ -1,5 +1,6 @@
 #include "backend.hpp"
 #include "cle_preamble_cl.h"
+#include <list>
 
 #include <unordered_map>
 
@@ -1261,6 +1262,49 @@ CreateProgramFromSource(const Device::Pointer & device, const std::string & kern
 }
 #endif
 
+
+// static constexpr size_t                                       MAX_PROGRAM_CACHE_SIZE = 64;
+// static std::unordered_map<std::string, std::shared_ptr<void>> program_cache;
+// static std::list<std::string>                                 program_lru;
+
+// void
+// cacheProgram(const std::string & key, std::shared_ptr<void> program)
+// {
+//   std::cout << "caching program: " << key << std::endl;
+//   if (program_cache.find(key) != program_cache.end())
+//   {
+//     // Program already exists, update LRU
+//     program_lru.remove(key);
+//     program_lru.push_back(key);
+//     std::cout << "\tProgram already cached, updating LRU." << std::endl;
+//     return;
+//   }
+//   if (program_cache.size() >= MAX_PROGRAM_CACHE_SIZE)
+//   {
+//     // Remove oldest
+//     auto oldest = program_lru.front();
+//     program_lru.pop_front();
+//     program_cache.erase(oldest);
+//     std::cout << "\tCache full, removing oldest program" << std::endl;
+//   }
+//   program_cache[key] = program;
+//   program_lru.push_back(key);
+//   std::cout << program_cache.size() << " programs cached." << std::endl;
+//   return;
+// }
+
+// std::shared_ptr<void>
+// getCachedProgram(const std::string & key)
+// {
+//   std::cout << "fetching program from cache: " << key << std::endl;
+//   auto it = program_cache.find(key);
+//   if (it != program_cache.end())
+//   {
+//     return it->second;
+//   }
+//   return nullptr;
+// }
+
 auto
 OpenCLBackend::buildKernel(const Device::Pointer & device,
                            const std::string &     kernel_source,
@@ -1272,13 +1316,17 @@ OpenCLBackend::buildKernel(const Device::Pointer & device,
   cl_int     err;
   auto       opencl_device = std::dynamic_pointer_cast<const OpenCLDevice>(device);
   const auto use_cache = is_cache_enabled();
+  // std::shared_ptr<void> program = nullptr;
 
   std::hash<std::string> hasher;
   const auto             source_hash = std::to_string(hasher(kernel_source));
   const auto             device_hash = std::to_string(hasher(opencl_device->getInfo()));
 
-  std::shared_ptr<void> program = nullptr;
-  if (use_cache)
+  // fetch the internal cache to avoid rebuilding
+  const auto cache_key = device_hash + "_" + source_hash;
+  auto       program = device->getProgramFromCache(cache_key);
+
+  if (program == nullptr && use_cache)
   {
     program = loadProgramFromCache(device, device_hash, source_hash);
   }
@@ -1291,6 +1339,9 @@ OpenCLBackend::buildKernel(const Device::Pointer & device,
       saveBinaryToCache(device_hash, source_hash, program, device);
     }
   }
+
+  device->addProgramToCache(cache_key, program);
+
   auto ocl_kernel = clCreateKernel(reinterpret_cast<cl_program>(program.get()), kernel_name.c_str(), &err);
   if (err != CL_SUCCESS)
   {
diff --git a/clic/src/opencldevice.cpp b/clic/src/opencldevice.cpp
@@ -434,6 +434,18 @@ OpenCLDevice::getInfoExtended() const -> std::string
   return result.str();
 }
 
+auto
+OpenCLDevice::getProgramFromCache(const std::string & key) const -> std::shared_ptr<void>
+{
+  return cache->getCachedProgram(key);
+}
+
+auto
+OpenCLDevice::addProgramToCache(const std::string & key, std::shared_ptr<void> program) -> void
+{
+  cache->cacheProgram(key, program);
+}
+
 
 #endif // USE_OPENCL
 
