Implement dynamic loading of libchdb.so via dlopen and auto-download during gem install

.rubocop.yml

@@ -4,6 +4,7 @@ AllCops:
     - 'spec/**/*' 
     - 'tmp/**/*'
     - 'ext/**/extconf.rb'
+    - 'pkg/**/*'
 
   NewCops: enable
 

INSTALLATION.md

@@ -5,55 +5,23 @@ This document will help you install the `chdb-ruby` ruby gem.
 
 ## Installation
 
-### Native Gems 
+### Compilation During Installation 
 
-Native (precompiled) gems are available for recent Ruby versions on these platforms:
+The gem will automatically compile native extensions during installation (`gem install`). Supported platforms are:
 
 - `aarch64-linux` 
 - `arm64-darwin` 
 - `x86_64-linux`
 - `x86_64-darwin`
 
-If you are using one of these Ruby versions on one of these platforms, the native gem is the recommended way to install chdb-ruby.
+The `chdb-ruby` gem does not provide a pure Ruby implementation. Installation will fail if your platform is unsupported.
 
-`chdb-ruby` gem does not provide a pure Ruby implementation. Installation will fail if your platform is unsupported.
+## Runtime Dependencies
 
-## Post-Installation: Setting Up libchdb C++ Library
+The `chdb-ruby` gem requires the `libchdb` C++ library as its core engine. The library will be automatically downloaded during installation. The extension uses `dlopen` to dynamically load the library. No manual configuration is required.
 
-After installing the `chdb-ruby` gem, you must also install the `libchdb` C++ library locally. If the library path is not in your system's default search paths, you'll need to configure the runtime library loading path.
-
-### 1. Download the C++ Library
-
-You can either:
-- Use the automated installation script:
-  ```bash
-  curl -sSL https://github.com/chdb-io/chdb-io.github.io/blob/main/install_libchdb.sh | bash
-  ```
-
-- Or manually download from chdb releases(example for arm64-darwin (v3.12)):
-  ```bash
-  wget https://github.com/chdb-io/chdb/releases/download/v3.12/macos-arm64-libchdb.tar.gz
-  tar -xzf macos-arm64-libchdb.tar.gz
-  ```
-
-### 2. Configure Library Path
-- MacOS:
-    ```bash
-    export DYLD_LIBRARY_PATH="/path/to/libchdb:$DYLD_LIBRARY_PATH"
-    ```
-  (Add to your shell config file like ~/.zshrc for persistence)
-
-- Linux:
-    ```bash
-    export LD_LIBRARY_PATH="/path/to/libchdb:$LD_LIBRARY_PATH"
-    ```
-
-### 3. Verify Installation
+## Verify Installation
 - Ruby:
     ```bash
     require 'chdb'
     ```
-
-- Troubleshooting(If you get "Library not loaded" errors):
-  - Verify the path in DYLD_LIBRARY_PATH/LD_LIBRARY_PATH is correct
-  - Ensure you downloaded the right version for your platform

chdb.gemspec

@@ -38,6 +38,18 @@ Gem::Specification.new do |s|
     'INSTALLATION.md',
     'LICENSE',
     'README.md',
+    'dependencies.yml',
+    'ext/chdb/chdb_handle.c',
+    'ext/chdb/chdb_handle.h',
+    'ext/chdb/chdb.c',
+    'ext/chdb/connection.c',
+    'ext/chdb/connection.h',
+    'ext/chdb/constants.h',
+    'ext/chdb/exception.c',
+    'ext/chdb/exception.h',
+    'ext/chdb/extconf.rb',
+    'ext/chdb/local_result.c',
+    'ext/chdb/local_result.h',
     'lib/chdb.rb',
     'lib/chdb/constants.rb',
     'lib/chdb/data_path.rb',
@@ -57,5 +69,5 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'csv', '~> 3.1'
 
-  # s.extensions << 'ext/chdb/extconf.rb'
+  s.extensions << 'ext/chdb/extconf.rb'
 end

ext/chdb/chdb.c

@@ -1,6 +1,7 @@
 #include <ruby.h>
 
 #include "chdb.h"
+#include "chdb_handle.h"
 #include "constants.h"
 #include "connection.h"
 #include "exception.h"
@@ -22,6 +23,7 @@ void Init_chdb_native()
     DEBUG_PRINT("Initializing chdb extension");
 
     init_exception();
+    init_chdb_handle();
     init_chdb_constants();
     init_local_result();
     init_connection();

ext/chdb/chdb_handle.c

@@ -0,0 +1,49 @@
+#include "chdb_handle.h"
+
+#include <dlfcn.h>
+#include <ruby.h>
+
+#include "exception.h"
+
+void *chdb_handle = NULL;
+connect_chdb_func connect_chdb_ptr = NULL;
+close_conn_func close_conn_ptr = NULL;
+query_conn_func query_conn_ptr = NULL;
+
+VALUE get_chdb_rb_path(void)
+{
+    VALUE chdb_module = rb_const_get(rb_cObject, rb_intern("ChDB"));
+    return rb_funcall(chdb_module, rb_intern("lib_file_path"), 0);
+}
+
+void init_chdb_handle()
+{
+    VALUE rb_path = get_chdb_rb_path();
+    VALUE lib_dir = rb_file_dirname(rb_file_dirname(rb_path));
+    VALUE lib_path = rb_str_cat2(lib_dir, "/lib/chdb/lib/libchdb.so");
+    // printf("chdb.rb path from Ruby: %s\n", StringValueCStr(lib_path));
+
+    connect_chdb_ptr = NULL;
+    close_conn_ptr = NULL;
+    query_conn_ptr = NULL;
+
+    chdb_handle = dlopen(RSTRING_PTR(lib_path), RTLD_LAZY | RTLD_GLOBAL);
+    if (!chdb_handle)
+    {
+        rb_raise(cChDBError, "Failed to load chdb library: %s\nCheck if libchdb.so exists at: %s",
+                 dlerror(), RSTRING_PTR(lib_path));
+    }
+
+    connect_chdb_ptr = (connect_chdb_func)dlsym(chdb_handle, "connect_chdb");
+    close_conn_ptr = (close_conn_func)dlsym(chdb_handle, "close_conn");
+    query_conn_ptr = (query_conn_func)dlsym(chdb_handle, "query_conn");
+
+    if (!connect_chdb_ptr || !close_conn_ptr || !query_conn_ptr)
+    {
+        rb_raise(cChDBError, "Symbol loading failed: %s\nMissing functions: connect_chdb(%p) close_conn(%p) query_conn(%p)",
+                 dlerror(),
+                 (void*)connect_chdb_ptr,
+                 (void*)close_conn_ptr,
+                 (void*)query_conn_ptr);
+    }
+}

ext/chdb/chdb_handle.h

@@ -0,0 +1,16 @@
+#ifndef CHDB_HANDLE_H
+#define CHDB_HANDLE_H
+
+typedef struct chdb_conn **(*connect_chdb_func)(int, char**);
+typedef void (*close_conn_func)(struct chdb_conn**);
+typedef struct local_result_v2 *(*query_conn_func)(struct chdb_conn*, const char*, const char*);
+
+extern connect_chdb_func connect_chdb_ptr;
+extern close_conn_func close_conn_ptr;
+extern query_conn_func query_conn_ptr;
+
+extern void *chdb_handle;
+
+void init_chdb_handle();
+
+#endif

ext/chdb/connection.c

@@ -1,10 +1,29 @@
 #include "connection.h"
 
+#include "chdb_handle.h"
 #include "constants.h"
 #include "exception.h"
 #include "include/chdb.h"
 #include "local_result.h"
 
+static void connection_free(void *ptr)
+{
+    Connection *conn = (Connection *)ptr;
+    DEBUG_PRINT("Closing connection: %p", (void*)conn->c_conn);
+    if (conn->c_conn)
+    {
+        close_conn_ptr(conn->c_conn);
+        conn->c_conn = NULL;
+    }
+    free(conn);
+}
+
+const rb_data_type_t ConnectionType =
+{
+    "Connection",
+    {NULL, connection_free, NULL},
+};
+
 void init_connection()
 {
     VALUE mChDB = rb_define_module("ChDB");
@@ -39,7 +58,7 @@ VALUE connection_initialize(VALUE self, VALUE argc, VALUE argv)
 
     Connection *conn;
     TypedData_Get_Struct(self, Connection, &ConnectionType, conn);
-    conn->c_conn = connect_chdb(c_argc, c_argv);
+    conn->c_conn = connect_chdb_ptr(c_argc, c_argv);
 
     if (!conn->c_conn)
     {
@@ -60,7 +79,7 @@ VALUE connection_query(VALUE self, VALUE query, VALUE format)
     Check_Type(query, T_STRING);
     Check_Type(format, T_STRING);
 
-    struct local_result_v2 *c_result = query_conn(
+    struct local_result_v2 *c_result = query_conn_ptr(
                                            *conn->c_conn,
                                            StringValueCStr(query),
                                            StringValueCStr(format)
@@ -92,27 +111,8 @@ VALUE connection_close(VALUE self)
 
     if (conn->c_conn)
     {
-        close_conn(conn->c_conn);
+        close_conn_ptr(conn->c_conn);
         conn->c_conn = NULL;
     }
     return Qnil;
 }
-
-static void connection_free(void *ptr)
-{
-    Connection *conn = (Connection *)ptr;
-    DEBUG_PRINT("Closing connection: %p", (void*)conn->c_conn);
-    if (conn->c_conn)
-    {
-        close_conn(conn->c_conn);
-    }
-    free(conn);
-}
-
-const rb_data_type_t ConnectionType =
-{
-    "Connection",
-    {NULL, connection_free, NULL},
-};
-
-// 其他 Connection 方法保持不变...