Merge branch 'main' into extract_uri_parser

Author: sergiobayona

.gitignore

@@ -16,3 +16,13 @@
 .idea
 test.ann
 Gemfile.lock
+
+/log/*.log
+test/dummy/db/*.sqlite3
+test/dummy/db/*.sqlite3-*
+test/dummy/log/*.log
+test/dummy/storage/
+test/dummy/tmp/
+*.keep
+
+tags

.rspec

@@ -1,3 +1,4 @@
---format documentation
---color
 --require spec_helper
+--require rails_helper
+--format documentation
+--color

.tool-versions

@@ -1 +1 @@
-ruby 3.4
+ruby 3.4.4

CHANGELOG.md

@@ -11,6 +11,8 @@
 
 ## [Unreleased]
 - [COMPAT] [https://github.com/patterns-ai-core/langchainrb/pull/980] Suppress a Ruby 3.4 warning for URI parser.
+- [BREAKING] [https://github.com/patterns-ai-core/langchainrb/pull/997] Remove `Langchain::Vectorsearch::Epsilla` class
+- [BREAKING] [https://github.com/patterns-ai-core/langchainrb/pull/1003] Response classes are now namespaced under `Langchain::LLM::Response`, converted to Rails engine
 
 ## [0.19.5]
 - [BREAKING] [https://github.com/patterns-ai-core/langchainrb/pull/859] Add metadata support to PgVector storage
@@ -237,7 +239,7 @@
 
 ## [0.8.0]
 - [BREAKING] Updated llama_cpp.rb to 0.9.4. The model file format used by the underlying llama.cpp library has changed to GGUF. llama.cpp ships with scripts to convert existing files and GGUF format models can be downloaded from HuggingFace.
-- Introducing Langchain::LLM::GoogleVertexAi LLM provider
+- Introducing Langchain::LLM::GoogleVertexAI LLM provider
 
 ## [0.7.5] - 2023-11-13
 - Fixes

Gemfile

@@ -2,13 +2,19 @@
 
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in langchain.gemspec
+# Specify your gem's dependencies in langchain.gemspec.
 gemspec
 
-gem "rake", "~> 13.0"
+group :test do
+  gem "sqlite3"
+end
 
-gem "rspec", "~> 3.0"
-
-gem "standard", ">= 1.35.1"
-# Lets add rubocop explicitly here, we are using only standardrb rules in .rubocop.yml
-gem "rubocop"
+# Development and test dependencies
+group :development, :test do
+  gem "rspec", "~> 3.0"
+  gem "rspec-rails"
+  gem "standard", ">= 1.35.1"
+  gem "rubocop"
+  gem "rubocop-rails-omakase", require: false
+  gem "rake", "~> 13.0"
+end

Rakefile

@@ -1,17 +1,7 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-require "standard/rake"
-require "yard"
-
-RSpec::Core::RakeTask.new(:spec)
+require "bundler/setup"
 
-task default: :spec
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
 
-Rake::Task["spec"].enhance do
-  Rake::Task["standard:fix"].invoke
-end
-
-YARD::Rake::YardocTask.new do |t|
-end
+require "bundler/gem_tasks"

bin/console

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path("..", __dir__)
+ENGINE_PATH = File.expand_path("../lib/langchain/engine", __dir__)
+APP_PATH = File.expand_path("../test/dummy/config/application", __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+
+require "rails"
+# Pick the frameworks you want:
+# require "active_model/railtie"
+# require "active_job/railtie"
+# require "active_record/railtie"
+# require "active_storage/engine"
+# require "action_controller/railtie"
+# require "action_mailer/railtie"
+# require "action_mailbox/engine"
+# require "action_text/engine"
+# require "action_view/railtie"
+# require "action_cable/engine"
+# require "rails/test_unit/railtie"
+require "rails/engine/commands"

bin/rails

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "bundler/setup"
+
+# explicit rubocop config increases performance slightly while avoiding config confusion.
+ARGV.unshift("--config", File.expand_path("../.rubocop.yml", __dir__))
+
+load Gem.bin_path("rubocop", "rubocop")

bin/rubocop

@@ -0,0 +1,19 @@
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.acronym "AI21"
+  inflect.acronym "CSV"
+  inflect.acronym "HTML"
+  inflect.acronym "JSON"
+  inflect.acronym "JSONL"
+  inflect.acronym "LLM"
+  inflect.acronym "PDF"
+  inflect.acronym "OpenAI"
+  inflect.acronym "OpenAIResponse"
+  inflect.acronym "OpenAIMessage"
+  inflect.acronym "MistralAI"
+  inflect.acronym "MistralAIResponse"
+  inflect.acronym "MistralAIMessage"
+  inflect.acronym "GoogleVertexAI"
+  inflect.acronym "VertexAI"
+  inflect.acronym "AI21Response"
+  inflect.acronym "AI21Validator"
+end

bin/setup

@@ -0,0 +1,2 @@
+Langchain::Engine.routes.draw do
+end

config/initializers/inflections.rb

@@ -19,11 +19,11 @@ Gem::Specification.new do |spec|
   spec.metadata["changelog_uri"] = "https://github.com/patterns-ai-core/langchainrb/blob/main/CHANGELOG.md"
   spec.metadata["documentation_uri"] = "https://rubydoc.info/gems/langchainrb"
 
-  # Specify which files should be added to the gem when it is released.
-  spec.files = Dir["LICENSE.txt", "README.md", "CHANGELOG.md", "lib/**/*"]
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    Dir["{app,config,db,lib}/**/*", "LICENSE.txt", "Rakefile", "README.md"]
+  end
+
+  spec.add_dependency "rails", ">= 6.1.0"
 
   # dependencies
   # Not sure if we should require this as it only applies to OpenAI usecase.

config/routes.rb

@@ -1,77 +1,7 @@
-# frozen_string_literal: true
-
 require "logger"
-require "pathname"
-require "zeitwerk"
-require "uri"
-require "json"
-
-loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/langchainrb.rb")
-loader.inflector.inflect(
-  "ai21" => "AI21",
-  "ai21_response" => "AI21Response",
-  "ai21_validator" => "AI21Validator",
-  "csv" => "CSV",
-  "google_vertex_ai" => "GoogleVertexAI",
-  "html" => "HTML",
-  "json" => "JSON",
-  "jsonl" => "JSONL",
-  "llm" => "LLM",
-  "mistral_ai" => "MistralAI",
-  "mistral_ai_response" => "MistralAIResponse",
-  "mistral_ai_message" => "MistralAIMessage",
-  "openai" => "OpenAI",
-  "openai_response" => "OpenAIResponse",
-  "openai_message" => "OpenAIMessage",
-  "pdf" => "PDF"
-)
-
-loader.collapse("#{__dir__}/langchain/llm/response")
+require "langchain/version"
+require "langchain/engine"
 
-loader.setup
-
-# Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
-#
-# = Installation
-# Install the gem and add to the application's Gemfile by executing:
-#
-#     $ bundle add langchainrb
-#
-# If bundler is not being used to manage dependencies, install the gem by executing:
-#
-#     $ gem install langchainrb
-#
-# Require the gem to start using it:
-#
-#     require "langchain"
-#
-# = Concepts
-#
-# == Processors
-# Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
-#
-# == Chunkers
-# Chunkers split data based on various available options such as delimeters, chunk sizes or custom-defined functions. Chunkers are used when data needs to be split up before being imported in vector databases.
-#
-# == Prompts
-# Prompts are structured inputs to the LLMs. Prompts provide instructions, context and other user input that LLMs use to generate responses.
-#
-# == Large Language Models (LLMs)
-# LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
-#
-# == Vectorsearch Databases
-# Vector database is a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
-#
-# == Embedding
-# Word embedding or word vector is an approach with which we represent documents and words. It is defined as a numeric vector input that allows words with similar meanings to have the same representation. It can approximate meaning and represent a word in a lower dimensional space.
-#
-#
-# = Logging
-#
-# Langchain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
-#
-# Langchain.logger.level = :info
 module Langchain
   class << self
     # @return [Logger]
@@ -126,6 +56,5 @@ def colorize_logger_msg(msg, severity)
   }.freeze
 
   self.logger ||= ::Logger.new($stdout, **LOGGER_OPTIONS)
-
   @root = Pathname.new(__dir__)
 end

langchain.gemspec

@@ -0,0 +1,5 @@
+module Langchain
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

lib/langchain.rb

@@ -0,0 +1,8 @@
+module Langchain
+  class Engine < ::Rails::Engine
+    isolate_namespace Langchain
+
+    config.autoload_paths << root.join("lib")
+    config.eager_load_paths << root.join("lib")
+  end
+end

lib/langchain/application_record.rb

@@ -37,7 +37,7 @@ def complete(prompt:, **params)
       parameters = complete_parameters params
 
       response = client.complete(prompt, parameters)
-      Langchain::LLM::AI21Response.new response, model: parameters[:model]
+      Langchain::LLM::Response::AI21Response.new response, model: parameters[:model]
     end
 
     #

lib/langchain/assistant/llm/adapters/mistral_ai.rb renamed to lib/langchain/assistant/llm/adapters/mistralai.rb

@@ -57,7 +57,7 @@ def initialize(api_key:, llm_options: {}, default_options: {})
     # @param top_k [Integer] The top k value to use
     # @param metadata [Hash] The metadata to use
     # @param stream [Boolean] Whether to stream the response
-    # @return [Langchain::LLM::AnthropicResponse] The completion
+    # @return [Langchain::LLM::Response::AnthropicResponse] The completion
     def complete(
       prompt:,
       model: @defaults[:completion_model],
@@ -88,7 +88,7 @@ def complete(
         client.complete(parameters: parameters)
       end
 
-      Langchain::LLM::AnthropicResponse.new(response)
+      Langchain::LLM::Response::AnthropicResponse.new(response)
     end
 
     # Generate a chat completion for given messages
@@ -106,7 +106,7 @@ def complete(
     # @option params [Hash] :thinking Enable extended thinking mode, e.g. { type: "enabled", budget_tokens: 4000 }
     # @option params [Integer] :top_k Only sample from the top K options for each subsequent token
     # @option params [Float] :top_p Use nucleus sampling.
-    # @return [Langchain::LLM::AnthropicResponse] The chat completion
+    # @return [Langchain::LLM::Response::AnthropicResponse] The chat completion
     def chat(params = {}, &block)
       set_extra_headers! if params[:tools]
 
@@ -129,7 +129,7 @@ def chat(params = {}, &block)
       response = response_from_chunks if block
       reset_response_chunks
 
-      Langchain::LLM::AnthropicResponse.new(response)
+      Langchain::LLM::Response::AnthropicResponse.new(response)
     end
 
     def with_api_error_handling