feat: fallback to acceptable results (#1129)

* refactor: add is_acceptable, fallback logic and generify evolving agent

* refine lint

* small

* lint

* lint

* lint

* feat: add is_acceptable to CoSTEERMultiFeedback

* feat: add in-memory workspace checkpoint and recovery

* feat: preserve symbolic links in workspace checkpoints and recovery

* lint

* lint

* feat: limit workspace checkpoint to files under 100KB

* feat: add workspace checkpoint size limit setting

* prompt

* lint

Author: you-n-g

rdagent/components/coder/CoSTEER/__init__.py

@@ -1,4 +1,5 @@
 import pickle
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 
@@ -10,8 +11,7 @@
     CoSTEERRAGStrategyV2,
 )
 from rdagent.core.developer import Developer
-from rdagent.core.evaluation import Evaluator
-from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvoAgent
+from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvaluator, RAGEvoAgent
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import Experiment
 from rdagent.log import rdagent_logger as logger
@@ -22,15 +22,13 @@ class CoSTEER(Developer[Experiment]):
     def __init__(
         self,
         settings: CoSTEERSettings,
-        eva: Evaluator,
+        eva: RAGEvaluator,
         es: EvolvingStrategy,
         *args,
         evolving_version: int = 2,
         max_seconds: int | None = None,
         with_knowledge: bool = True,
-        with_feedback: bool = True,
         knowledge_self_gen: bool = True,
-        filter_final_evo: bool = True,
         max_loop: int | None = None,
         **kwargs,
     ) -> None:
@@ -47,9 +45,7 @@ def __init__(
         )
 
         self.with_knowledge = with_knowledge
-        self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
-        self.filter_final_evo = filter_final_evo
         self.evolving_strategy = es
         self.evaluator = eva
         self.evolving_version = evolving_version
@@ -71,25 +67,37 @@ def __init__(
             )
         )
 
+    def _get_last_fb(self) -> CoSTEERMultiFeedback:
+        fb = self.evolve_agent.evolving_trace[-1].feedback
+        assert fb is not None, "feedback is None"
+        assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback"
+        return fb
+
     def develop(self, exp: Experiment) -> Experiment:
 
         # init intermediate items
         evo_exp = EvolvingItem.from_experiment(exp)
 
-        self.evolve_agent = RAGEvoAgent(
+        self.evolve_agent = RAGEvoAgent[EvolvingItem](
             max_loop=self.max_loop,
             evolving_strategy=self.evolving_strategy,
             rag=self.rag,
             with_knowledge=self.with_knowledge,
-            with_feedback=self.with_feedback,
+            with_feedback=True,
             knowledge_self_gen=self.knowledge_self_gen,
             enable_filelock=self.settings.enable_filelock,
             filelock_path=self.settings.filelock_path,
         )
 
+        # Evolving the solution
         start_datetime = datetime.now()
+        fallback_evo_exp = None
         for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator):
             assert isinstance(evo_exp, Experiment)  # multiple inheritance
+            if self._get_last_fb().is_acceptable():
+                fallback_evo_exp = deepcopy(evo_exp)
+                fallback_evo_exp.create_ws_ckp()  # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation.
+
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
                 logger.info(f"evolving workspace: {sw}")
@@ -100,8 +108,16 @@ def develop(self, exp: Experiment) -> Experiment:
                 logger.info("Global timer is timeout, stop evolving")
                 break
 
-        if self.with_feedback and self.filter_final_evo:
-            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self.evolve_agent.evolving_trace[-1].feedback)
+        # if the final feedback is not finished(therefore acceptable), we will use the fallback solution.
+        try:
+            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self._get_last_fb())
+        except CoderError:
+            if fallback_evo_exp is not None:
+                logger.info("Fallback to the fallback solution.")
+                evo_exp = fallback_evo_exp
+                evo_exp.recover_ws_ckp()  # NOTE: recovering checkpoints for restoring files in the workspace to prevent inplace mutation.
+            else:
+                raise
 
         exp.sub_workspace_list = evo_exp.sub_workspace_list
         exp.experiment_workspace = evo_exp.experiment_workspace

rdagent/components/coder/CoSTEER/evaluators.py

@@ -181,6 +181,9 @@ def append(self, feedback: CoSTEERSingleFeedback) -> None:
     def __iter__(self):
         return iter(self.feedback_list)
 
+    def is_acceptable(self) -> bool:
+        return all(feedback.is_acceptable() for feedback in self.feedback_list)
+
     def finished(self) -> bool:
         """
         In some implementations, tasks may fail multiple times, leading agents to skip the implementation.

rdagent/components/coder/CoSTEER/evolvable_subjects.py

@@ -25,7 +25,7 @@ def __init__(
             self.sub_gt_implementations = sub_gt_implementations
 
     @classmethod
-    def from_experiment(cls, exp: Experiment) -> Experiment:
+    def from_experiment(cls, exp: Experiment) -> "EvolvingItem":
         ei = cls(sub_tasks=exp.sub_tasks)
         ei.based_experiments = exp.based_experiments
         ei.experiment_workspace = exp.experiment_workspace

rdagent/core/conf.py

@@ -55,6 +55,11 @@ class RDAgentSettings(ExtendedBaseSettings):
 
     # workspace conf
     workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace"
+    workspace_ckp_size_limit: int = 0
+    """
+    the checkpoint for the workspace is a zip file.
+    0 (or any value <=0) means *no* size limit for files in workspace checkpoints
+    """
 
     # multi processing conf
     multi_proc_n: int = 1

rdagent/core/evaluation.py

@@ -12,6 +12,13 @@ class Feedback:
         The building process of feedback will should be in evaluator
     """
 
+    def is_acceptable(self) -> bool:
+        """
+        Sometimes, the solution is already acceptable, but we still want to refine it.
+        So we use different logic to determine whether the solution is acceptable or finished.
+        """
+        return self.__bool__()
+
     def finished(self) -> bool:
         """
         In some implementations, tasks may fail multiple times, leading agents to skip the implementation.

rdagent/core/evolving_agent.py

@@ -2,24 +2,21 @@
 
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from contextlib import nullcontext
+from typing import Any, Generic, TypeVar
 
 from filelock import FileLock
 from tqdm import tqdm
 
-if TYPE_CHECKING:
-    from rdagent.core.evolving_framework import EvolvableSubjects
-
-from contextlib import nullcontext
-
 from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback
-from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep
+from rdagent.core.evolving_framework import EvolvableSubjects, EvolvingStrategy, EvoStep
 from rdagent.log import rdagent_logger as logger
 
 ASpecificEvaluator = TypeVar("ASpecificEvaluator", bound=Evaluator)
+ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects)
 
 
-class EvoAgent(ABC, Generic[ASpecificEvaluator]):
+class EvoAgent(ABC, Generic[ASpecificEvaluator, ASpecificEvolvableSubjects]):
 
     def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None:
         self.max_loop = max_loop
@@ -28,9 +25,9 @@ def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None:
     @abstractmethod
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: ASpecificEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         """
         yield EvolvableSubjects for caller for easier process control and logging.
         """
@@ -47,7 +44,7 @@ def evaluate(
         raise NotImplementedError
 
 
-class RAGEvoAgent(EvoAgent[RAGEvaluator]):
+class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[ASpecificEvolvableSubjects]):
 
     def __init__(
         self,
@@ -63,7 +60,7 @@ def __init__(
     ) -> None:
         super().__init__(max_loop, evolving_strategy)
         self.rag = rag
-        self.evolving_trace: list[EvoStep] = []
+        self.evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] = []
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
@@ -72,9 +69,9 @@ def __init__(
 
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: RAGEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         for evo_loop_id in tqdm(range(self.max_loop), "Implementing"):
             with logger.tag(f"evo_loop_{evo_loop_id}"):
                 # 1. RAG
@@ -91,7 +88,7 @@ def multistep_evolve(
                 )
 
                 # 3. Pack evolve results
-                es = EvoStep(evo, queried_knowledge)
+                es = EvoStep[ASpecificEvolvableSubjects](evo, queried_knowledge)
 
                 # 4. Evaluation
                 if self.with_feedback:

rdagent/core/evolving_framework.py

@@ -3,7 +3,7 @@
 import copy
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from rdagent.core.evaluation import EvaluableObj
 from rdagent.core.knowledge_base import KnowledgeBase
@@ -36,8 +36,11 @@ def clone(self) -> EvolvableSubjects:
         return copy.deepcopy(self)
 
 
+ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects)
+
+
 @dataclass
-class EvoStep:
+class EvoStep(Generic[ASpecificEvolvableSubjects]):
     """At a specific step,
     based on
     - previous trace
@@ -48,23 +51,24 @@ class EvoStep:
     (optional) After evaluation, we get feedback `feedback`.
     """
 
-    evolvable_subjects: EvolvableSubjects
+    evolvable_subjects: ASpecificEvolvableSubjects
+
     queried_knowledge: QueriedKnowledge | None = None
     feedback: Feedback | None = None
 
 
-class EvolvingStrategy(ABC):
+class EvolvingStrategy(ABC, Generic[ASpecificEvolvableSubjects]):
     def __init__(self, scen: Scenario) -> None:
         self.scen = scen
 
     @abstractmethod
     def evolve(
         self,
-        *evo: EvolvableSubjects,
-        evolving_trace: list[EvoStep] | None = None,
+        *evo: ASpecificEvolvableSubjects,
+        evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] | None = None,
         queried_knowledge: QueriedKnowledge | None = None,
         **kwargs: Any,
-    ) -> EvolvableSubjects:
+    ) -> ASpecificEvolvableSubjects:
         """The evolving trace is a list of (evolvable_subjects, feedback) ordered
         according to the time.
 
@@ -74,7 +78,7 @@ def evolve(
         """
 
 
-class RAGStrategy(ABC):
+class RAGStrategy(ABC, Generic[ASpecificEvolvableSubjects]):
     """Retrieval Augmentation Generation Strategy"""
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
@@ -91,7 +95,7 @@ def load_or_init_knowledge_base(
     @abstractmethod
     def query(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         evolving_trace: list[EvoStep],
         **kwargs: Any,
     ) -> QueriedKnowledge | None:
@@ -100,7 +104,7 @@ def query(
     @abstractmethod
     def generate_knowledge(
         self,
-        evolving_trace: list[EvoStep],
+        evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]],
         *,
         return_knowledge: bool = False,
         **kwargs: Any,