Add support for retrieving previous dev_mode from state

- Introduced `_get_prev_dev_mode_from_state` method to read the development mode from the raw state.json, ensuring compatibility with engine versions.
- Updated `_configure` method to utilize the new method for determining previous dev_mode status.
- Enhanced pipeline state migration to include dev_mode and initial working directory in the local state for engine version upgrades.

Author: alkaline-0

dlt/pipeline/pipeline.py

@@ -1235,6 +1235,19 @@ def _recreate_initial_state(self) -> None:
         self._set_dataset_name(None)
         self._save_state(state)
 
+    def _get_prev_dev_mode_from_state(self) -> bool:
+        """Reads previous dev_mode from raw state.json only if engine supports it (>=2)."""
+        try:
+            raw = json_decode_state(self._pipeline_storage.load(Pipeline.STATE_FILE))
+        except FileNotFoundError:
+            return False
+        engine = int(raw.get("_state_engine_version", 1))
+        if engine >= 2:
+            return bool(raw.get("_local", {}).get("dev_mode", False))
+        else:
+            # legacy engine does not record dev flag
+            return False
+
     def _configure(
         self, import_schema_path: str, export_schema_path: str, must_attach_to_local_pipeline: bool
     ) -> None:
@@ -1258,7 +1271,7 @@ def _configure(
             )
 
         self.must_attach_to_local_pipeline = must_attach_to_local_pipeline
-        prev_dev_mode: bool = self.get_local_state_val("dev_mode") if has_state else False
+        prev_dev_mode: bool = self._get_prev_dev_mode_from_state() if has_state else False
         should_recreate_pipeline: bool = prev_dev_mode and not self.dev_mode
         self.set_local_state_val("dev_mode", self.dev_mode)  # create schema storage
         if has_state and not should_recreate_pipeline:

dlt/pipeline/state_sync.py

@@ -57,7 +57,10 @@ def migrate_pipeline_state(
     if from_engine == to_engine:
         return cast(TPipelineState, state)
     if from_engine == 1 and to_engine > 1:
-        state["_local"] = {}
+        state["_local"] = {
+            "dev_mode": state.get("dev_mode", False),
+            "initial_cwd": os.path.abspath(dlt.current.run_context().local_dir),
+        }
         from_engine = 2
     if from_engine == 2 and to_engine > 2:
         # you may want to recompute hash