Cleanup and development of flag to enable/disable rendering (#1308)

Add createRenderer flag to enable/disable OpenGL rendering

Co-authored-by: Eric Undersander <eundersander@gmail.com>
Co-authored-by: Lawrence Chen <ldchen@fb.com>
Co-authored-by: Erik Wijmans <ewijmans2@gmail.com>

Author: 4 people

examples/tutorials/colabs/replay_tutorial.ipynb

@@ -83,7 +83,11 @@
    "source": [
     "\n",
     "\n",
-    "def make_configuration():\n",
+    "def make_configuration(settings):\n",
+    "    make_video_during_sim = False\n",
+    "    if \"make_video_during_sim\" in settings:\n",
+    "        make_video_during_sim = settings[\"make_video_during_sim\"]\n",
+    "\n",
     "    # simulator configuration\n",
     "    backend_cfg = habitat_sim.SimulatorConfiguration()\n",
     "    backend_cfg.scene_id = os.path.join(\n",
@@ -95,6 +99,7 @@
     "    # Enable gfx replay save. See also our call to sim.gfx_replay_manager.save_keyframe()\n",
     "    # below.\n",
     "    backend_cfg.enable_gfx_replay_save = True\n",
+    "    backend_cfg.create_renderer = make_video_during_sim\n",
     "\n",
     "    sensor_cfg = habitat_sim.CameraSensorSpec()\n",
     "    sensor_cfg.resolution = [544, 720]\n",
@@ -216,14 +221,15 @@
     "    args, _ = parser.parse_known_args()\n",
     "    show_video = args.show_video\n",
     "    make_video = args.make_video\n",
+    "    make_video_during_sim = False\n",
     "else:\n",
     "    show_video = False\n",
     "    make_video = False\n",
     "\n",
     "if make_video and not os.path.exists(output_path):\n",
     "    os.mkdir(output_path)\n",
     "\n",
-    "cfg = make_configuration()\n",
+    "cfg = make_configuration({\"make_video_during_sim\": make_video_during_sim})\n",
     "sim = None\n",
     "replay_filepath = \"./replay.json\"\n",
     "\n",
@@ -290,7 +296,7 @@
     "    duration=1.0,\n",
     "    agent_vel=np.array([0.5, 0.0, 0.0]),\n",
     "    look_rotation_vel=25.0,\n",
-    "    get_frames=make_video,\n",
+    "    get_frames=make_video_during_sim,\n",
     ")"
    ]
   },
@@ -333,7 +339,7 @@
     "    duration=2.0,\n",
     "    agent_vel=np.array([0.0, 0.0, -0.4]),\n",
     "    look_rotation_vel=-5.0,\n",
-    "    get_frames=make_video,\n",
+    "    get_frames=make_video_during_sim,\n",
     ")"
    ]
   },
@@ -361,7 +367,7 @@
     "    duration=2.0,\n",
     "    agent_vel=np.array([0.4, 0.0, 0.0]),\n",
     "    look_rotation_vel=-10.0,\n",
-    "    get_frames=make_video,\n",
+    "    get_frames=make_video_during_sim,\n",
     ")"
    ]
   },
@@ -381,7 +387,7 @@
    "outputs": [],
    "source": [
     "\n",
-    "if make_video:\n",
+    "if make_video_during_sim:\n",
     "    vut.make_video(\n",
     "        observations,\n",
     "        \"rgba_camera\",\n",

examples/tutorials/nb_python/replay_tutorial.py

@@ -70,7 +70,11 @@
 # %%
 
 
-def make_configuration():
+def make_configuration(settings):
+    make_video_during_sim = False
+    if "make_video_during_sim" in settings:
+        make_video_during_sim = settings["make_video_during_sim"]
+
     # simulator configuration
     backend_cfg = habitat_sim.SimulatorConfiguration()
     backend_cfg.scene_id = os.path.join(
@@ -82,6 +86,7 @@ def make_configuration():
     # Enable gfx replay save. See also our call to sim.gfx_replay_manager.save_keyframe()
     # below.
     backend_cfg.enable_gfx_replay_save = True
+    backend_cfg.create_renderer = make_video_during_sim
 
     sensor_cfg = habitat_sim.CameraSensorSpec()
     sensor_cfg.resolution = [544, 720]
@@ -177,14 +182,15 @@ def configure_lighting(sim):
     args, _ = parser.parse_known_args()
     show_video = args.show_video
     make_video = args.make_video
+    make_video_during_sim = False
 else:
     show_video = False
     make_video = False
 
 if make_video and not os.path.exists(output_path):
     os.mkdir(output_path)
 
-cfg = make_configuration()
+cfg = make_configuration({"make_video_during_sim": make_video_during_sim})
 sim = None
 replay_filepath = "./replay.json"
 
@@ -225,7 +231,7 @@ def configure_lighting(sim):
     duration=1.0,
     agent_vel=np.array([0.5, 0.0, 0.0]),
     look_rotation_vel=25.0,
-    get_frames=make_video,
+    get_frames=make_video_during_sim,
 )
 
 # %% [markdown]
@@ -255,7 +261,7 @@ def configure_lighting(sim):
     duration=2.0,
     agent_vel=np.array([0.0, 0.0, -0.4]),
     look_rotation_vel=-5.0,
-    get_frames=make_video,
+    get_frames=make_video_during_sim,
 )
 
 # %% [markdown]
@@ -270,14 +276,14 @@ def configure_lighting(sim):
     duration=2.0,
     agent_vel=np.array([0.4, 0.0, 0.0]),
     look_rotation_vel=-10.0,
-    get_frames=make_video,
+    get_frames=make_video_during_sim,
 )
 
 # %% [markdown]
 # ## End the episode. Render the episode observations to a video.
 # %%
 
-if make_video:
+if make_video_during_sim:
     vut.make_video(
         observations,
         "rgba_camera",

habitat_sim/simulator.py

@@ -539,7 +539,8 @@ def __init__(self, sim: Simulator, agent: Agent, sensor_id: str) -> None:
 
         self._spec = self._sensor_object.specification()
 
-        self._sim.renderer.bind_render_target(self._sensor_object)
+        if self._sim.renderer is not None:
+            self._sim.renderer.bind_render_target(self._sensor_object)
 
         if self._spec.gpu2gpu_transfer:
             assert cuda_enabled, "Must build habitat sim with cuda for gpu2gpu-transfer"
@@ -605,6 +606,7 @@ def __init__(self, sim: Simulator, agent: Agent, sensor_id: str) -> None:
         )
 
     def draw_observation(self) -> None:
+        assert self._sim.renderer is not None
         # see if the sensor is attached to a scene graph, otherwise it is invalid,
         # and cannot make any observation
         if not self._sensor_object.object:
@@ -615,6 +617,7 @@ def draw_observation(self) -> None:
         self._sim.renderer.draw(self._sensor_object, self._sim)
 
     def _draw_observation_async(self) -> None:
+        assert self._sim.renderer is not None
         if (
             self._spec.sensor_type == SensorType.SEMANTIC
             and self._sim.get_active_scene_graph()
@@ -669,6 +672,7 @@ def _draw_observation_async(self) -> None:
         )
 
     def get_observation(self) -> Union[ndarray, "Tensor"]:
+        assert self._sim.renderer is not None
         tgt = self._sensor_object.render_target
 
         if self._spec.gpu2gpu_transfer:
@@ -682,7 +686,6 @@ def get_observation(self) -> Union[ndarray, "Tensor"]:
 
                 obs = self._buffer.flip(0)  # type: ignore[union-attr]
         else:
-
             if self._spec.sensor_type == SensorType.SEMANTIC:
                 tgt.read_frame_object_id(self.view)
             elif self._spec.sensor_type == SensorType.DEPTH:

src/esp/assets/ResourceManager.cpp

@@ -124,7 +124,15 @@ void ResourceManager::buildImporters() {
 
 }  // buildImporters
 
+bool ResourceManager::getCreateRenderer() const {
+  return metadataMediator_->getCreateRenderer();
+}
+
 void ResourceManager::initDefaultPrimAttributes() {
+  if (!getCreateRenderer()) {
+    return;
+  }
+
   // by this point, we should have a GL::Context so load the bb primitive.
   // TODO: replace this completely with standard mesh (i.e. treat the bb
   // wireframe cube no differently than other primivite-based rendered
@@ -996,7 +1004,9 @@ void ResourceManager::buildPrimitiveAssetData(
   // compute the mesh bounding box
   primMeshData->BB = computeMeshBB(primMeshData.get());
 
-  primMeshData->uploadBuffersToGPU(false);
+  if (getCreateRenderer()) {
+    primMeshData->uploadBuffersToGPU(false);
+  }
 
   // make MeshMetaData
   int meshStart = nextMeshID_++;
@@ -1100,7 +1110,9 @@ scene::SceneNode* ResourceManager::createRenderAssetInstancePTex(
   for (int iMesh = start; iMesh <= end; ++iMesh) {
     auto* pTexMeshData = dynamic_cast<PTexMeshData*>(meshes_.at(iMesh).get());
 
-    pTexMeshData->uploadBuffersToGPU(false);
+    if (getCreateRenderer()) {
+      pTexMeshData->uploadBuffersToGPU(false);
+    }
 
     for (int jSubmesh = 0; jSubmesh < pTexMeshData->getSize(); ++jSubmesh) {
       scene::SceneNode& node = instanceRoot->createChild();
@@ -1160,7 +1172,9 @@ bool ResourceManager::loadRenderAssetIMesh(const AssetInfo& info) {
 
   for (int meshIDLocal = 0; meshIDLocal < instanceMeshes.size();
        ++meshIDLocal) {
-    instanceMeshes[meshIDLocal]->uploadBuffersToGPU(false);
+    if (getCreateRenderer()) {
+      instanceMeshes[meshIDLocal]->uploadBuffersToGPU(false);
+    }
     meshes_.emplace(meshStart + meshIDLocal,
                     std::move(instanceMeshes[meshIDLocal]));
 
@@ -1204,7 +1218,7 @@ scene::SceneNode* ResourceManager::createRenderAssetInstanceIMesh(
     // That means One CANNOT query the data like e.g.,
     // meshes_.at(iMesh)->getMeshData()->hasAttribute(Mn::Trade::MeshAttribute::Tangent)
     // It will SEGFAULT!
-    createDrawable(*(meshes_.at(iMesh)->getMagnumGLMesh()),  // render mesh
+    createDrawable(meshes_.at(iMesh)->getMagnumGLMesh(),  // render mesh
                    meshAttributeFlags,                 // mesh attribute flags
                    node,                               // scene node
                    creation.lightSetupKey,             // lightSetup key
@@ -1239,7 +1253,8 @@ bool ResourceManager::loadRenderAssetGeneral(const AssetInfo& info) {
   assimpmetadata->configuration().setValue("ImportColladaIgnoreUpDirection",
                                            "true");
 #endif
-  {
+
+  if (requiresTextures_) {
     Cr::PluginManager::PluginMetadata* const metadata =
         importerManager_.metadata("BasisImporter");
     Mn::GL::Context& context = Mn::GL::Context::current();
@@ -1460,6 +1475,8 @@ bool ResourceManager::buildTrajectoryVisualization(
   // compute the mesh bounding box
   visMeshData->BB = computeMeshBB(visMeshData.get());
 
+  ESP_CHECK(getCreateRenderer(),
+            "buildTrajectoryVisualization requires a renderer");
   visMeshData->uploadBuffersToGPU(false);
 
   // make MeshMetaData
@@ -1507,8 +1524,13 @@ int ResourceManager::loadNavMeshVisualization(esp::nav::PathFinder& pathFinder,
                                               DrawableGroup* drawables) {
   int navMeshPrimitiveID = ID_UNDEFINED;
 
-  if (!pathFinder.isLoaded())
+  if (!pathFinder.isLoaded()) {
     return navMeshPrimitiveID;
+  }
+
+  if (!getCreateRenderer()) {
+    return navMeshPrimitiveID;
+  }
 
   // create the mesh
   std::vector<Magnum::UnsignedInt> indices;
@@ -1795,7 +1817,9 @@ void ResourceManager::loadMeshes(Importer& importer,
     // compute the mesh bounding box
     gltfMeshData->BB = computeMeshBB(gltfMeshData.get());
 
-    gltfMeshData->uploadBuffersToGPU(false);
+    if (getCreateRenderer()) {
+      gltfMeshData->uploadBuffersToGPU(false);
+    }
     meshes_.emplace(meshStart + iMesh, std::move(gltfMeshData));
   }
 }  // ResourceManager::loadMeshes
@@ -2058,7 +2082,15 @@ void ResourceManager::addComponent(
   // Add a drawable if the object has a mesh and the mesh is loaded
   if (meshIDLocal != ID_UNDEFINED) {
     const int meshID = metaData.meshIndex.first + meshIDLocal;
-    Magnum::GL::Mesh& mesh = *meshes_.at(meshID)->getMagnumGLMesh();
+    Magnum::GL::Mesh* mesh = meshes_.at(meshID)->getMagnumGLMesh();
+    if (getCreateRenderer()) {
+      CORRADE_ASSERT(mesh,
+                     "::addComponent() : GL mesh expected but not found", );
+    } else {
+      CORRADE_ASSERT(!mesh,
+                     "addComponent() : encountered unexpected GL mesh with "
+                     "createRenderer==false", );
+    }
     Mn::ResourceKey materialKey = meshTransformNode.materialID;
 
     gfx::Drawable::Flags meshAttributeFlags{};
@@ -2115,20 +2147,20 @@ void ResourceManager::addPrimitiveToDrawables(int primitiveID,
   // so do not need to worry about the tangent or bitangent.
   // it might be changed in the future.
   gfx::Drawable::Flags meshAttributeFlags{};
-  createDrawable(*primitive_meshes_.at(primitiveID),  // render mesh
-                 meshAttributeFlags,                  // meshAttributeFlags
-                 node,                                // scene node
-                 NO_LIGHT_KEY,                        // lightSetup key
-                 WHITE_MATERIAL_KEY,                  // material key
-                 drawables);                          // drawable group
+  createDrawable(primitive_meshes_.at(primitiveID).get(),  // render mesh
+                 meshAttributeFlags,                       // meshAttributeFlags
+                 node,                                     // scene node
+                 NO_LIGHT_KEY,                             // lightSetup key
+                 WHITE_MATERIAL_KEY,                       // material key
+                 drawables);                               // drawable group
 }
 
 void ResourceManager::removePrimitiveMesh(int primitiveID) {
   CHECK(primitive_meshes_.count(primitiveID));
   primitive_meshes_.erase(primitiveID);
 }
 
-void ResourceManager::createDrawable(Mn::GL::Mesh& mesh,
+void ResourceManager::createDrawable(Mn::GL::Mesh* mesh,
                                      gfx::Drawable::Flags& meshAttributeFlags,
                                      scene::SceneNode& node,
                                      const Mn::ResourceKey& lightSetupKey,
@@ -2508,5 +2540,6 @@ void ResourceManager::createConvexHullDecomposition(
   }
 }
 #endif
+
 }  // namespace assets
 }  // namespace esp

src/esp/assets/ResourceManager.h

@@ -88,6 +88,8 @@ namespace assets {
  */
 class ResourceManager {
  public:
+  bool getCreateRenderer() const;
+
   /** @brief Stores references to a set of drawable elements */
   using DrawableGroup = gfx::DrawableGroup;
   /** @brief Convenience typedef for Importer class */
@@ -488,7 +490,7 @@ class ResourceManager {
    * gfx::Drawable.
    */
 
-  void createDrawable(Mn::GL::Mesh& mesh,
+  void createDrawable(Mn::GL::Mesh* mesh,
                       gfx::Drawable::Flags& meshAttributeFlags,
                       scene::SceneNode& node,
                       const Mn::ResourceKey& lightSetupKey,

src/esp/gfx/Drawable.cpp

@@ -11,7 +11,7 @@ namespace esp {
 namespace gfx {
 uint64_t Drawable::drawableIdCounter = 0;
 Drawable::Drawable(scene::SceneNode& node,
-                   Magnum::GL::Mesh& mesh,
+                   Magnum::GL::Mesh* mesh,
                    DrawableType type,
                    DrawableGroup* group /* = nullptr */)
     : Magnum::SceneGraph::Drawable3D{node, group},