[Sensor] Fisheye and Equirectangular semantic sensors

examples/settings.py

@@ -17,11 +17,15 @@
     "color_sensor": True,  # RGB sensor (default: ON)
     "semantic_sensor": False,  # semantic sensor (default: OFF)
     "depth_sensor": False,  # depth sensor (default: OFF)
-    "ortho_sensor": False,  # Orthographic RGB sensor (default: OFF)
+    "ortho_rgb_sensor": False,  # Orthographic RGB sensor (default: OFF)
+    "ortho_depth_sensor": False,  # Orthographic RGB sensor (default: OFF)
+    "ortho_semantic_sensor": False,  # Orthographic RGB sensor (default: OFF)
     "fisheye_rgb_sensor": False,
     "fisheye_depth_sensor": False,
+    "fisheye_semantic_sensor": False,
     "equirect_rgb_sensor": False,
     "equirect_depth_sensor": False,
+    "equirect_semantic_sensor": False,
     "seed": 1,
     "silent": False,  # do not print log info (default: OFF)
     # settings exclusive to example.py
@@ -95,15 +99,33 @@ def make_cfg(settings):
         semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
         sensor_specs.append(semantic_sensor_spec)
 
-    if settings["ortho_sensor"]:
+    if settings["ortho_rgb_sensor"]:
         ortho_sensor_spec = habitat_sim.CameraSensorSpec()
-        ortho_sensor_spec.uuid = "ortho_sensor"
+        ortho_sensor_spec.uuid = "ortho_rgb_sensor"
         ortho_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
         ortho_sensor_spec.resolution = [settings["height"], settings["width"]]
         ortho_sensor_spec.position = [0, settings["sensor_height"], 0]
         ortho_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.ORTHOGRAPHIC
         sensor_specs.append(ortho_sensor_spec)
 
+    if settings["ortho_depth_sensor"]:
+        ortho_depth_sensor_spec = habitat_sim.CameraSensorSpec()
+        ortho_depth_sensor_spec.uuid = "ortho_depth_sensor"
+        ortho_depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
+        ortho_depth_sensor_spec.resolution = [settings["height"], settings["width"]]
+        ortho_depth_sensor_spec.position = [0, settings["sensor_height"], 0]
+        ortho_depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.ORTHOGRAPHIC
+        sensor_specs.append(ortho_depth_sensor_spec)
+
+    if settings["ortho_semantic_sensor"]:
+        ortho_sensor_spec = habitat_sim.CameraSensorSpec()
+        ortho_sensor_spec.uuid = "ortho_semantic_sensor"
+        ortho_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
+        ortho_sensor_spec.resolution = [settings["height"], settings["width"]]
+        ortho_sensor_spec.position = [0, settings["sensor_height"], 0]
+        ortho_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.ORTHOGRAPHIC
+        sensor_specs.append(ortho_sensor_spec)
+
     # TODO Figure out how to implement copying of specs
     def create_fisheye_spec(**kw_args):
         fisheye_sensor_spec = habitat_sim.FisheyeSensorDoubleSphereSpec()
@@ -113,17 +135,15 @@ def create_fisheye_spec(**kw_args):
             habitat_sim.FisheyeSensorModelType.DOUBLE_SPHERE
         )
 
-        # The default value (alpha, xi) is set to match the lens "BF2M2020S23" found in Table 3 of this paper:
+        # The default value (alpha, xi) is set to match the lens "GoPro" found in Table 3 of this paper:
         # Vladyslav Usenko, Nikolaus Demmel and Daniel Cremers: The Double Sphere
         # Camera Model, The International Conference on 3D Vision (3DV), 2018
         # You can find the intrinsic parameters for the other lenses in the same table as well.
-        # fisheye_sensor_spec.alpha = 0.59  # default
-        # fisheye_sensor_spec.xi = -0.18  # default
+        fisheye_sensor_spec.xi = -0.27
+        fisheye_sensor_spec.alpha = 0.57
+        fisheye_sensor_spec.focal_length = [364.84, 364.86]
 
         fisheye_sensor_spec.resolution = [settings["height"], settings["width"]]
-        fisheye_sensor_spec.focal_length = [
-            min(settings["height"], settings["width"]) * 0.5
-        ] * 2
         # The default principal_point_offset is the middle of the image
         fisheye_sensor_spec.principal_point_offset = mn.Vector2(
             settings["height"] // 2,
@@ -139,26 +159,49 @@ def create_fisheye_spec(**kw_args):
         sensor_specs.append(fisheye_rgb_sensor_spec)
     if settings["fisheye_depth_sensor"]:
         fisheye_depth_sensor_spec = create_fisheye_spec(
-            uuid="fisheye_depth_sensor", sensor_type=habitat_sim.SensorType.DEPTH
+            uuid="fisheye_depth_sensor",
+            sensor_type=habitat_sim.SensorType.DEPTH,
+            channels=1,
         )
         sensor_specs.append(fisheye_depth_sensor_spec)
+    if settings["fisheye_semantic_sensor"]:
+        fisheye_semantic_sensor_spec = create_fisheye_spec(
+            uuid="fisheye_semantic_sensor",
+            sensor_type=habitat_sim.SensorType.SEMANTIC,
+            channels=1,
+        )
+        sensor_specs.append(fisheye_semantic_sensor_spec)
+
+    def create_equirect_spec(**kw_args):
+        equirect_sensor_spec = habitat_sim.EquirectangularSensorSpec()
+        equirect_sensor_spec.uuid = "equirect_rgb_sensor"
+        equirect_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
+        equirect_sensor_spec.resolution = [settings["height"], settings["width"]]
+        equirect_sensor_spec.position = [0, settings["sensor_height"], 0]
+        for k in kw_args:
+            setattr(equirect_sensor_spec, k, kw_args[k])
+        return equirect_sensor_spec
 
     if settings["equirect_rgb_sensor"]:
-        equirect_rgb_sensor_spec = habitat_sim.EquirectangularSensorSpec()
-        equirect_rgb_sensor_spec.uuid = "equirect_rgb_sensor"
-        equirect_rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
-        equirect_rgb_sensor_spec.resolution = [settings["height"], settings["width"]]
-        equirect_rgb_sensor_spec.position = [0, settings["sensor_height"], 0]
+        equirect_rgb_sensor_spec = create_equirect_spec(uuid="equirect_rgb_sensor")
         sensor_specs.append(equirect_rgb_sensor_spec)
 
     if settings["equirect_depth_sensor"]:
-        equirect_depth_sensor_spec = habitat_sim.EquirectangularSensorSpec()
-        equirect_depth_sensor_spec.uuid = "equirect_depth_sensor"
-        equirect_depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
-        equirect_depth_sensor_spec.resolution = [settings["height"], settings["width"]]
-        equirect_depth_sensor_spec.position = [0, settings["sensor_height"], 0]
+        equirect_depth_sensor_spec = create_equirect_spec(
+            uuid="equirect_depth_sensor",
+            sensor_type=habitat_sim.SensorType.DEPTH,
+            channels=1,
+        )
         sensor_specs.append(equirect_depth_sensor_spec)
 
+    if settings["equirect_semantic_sensor"]:
+        equirect_semantic_sensor_spec = create_equirect_spec(
+            uuid="equirect_semantic_sensor",
+            sensor_type=habitat_sim.SensorType.SEMANTIC,
+            channels=1,
+        )
+        sensor_specs.append(equirect_semantic_sensor_spec)
+
     # create agent specifications
     agent_cfg = habitat_sim.agent.AgentConfiguration()
     agent_cfg.sensor_specifications = sensor_specs

src/esp/gfx/CubeMap.cpp

@@ -30,9 +30,10 @@ namespace Cr = Corrade;
 namespace esp {
 namespace gfx {
 
-// TODO:
-// const Mn::GL::Framebuffer::ColorAttachment objectIdAttachment =
-//    Mn::GL::Framebuffer::ColorAttachment{1};
+const Mn::GL::Framebuffer::ColorAttachment rgbaAttachment =
+    Mn::GL::Framebuffer::ColorAttachment{0};
+const Mn::GL::Framebuffer::ColorAttachment objectIdAttachment =
+    Mn::GL::Framebuffer::ColorAttachment{1};
 
 /**
  * @brief check if the class instance is created with corresponding texture
@@ -55,6 +56,12 @@ void textureTypeSanityCheck(CubeMap::Flags& flag,
                          << "instance was not created with depth "
                             "texture output enabled.", );
       return;
+    case CubeMap::TextureType::ObjectId:
+      CORRADE_ASSERT(flag & CubeMap::Flag::ObjectIdTexture,
+                     functionNameStr.c_str()
+                         << "instance was not created with object id"
+                            "texture output enabled.", );
+      return;
       break;
 
     case CubeMap::TextureType::Count:
@@ -89,6 +96,9 @@ const char* getTextureTypeFilenameString(CubeMap::TextureType type) {
     case CubeMap::TextureType::Depth:
       return "depth";
       break;
+    case CubeMap::TextureType::ObjectId:
+      return "objectId";
+      break;
     case CubeMap::TextureType::Count:
       break;
   }
@@ -107,10 +117,9 @@ Mn::PixelFormat getPixelFormat(CubeMap::TextureType type) {
     case CubeMap::TextureType::Depth:
       return Mn::PixelFormat::R32F;
       break;
-      /*
-      case CubeMap::TextureType::ObjectId:
+    case CubeMap::TextureType::ObjectId:
       return Mn::PixelFormat::R32UI;
-      */
+      break;
     case CubeMap::TextureType::Count:
       break;
   }
@@ -180,6 +189,16 @@ void CubeMap::recreateTexture() {
         .setMagnificationFilter(Mn::GL::SamplerFilter::Nearest)
         .setStorage(1, Mn::GL::TextureFormat::DepthComponent32F, size);
   }
+
+  // object id texture
+  if (flags_ & Flag::ObjectIdTexture) {
+    auto& objectIdTexture = texture(TextureType::ObjectId);
+    objectIdTexture = Mn::GL::CubeMapTexture{};
+    objectIdTexture.setWrapping(Mn::GL::SamplerWrapping::ClampToEdge)
+        .setMinificationFilter(Mn::GL::SamplerFilter::Nearest)
+        .setMagnificationFilter(Mn::GL::SamplerFilter::Nearest)
+        .setStorage(1, Mn::GL::TextureFormat::R32UI, size);
+  }
 }
 
 void CubeMap::recreateFramebuffer() {
@@ -201,17 +220,15 @@ void CubeMap::recreateFramebuffer() {
 
 void CubeMap::attachFramebufferRenderbuffer() {
   for (unsigned int index = 0; index < 6; ++index) {
+    Magnum::GL::CubeMapCoordinate cubeMapCoord =
+        convertFaceIndexToCubeMapCoordinate(index);
+
     if (flags_ & Flag::ColorTexture) {
-      Magnum::GL::CubeMapCoordinate cubeMapCoord =
-          convertFaceIndexToCubeMapCoordinate(index);
       frameBuffer_[index].attachCubeMapTexture(
-          Mn::GL::Framebuffer::ColorAttachment{0}, texture(TextureType::Color),
-          cubeMapCoord, 0);
+          rgbaAttachment, texture(TextureType::Color), cubeMapCoord, 0);
     }
 
     if (flags_ & Flag::DepthTexture) {
-      Magnum::GL::CubeMapCoordinate cubeMapCoord =
-          convertFaceIndexToCubeMapCoordinate(index);
       frameBuffer_[index].attachCubeMapTexture(
           Mn::GL::Framebuffer::BufferAttachment::Depth,
           texture(TextureType::Depth), cubeMapCoord, 0);
@@ -220,38 +237,55 @@ void CubeMap::attachFramebufferRenderbuffer() {
           Mn::GL::Framebuffer::BufferAttachment::Depth,
           optionalDepthBuffer_[index]);
     }
+
+    if (flags_ & Flag::ObjectIdTexture) {
+      frameBuffer_[index].attachCubeMapTexture(
+          objectIdAttachment, texture(TextureType::ObjectId), cubeMapCoord, 0);
+    }
   }
 }
 
 void CubeMap::prepareToDraw(unsigned int cubeSideIndex,
-                            RenderCamera::Flags flags) {
+                            RenderCamera::Flags renderCameraFlags) {
   // Note: we ONLY need to map shader output to color attachment when necessary,
   // which means in depth texture mode, we do NOT need to do this
-  if (flags_ & CubeMap::Flag::ColorTexture) {
+  if (flags_ & CubeMap::Flag::ColorTexture ||
+      flags_ & CubeMap::Flag::ObjectIdTexture) {
     mapForDraw(cubeSideIndex);
-    if (flags & RenderCamera::Flag::ClearColor) {
-      frameBuffer_[cubeSideIndex].clearColor(0,  // color attachment
-                                             Mn::Vector4ui{0}  // clear color
-      );
-    }
   }
 
-  if (flags & RenderCamera::Flag::ClearDepth) {
+  if (flags_ & CubeMap::Flag::ColorTexture &&
+      renderCameraFlags & RenderCamera::Flag::ClearColor) {
+    frameBuffer_[cubeSideIndex].clearColor(
+        0,                      // color attachment
+        Mn::Color4{0, 0, 0, 1}  // clear color
+    );
+  }
+
+  if (renderCameraFlags & RenderCamera::Flag::ClearDepth) {
     frameBuffer_[cubeSideIndex].clearDepth(1.0f);
   }
 
+  if (flags_ & CubeMap::Flag::ObjectIdTexture &&
+      renderCameraFlags & RenderCamera::Flag::ClearObjectId) {
+    frameBuffer_[cubeSideIndex].clearColor(1, Mn::Vector4ui{});
+  }
+
   CORRADE_INTERNAL_ASSERT(frameBuffer_[cubeSideIndex].checkStatus(
                               Mn::GL::FramebufferTarget::Draw) ==
                           Mn::GL::Framebuffer::Status::Complete);
 }
 
 void CubeMap::mapForDraw(unsigned int index) {
-  frameBuffer_[index].mapForDraw({
-      {Mn::Shaders::Generic3D::ColorOutput,
-       Mn::GL::Framebuffer::ColorAttachment{0}},
-      // TODO:
-      //{Mn::Shaders::Generic3D::ObjectIdOutput, objectIdAttachment}
-  });
+  frameBuffer_[index].mapForDraw(
+      {{Mn::Shaders::Generic3D::ColorOutput,
+        (flags_ & CubeMap::Flag::ColorTexture
+             ? rgbaAttachment
+             : Mn::GL::Framebuffer::DrawAttachment::None)},
+       {Mn::Shaders::Generic3D::ObjectIdOutput,
+        (flags_ & CubeMap::Flag::ObjectIdTexture
+             ? objectIdAttachment
+             : Mn::GL::Framebuffer::DrawAttachment::None)}});
 }
 
 Mn::GL::CubeMapTexture& CubeMap::getTexture(TextureType type) {
@@ -302,9 +336,17 @@ bool CubeMap::saveTexture(TextureType type,
           return false;
         }
       } break;
-
+      /*
+      case CubeMap::TextureType::ObjectId:
+        // TODO: save object Id texture
+        break;
+        */
       case CubeMap::TextureType::Count:
         break;
+
+      default:
+        CORRADE_INTERNAL_ASSERT_UNREACHABLE();
+        break;
     }
     CORRADE_ASSERT(!filename.empty(),
                    "CubeMap::saveTexture(): Unknown texture type.", false);
@@ -383,6 +425,11 @@ void CubeMap::loadTexture(TextureType type,
                         imageView);
       } break;
 
+      case TextureType::ObjectId:
+        // TODO: object Id texture
+        CORRADE_INTERNAL_ASSERT_UNREACHABLE();
+        break;
+
       case TextureType::Count:
         CORRADE_INTERNAL_ASSERT_UNREACHABLE();
         break;
@@ -397,7 +444,7 @@ void CubeMap::loadTexture(TextureType type,
 
 void CubeMap::renderToTexture(CubeMapCamera& camera,
                               scene::SceneGraph& sceneGraph,
-                              RenderCamera::Flags flags) {
+                              RenderCamera::Flags renderCameraFlags) {
   CORRADE_ASSERT(camera.isInSceneGraph(sceneGraph),
                  "CubeMap::renderToTexture(): camera is NOT attached to the "
                  "current scene graph.", );
@@ -422,16 +469,16 @@ void CubeMap::renderToTexture(CubeMapCamera& camera,
   for (int iFace = 0; iFace < 6; ++iFace) {
     frameBuffer_[iFace].bind();
     camera.switchToFace(iFace);
-    prepareToDraw(iFace, flags);
+    prepareToDraw(iFace, renderCameraFlags);
 
     // TODO:
-    // camera should have flags so that it can do "low quality" rendering,
-    // e.g., no normal maps, no specular lighting, low-poly meshes,
+    // camera should have renderCameraFlags so that it can do "low quality"
+    // rendering, e.g., no normal maps, no specular lighting, low-poly meshes,
     // low-quality textures.
 
     for (auto& it : sceneGraph.getDrawableGroups()) {
       if (it.second.prepareForDraw(camera)) {
-        camera.draw(it.second, flags);
+        camera.draw(it.second, renderCameraFlags);
       }
     }
   }  // iFace

src/esp/gfx/CubeMap.h

@@ -34,7 +34,11 @@ class CubeMap {
      * HDR depth texture
      */
     Depth,
-    // TODO: ObjectId
+    /**
+     * object id (uint) texture
+     */
+    ObjectId,
+
     // TODO: HDR color
 
     Count,
@@ -50,8 +54,9 @@ class CubeMap {
      */
     DepthTexture = 1 << 1,
     /**
-     * TODO: ObjectId
+     * create ObjectId cubemap
      */
+    ObjectIdTexture = 1 << 2,
     /**
      * Build mipmap for cubemap color texture
      * By default, NO mipmap will be built, only 1 level

src/esp/gfx/CubeMapShaderBase.cpp

@@ -41,5 +41,15 @@ CubeMapShaderBase& CubeMapShaderBase::bindDepthTexture(
   return *this;
 }
 
+CubeMapShaderBase& CubeMapShaderBase::bindObjectIdTexture(
+    Mn::GL::CubeMapTexture& texture) {
+  CORRADE_ASSERT(flags_ & CubeMapShaderBase::Flag::ObjectIdTexture,
+                 "CubeMapShaderBase::bindObjectIdTexture(): the shader was not "
+                 "created with object id texture enabled",
+                 *this);
+  texture.bind(CubeMapShaderBaseTexUnitSpace::TextureUnit::ObjectId);
+  return *this;
+}
+
 }  // namespace gfx
 }  // namespace esp