[Sensor] Visualize the semantic observations

src/esp/gfx/RenderTarget.cpp

@@ -238,8 +238,21 @@ struct RenderTarget::Impl {
     return framebuffer_.viewport().size();
   }
 
-  Magnum::GL::Texture2D& getDepthTexture() { return depthRenderTexture_; }
-  Magnum::GL::Texture2D& getObjectIdTexture() { return objectIdTexture_; }
+  Magnum::GL::Texture2D& getDepthTexture() {
+    CORRADE_ASSERT(flags_ & Flag::DepthTextureAttachment,
+                   "RenderTarget::Impl::getDepthTexture(): this render target "
+                   "was not created with depth texture enabled.",
+                   depthRenderTexture_);
+    return depthRenderTexture_;
+  }
+  Magnum::GL::Texture2D& getObjectIdTexture() {
+    CORRADE_ASSERT(
+        flags_ & Flag::ObjectIdAttachment,
+        "RenderTarget::Impl::getObjectIdTexture(): this render target "
+        "was not created with object id texture enabled.",
+        objectIdTexture_);
+    return objectIdTexture_;
+  }
 
 #ifdef ESP_BUILD_WITH_CUDA
   void readFrameRgbaGPU(uint8_t* devPtr) {

src/esp/gfx/Renderer.cpp

@@ -81,12 +81,18 @@ struct Renderer::Impl {
         mesh_ = Mn::GL::Mesh{};
         mesh_->setCount(3);
       }
-      if (type == sensor::SensorType::Depth) {
-        Magnum::Resource<Mn::GL::AbstractShaderProgram, TextureVisualizerShader>
-            shader = getShader<TextureVisualizerShader>(
-                esp::gfx::Renderer::Impl::RendererShaderType::
-                    DepthTextureVisualizer);
+      esp::gfx::Renderer::Impl::RendererShaderType rendererShaderType =
+          esp::gfx::Renderer::Impl::RendererShaderType::DepthTextureVisualizer;
+
+      if (type == sensor::SensorType::Semantic) {
+        rendererShaderType =
+            gfx::Renderer::Impl::RendererShaderType::ObjectIdTextureVisualizer;
+      }
 
+      Magnum::Resource<Mn::GL::AbstractShaderProgram, TextureVisualizerShader>
+          shader = getShader<TextureVisualizerShader>(rendererShaderType);
+
+      if (type == sensor::SensorType::Depth) {
 #ifdef ENABLE_VISUALIZATION_WORKAROUND_ON_MAC
         // create a BufferImage instance, if not already
         if (!depthBufferImage_) {
@@ -96,9 +102,9 @@ struct Renderer::Impl {
         tgt.getDepthTexture().image(0, *depthBufferImage_,
                                     Mn::GL::BufferUsage::StaticRead);
 
-        // This takes the above output image (which is depth) and "reinterprets"
-        // it as R32F. In other words, the image below serves as an "image
-        // view".
+        // This takes the above output image (which is depth) and
+        // "reinterprets" it as R32F. In other words, the image below serves
+        // as an "image view".
         Mn::GL::BufferImage2D clonedDepthImage{
             depthBufferImage_->storage(), Mn::PixelFormat::R32F,
             depthBufferImage_->size(),
@@ -123,11 +129,13 @@ struct Renderer::Impl {
         shader->bindDepthTexture(tgt.getDepthTexture());
 #endif
         shader->setDepthUnprojection(*visualSensor.depthUnprojection());
-        shader->setColorMapTransformation(colorMapOffset, colorMapScale);
-        tgt.renderReEnter();
-        shader->draw(*mesh_);
-        tgt.renderExit();
+      } else if (type == sensor::SensorType::Semantic) {
+        shader->bindObjectIdTexture(tgt.getObjectIdTexture());
       }
+      shader->setColorMapTransformation(colorMapOffset, colorMapScale);
+      tgt.renderReEnter();
+      shader->draw(*mesh_);
+      tgt.renderExit();
 
       // TODO object id
       Mn::GL::Renderer::enable(Mn::GL::Renderer::Feature::DepthTest);
@@ -173,6 +181,7 @@ struct Renderer::Impl {
       default:
         // I need this default, since sensor type list is long, and without
         // default clang-tidy will complain
+        CORRADE_INTERNAL_ASSERT_UNREACHABLE();
         break;
     }
 
@@ -195,7 +204,7 @@ struct Renderer::Impl {
   enum class RendererShaderType : uint8_t {
     DepthShader = 0,
     DepthTextureVisualizer = 1,
-    // ObjectIdTextureVisualizer = 2,
+    ObjectIdTextureVisualizer = 2,
   };
   template <typename T>
   Mn::Resource<Mn::GL::AbstractShaderProgram, T> getShader(
@@ -210,7 +219,10 @@ struct Renderer::Impl {
         key = Mn::ResourceKey{"depthVisualizer"};
         break;
 
-      // TODO: object id
+      case RendererShaderType::ObjectIdTextureVisualizer:
+        key = Mn::ResourceKey{"objectIdVisualizer"};
+        break;
+
       default:
         CORRADE_INTERNAL_ASSERT_UNREACHABLE();
         break;
@@ -230,6 +242,12 @@ struct Renderer::Impl {
             new TextureVisualizerShader{
                 {TextureVisualizerShader::Flag::DepthTexture}},
             Mn::ResourceDataState::Final, Mn::ResourcePolicy::ReferenceCounted);
+      } else if (type == RendererShaderType::ObjectIdTextureVisualizer) {
+        shaderManager_.set<Mn::GL::AbstractShaderProgram>(
+            shader.key(),
+            new TextureVisualizerShader{
+                {TextureVisualizerShader::Flag::ObjectIdTexture}},
+            Mn::ResourceDataState::Final, Mn::ResourcePolicy::ReferenceCounted);
       }
     }
     CORRADE_INTERNAL_ASSERT(shader);

src/esp/gfx/TextureVisualizerShader.cpp

@@ -36,7 +36,9 @@ TextureVisualizerShader::TextureVisualizerShader(Flags flags) : flags_(flags) {
   if (flags_ & Flag::DepthTexture) {
     ++countMutuallyExclusive;
   }
-  // if(flags & Flag::ObjectIdTexture) ++countMutuallyExclusive;
+  if (flags & Flag::ObjectIdTexture) {
+    ++countMutuallyExclusive;
+  }
   CORRADE_ASSERT(countMutuallyExclusive <= 1,
                  "TextureVisualizerShader::TextureVisualizerShader: "
                  "Flag::DepthTexture and "
@@ -62,9 +64,9 @@ TextureVisualizerShader::TextureVisualizerShader(Flags flags) : flags_(flags) {
   frag.addSource("#define EXPLICIT_ATTRIB_LOCATION\n")
       .addSource(Cr::Utility::formatString(
           "#define OUTPUT_ATTRIBUTE_LOCATION_COLOR {}\n", ColorOutput))
-      .addSource(flags_ == Flag::DepthTexture ? "#define DEPTH_TEXTURE\n" : "")
-      // .addSource(flags_ == Flag::ObjectIdTexture? "#define
-      // OBJECT_ID_TEXTURE\n" : "")
+      .addSource(flags_ & Flag::DepthTexture ? "#define DEPTH_TEXTURE\n" : "")
+      .addSource(flags_ & Flag::ObjectIdTexture ? "#define OBJECT_ID_TEXTURE\n"
+                                                : "")
       .addSource(rs.get("textureVisualizer.frag"));
 
   CORRADE_INTERNAL_ASSERT_OUTPUT(Mn::GL::Shader::compile({vert, frag}));
@@ -90,18 +92,23 @@ TextureVisualizerShader::TextureVisualizerShader(Flags flags) : flags_(flags) {
   const Mn::Vector2i size{int(map.size()), 1};
   colorMapTexture_.setMinificationFilter(Mn::GL::SamplerFilter::Linear)
       .setMagnificationFilter(Mn::GL::SamplerFilter::Linear)
-      .setWrapping(Mn::GL::SamplerWrapping::Repeat)
       .setStorage(1, Mn::GL::TextureFormat::SRGB8Alpha8, size)
       .setSubImage(0, {},
                    Mn::ImageView2D{Mn::PixelFormat::RGB8Srgb, size, map});
+  if (flags_ & Flag::DepthTexture) {
+    colorMapTexture_.setWrapping(Mn::GL::SamplerWrapping::ClampToEdge);
+  } else if (flags_ & Flag::ObjectIdTexture) {
+    colorMapTexture_.setWrapping(Mn::GL::SamplerWrapping::Repeat);
+  }
   colorMapTexture_.bind(ColorMapTextureUnit);
 
   // set default offset, scale based on flags
   if (flags_ & Flag::DepthTexture) {
     setColorMapTransformation(1.0f / 512.0f, 1.0f / 1000.0f);
+  } else if (flags & Flag::ObjectIdTexture) {
+    setColorMapTransformation(1.0f / 512.0f,
+                              1.0f / 108.0f);  // initial guess: 108 objects
   }
-  // TODO:
-  // else set ObjectIdTexture
 }
 
 TextureVisualizerShader& TextureVisualizerShader::setColorMapTransformation(
@@ -129,6 +136,17 @@ TextureVisualizerShader& TextureVisualizerShader::bindDepthTexture(
   return *this;
 }
 
+TextureVisualizerShader& TextureVisualizerShader::bindObjectIdTexture(
+    Mn::GL::Texture2D& texture) {
+  CORRADE_ASSERT(
+      flags_ & Flag::ObjectIdTexture,
+      "TextureVisualizerShader::bindObjectIdTexture(): the shader was "
+      "not created with object Id texture enabled",
+      *this);
+  texture.bind(SourceTextureUnit);
+  return *this;
+}
+
 TextureVisualizerShader& TextureVisualizerShader::setDepthUnprojection(
     const Mn::Vector2& depthUnprojection) {
   CORRADE_ASSERT(flags_ & Flag::DepthTexture,

src/esp/gfx/TextureVisualizerShader.h

@@ -33,10 +33,13 @@ class TextureVisualizerShader : public Magnum::GL::AbstractShaderProgram {
    */
   enum class Flag : Magnum::UnsignedShort {
     /**
-     * @brief visualize depth texture
+     * visualize depth texture
      */
     DepthTexture = 1 << 0,
-    // ObjectIdTexture = 1 << 1,
+    /**
+     * visualize object-id texture (semantic info)
+     */
+    ObjectIdTexture = 1 << 1,
   };
 
   typedef Corrade::Containers::EnumSet<Flag> Flags;
@@ -76,6 +79,13 @@ class TextureVisualizerShader : public Magnum::GL::AbstractShaderProgram {
    */
   TextureVisualizerShader& bindDepthTexture(Magnum::GL::Texture2D& texture);
 
+  /**
+   * @brief Bind object Id texture. It can only be called when
+   * Flag::ObjectIdTexture is set.
+   * @return Reference to self (for method chaining)
+   */
+  TextureVisualizerShader& bindObjectIdTexture(Magnum::GL::Texture2D& texture);
+
   /**
    * @brief Set the depth unprojection parameters directly. It can only be
    * called when Flag::DepthTexture is set.

src/shaders/textureVisualizer.frag

@@ -7,7 +7,13 @@ precision highp float;
 in highp vec2 textureCoordinates;
 
 // ------------ uniforms --------------------
+#ifdef DEPTH_TEXTURE
 uniform highp sampler2D sourceTexture;
+#endif
+
+#ifdef OBJECT_ID_TEXTURE
+uniform highp usampler2D sourceTexture;
+#endif
 
 uniform highp sampler2D colorMapTexture;
 uniform highp vec2 colorMapOffsetScale;
@@ -31,7 +37,7 @@ void main() {
   highp float originalDepth =
       depthUnprojection[1] / (depth + depthUnprojection[0]);
 #elif defined(OBJECT_ID_TEXTURE)
-#error sorry, visualizing object_id texture is under construction.
+  highp uint objectId = texture(sourceTexture, textureCoordinates).r;
 #endif
 
   fragmentColor = texture(colorMapTexture, vec2(
@@ -42,7 +48,7 @@ void main() {
         #ifdef DEPTH_TEXTURE
         originalDepth
         #elif defined(OBJECT_ID_TEXTURE)
-        #error sorry, visualizing object_id texture is under construction.
+        objectId
         #else
         #error sorry, the type of the texture to be visualized is unknown.
         #endif

src/utils/viewer/viewer.cpp

@@ -377,9 +377,14 @@ Key Commands:
   // NOTE: Mouse + shift is to select object on the screen!!
   void createPickedObjectVisualizer(unsigned int objectId);
   std::unique_ptr<ObjectPickingHelper> objectPickingHelper_;
-  // returns the number of visible drawables (meshVisualizer drawables are not
-  // included)
-  bool depthMode_ = false;
+
+  enum class VisualizeMode : uint8_t {
+    RGBA = 0,
+    Depth,
+    Semantic,
+    VisualizeModeCount,
+  };
+  VisualizeMode visualizeMode_ = VisualizeMode::RGBA;
 
   Mn::DebugTools::GLFrameProfiler profiler_{};
 
@@ -487,6 +492,18 @@ void addSensors(esp::agent::AgentConfiguration& agentConfig,
     spec->sensorSubType = esp::sensor::SensorSubType::Equirectangular;
     spec->resolution = esp::vec2i(viewportSize[1], viewportSize[0]);
   }
+
+  // add a rgb semantic sensor
+  agentConfig.sensorSpecifications.emplace_back(
+      esp::sensor::CameraSensorSpec::create());
+  {
+    auto spec = static_cast<esp::sensor::CameraSensorSpec*>(
+        agentConfig.sensorSpecifications.back().get());
+    spec->uuid = "semantic";
+    spec->sensorType = esp::sensor::SensorType::Semantic;
+    spec->sensorSubType = esp::sensor::SensorSubType::Pinhole;
+    spec->resolution = esp::vec2i(viewportSize[1], viewportSize[0]);
+  }
 }
 
 Viewer::Viewer(const Arguments& arguments)
@@ -1130,21 +1147,33 @@ void Viewer::drawEvent() {
 
   uint32_t visibles = renderCamera_->getPreviousNumVisibleDrawables();
 
-  if (depthMode_) {
+  if (visualizeMode_ == VisualizeMode::Depth ||
+      visualizeMode_ == VisualizeMode::Semantic) {
     // ================ Depth Visualization ==================================
     std::string sensorId = "depth";
-    if (sensorMode_ == VisualSensorMode::Fisheye) {
-      sensorId = "depth_fisheye";
-    } else if (sensorMode_ == VisualSensorMode::Equirectangular) {
-      sensorId = "depth_equirectangular";
+    if (visualizeMode_ == VisualizeMode::Depth) {
+      if (sensorMode_ == VisualSensorMode::Fisheye) {
+        sensorId = "depth_fisheye";
+      } else if (sensorMode_ == VisualSensorMode::Equirectangular) {
+        sensorId = "depth_equirectangular";
+      }
+    } else if (visualizeMode_ == VisualizeMode::Semantic) {
+      sensorId = "semantic";
+      // TODO: add semantic fisheye and equiRec
     }
 
     simulator_->drawObservation(defaultAgentId_, sensorId);
     esp::gfx::RenderTarget* sensorRenderTarget =
         simulator_->getRenderTarget(defaultAgentId_, sensorId);
-    simulator_->visualizeObservation(defaultAgentId_, sensorId,
-                                     1.0f / 512.0f,  // colorMapOffset
-                                     1.0f / 24.0f);  // colorMapScale
+    if (visualizeMode_ == VisualizeMode::Depth) {
+      simulator_->visualizeObservation(defaultAgentId_, sensorId,
+                                       1.0f / 512.0f,  // colorMapOffset
+                                       1.0f / 12.0f);  // colorMapScale
+    } else if (visualizeMode_ == VisualizeMode::Semantic) {
+      simulator_->visualizeObservation(defaultAgentId_, sensorId,
+                                       1.0f / 512.0f,  // colorMapOffset
+                                       1.0f / 50.0f);  // colorMapScale
+    }
     sensorRenderTarget->blitRgbaToDefault();
   } else {
     if (sensorMode_ == VisualSensorMode::Camera) {
@@ -1337,14 +1366,15 @@ void Viewer::bindRenderTarget() {
     if (it.second.get().isVisualSensor()) {
       esp::sensor::VisualSensor& visualSensor =
           static_cast<esp::sensor::VisualSensor&>(it.second.get());
-      if (depthMode_) {
+      if (visualizeMode_ == VisualizeMode::Depth ||
+          visualizeMode_ == VisualizeMode::Semantic) {
         simulator_->getRenderer()->bindRenderTarget(
             visualSensor, {esp::gfx::Renderer::Flag::VisualizeTexture});
       } else {
         simulator_->getRenderer()->bindRenderTarget(visualSensor);
       }
-    }
-  }
+    }  // if
+  }    // for
 }
 
 void Viewer::viewportEvent(ViewportEvent& event) {
@@ -1554,9 +1584,24 @@ void Viewer::keyPressEvent(KeyEvent& event) {
       getAgentCamera().resetZoom();
       break;
     case KeyEvent::Key::Seven:
-      depthMode_ = !depthMode_;
+      visualizeMode_ = static_cast<VisualizeMode>(
+          (uint8_t(visualizeMode_) + 1) %
+          uint8_t(VisualizeMode::VisualizeModeCount));
       bindRenderTarget();
-      LOG(INFO) << "Depth sensor is " << (depthMode_ ? "ON" : "OFF");
+      switch (visualizeMode_) {
+        case VisualizeMode::RGBA:
+          LOG(INFO) << "Visualizing COLOR sensor observation.";
+          break;
+        case VisualizeMode::Depth:
+          LOG(INFO) << "Visualizing DEPTH sensor observation.";
+          break;
+        case VisualizeMode::Semantic:
+          LOG(INFO) << "Visualizing SEMANTIC sensor observation.";
+          break;
+        default:
+          CORRADE_INTERNAL_ASSERT_UNREACHABLE();
+          break;
+      }
       break;
     case KeyEvent::Key::Eight:
       addPrimitiveObject();