[Sensor] Fisheye Sensor

src/esp/gfx/CMakeLists.txt

@@ -1,5 +1,7 @@
 set(
   gfx_SOURCES
+  CubeMap.cpp
+  CubeMap.h
   DepthUnprojection.cpp
   DepthUnprojection.h
   Drawable.cpp
@@ -41,6 +43,10 @@ set(
   PbrShader.h
   PbrDrawable.cpp
   PbrDrawable.h
+  DoubleSphereCameraShader.cpp
+  DoubleSphereCameraShader.h
+  FisheyeShader.cpp
+  FisheyeShader.h
 )
 
 # If ptex support is enabled add relevant source files

src/esp/gfx/CubeMap.cpp

@@ -155,6 +155,7 @@ void CubeMap::recreateTexture() {
   // color texture
   if (flags_ & Flag::ColorTexture) {
     auto& colorTexture = texture(TextureType::Color);
+    colorTexture = Mn::GL::CubeMapTexture{};
     colorTexture.setWrapping(Mn::GL::SamplerWrapping::ClampToEdge)
         .setMinificationFilter(Mn::GL::SamplerFilter::Linear,
                                Mn::GL::SamplerMipmap::Linear)
@@ -172,6 +173,7 @@ void CubeMap::recreateTexture() {
   // depth texture
   if (flags_ & Flag::DepthTexture) {
     auto& depthTexture = texture(TextureType::Depth);
+    depthTexture = Mn::GL::CubeMapTexture{};
     depthTexture.setWrapping(Mn::GL::SamplerWrapping::ClampToEdge)
         .setMinificationFilter(Mn::GL::SamplerFilter::Nearest)
         .setMagnificationFilter(Mn::GL::SamplerFilter::Nearest)

src/esp/gfx/CubeMap.h

@@ -37,7 +37,6 @@ class CubeMap {
     // TODO: ObjectId
     // TODO: HDR color
 
-    // you will have to update it when new type is added
     Count,
   };
 

src/esp/gfx/DoubleSphereCameraShader.cpp

@@ -0,0 +1,120 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+#include "DoubleSphereCameraShader.h"
+
+#include <Corrade/Containers/Reference.h>
+#include <Corrade/Utility/Assert.h>
+#include <Corrade/Utility/FormatStl.h>
+#include <Corrade/Utility/Resource.h>
+#include <Magnum/GL/Shader.h>
+#include <Magnum/GL/Texture.h>
+#include <Magnum/GL/Version.h>
+
+#include <sstream>
+
+// This is to import the "resources" at runtime. When the resource is
+// compiled into static library, it must be explicitly initialized via this
+// macro, and should be called *outside* of any namespace.
+static void importShaderResources() {
+  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+}
+
+namespace Mn = Magnum;
+namespace Cr = Corrade;
+
+namespace esp {
+namespace gfx {
+DoubleSphereCameraShader::DoubleSphereCameraShader(FisheyeShader::Flags flags)
+    : FisheyeShader(flags) {
+  if (!Cr::Utility::Resource::hasGroup("default-shaders")) {
+    importShaderResources();
+  }
+
+#ifdef MAGNUM_TARGET_WEBGL
+  Mn::GL::Version glVersion = Mn::GL::Version::GLES300;
+#else
+  Mn::GL::Version glVersion = Mn::GL::Version::GL410;
+#endif
+
+  // this is not the file name, but the group name in the config file
+  // see Shaders.conf in the shaders folder
+  const Cr::Utility::Resource rs{"default-shaders"};
+
+  Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
+  Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};
+
+  // Add macros
+  vert.addSource(rs.get("bigTriangle.vert"));
+
+  std::stringstream outputAttributeLocationsStream;
+
+  if (flags_ & FisheyeShader::Flag::ColorTexture) {
+    outputAttributeLocationsStream << Cr::Utility::formatString(
+        "#define OUTPUT_ATTRIBUTE_LOCATION_COLOR {}\n", ColorOutput);
+  }
+  /* TODO:
+  outputAttributeLocationsStream << Cr::Utility::formatString(
+      "#define OUTPUT_ATTRIBUTE_LOCATION_OBJECT_ID {}\n", ObjectIdOutput);
+  */
+
+  frag.addSource(outputAttributeLocationsStream.str())
+      .addSource(flags_ & FisheyeShader::Flag::ColorTexture
+                     ? "#define COLOR_TEXTURE\n"
+                     : "")
+      .addSource(flags_ & FisheyeShader::Flag::DepthTexture
+                     ? "#define DEPTH_TEXTURE\n"
+                     : "")
+      .addSource(rs.get("doubleSphereCamera.frag"));
+
+  CORRADE_INTERNAL_ASSERT_OUTPUT(Mn::GL::Shader::compile({vert, frag}));
+
+  attachShaders({vert, frag});
+
+  CORRADE_INTERNAL_ASSERT_OUTPUT(link());
+
+  // set texture binding points in the shader
+  if (flags_ & FisheyeShader::Flag::ColorTexture) {
+    setUniform(uniformLocation("ColorTexture"),
+               fisheyeShaderTexUnitSpace::TextureUnit::Color);
+  }
+  if (flags_ & FisheyeShader::Flag::DepthTexture) {
+    setUniform(uniformLocation("DepthTexture"),
+               fisheyeShaderTexUnitSpace::TextureUnit::Depth);
+  }
+  // TODO: handle the other flags, ObjectIdTexture
+
+  // cache the uniform locations
+  // it hurts the performance to call glGetUniformLocation() every frame due
+  // to string operations. therefore, cache the locations in the constructor
+
+  focalLengthUniform_ = uniformLocation("FocalLength");
+  principalPointOffsetUniform_ = uniformLocation("PrincipalPointOffset");
+  alphaUniform_ = uniformLocation("Alpha");
+  xiUniform_ = uniformLocation("Xi");
+}
+
+DoubleSphereCameraShader& DoubleSphereCameraShader::setFocalLength(
+    Magnum::Vector2 focalLength) {
+  setUniform(focalLengthUniform_, focalLength);
+  return *this;
+}
+
+DoubleSphereCameraShader& DoubleSphereCameraShader::setPrincipalPointOffset(
+    Magnum::Vector2 offset) {
+  setUniform(principalPointOffsetUniform_, offset);
+  return *this;
+}
+
+DoubleSphereCameraShader& DoubleSphereCameraShader::setAlpha(float alpha) {
+  setUniform(alphaUniform_, alpha);
+  return *this;
+}
+
+DoubleSphereCameraShader& DoubleSphereCameraShader::setXi(float xi) {
+  setUniform(xiUniform_, xi);
+  return *this;
+}
+
+}  // namespace gfx
+}  // namespace esp

src/esp/gfx/DoubleSphereCameraShader.h

@@ -0,0 +1,85 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#ifndef ESP_GFX_DOUBLESPHERECAMERASHADER_H_
+#define ESP_GFX_DOUBLESPHERECAMERASHADER_H_
+
+#include <Corrade/Containers/EnumSet.h>
+#include <Magnum/Shaders/Generic.h>
+
+#include "FisheyeShader.h"
+#include "esp/core/esp.h"
+
+namespace esp {
+namespace gfx {
+class DoubleSphereCameraShader : public FisheyeShader {
+ public:
+  enum : Magnum::UnsignedInt {
+    /**
+     * Color shader output. @ref shaders-generic "Generic output",
+     * present always. Expects three- or four-component floating-point
+     * or normalized buffer attachment.
+     */
+    ColorOutput = Magnum::Shaders::Generic3D::ColorOutput,
+
+    // TODO
+    /**
+     * Object ID shader output. @ref shaders-generic "Generic output",
+     * present only if @ref FisheyeShader::Flag::ObjectId is set. Expects a
+     * single-component unsigned integral attachment. Writes the value
+     * set in @ref setObjectId() there.
+     */
+    // ObjectIdOutput = Magnum::Shaders::Generic3D::ObjectIdOutput,
+  };
+
+  explicit DoubleSphereCameraShader(FisheyeShader::Flags flags = {
+                                        FisheyeShader::Flag::ColorTexture});
+
+  /**
+   *  @brief Set the focal length of the fisheye camera
+   *  @param focalLength, the focal length x, y directions
+   *  @return Reference to self (for method chaining)
+   */
+  DoubleSphereCameraShader& setFocalLength(Magnum::Vector2 focalLength);
+  /**
+   *  @brief Set the offset of the principal point of the fisheye camera
+   *  @param offset, the offset of the principal point in pixels
+   *  @return Reference to self (for method chaining)
+   */
+  DoubleSphereCameraShader& setPrincipalPointOffset(Magnum::Vector2 offset);
+  /**
+   * @brief Set the alpha value in the "Double Sphere Camera" model.
+   * See details in:
+   * Vladyslav Usenko, Nikolaus Demmel and Daniel Cremers: The Double Sphere
+   * Camera Model, The International Conference on 3D Vision (3DV), 2018
+   *  @param alpha, the alpha value (0.0 <= alpha < 1.0)
+   *  @return Reference to self (for method chaining)
+   */
+  DoubleSphereCameraShader& setAlpha(float alpha);
+
+  /**
+   * @brief Set the Xi value in the "Double Sphere Camera" model.
+   * See details in:
+   * Vladyslav Usenko, Nikolaus Demmel and Daniel Cremers: The Double Sphere
+   * Camera Model, The International Conference on 3D Vision (3DV), 2018
+   * @param xi, the Xi value
+   * @return Reference to self (for method chaining)
+   */
+  DoubleSphereCameraShader& setXi(float xi);
+
+  /**
+   * @brief deconstructor
+   */
+  ~DoubleSphereCameraShader() override = default;
+
+ protected:
+  int focalLengthUniform_ = ID_UNDEFINED;
+  int principalPointOffsetUniform_ = ID_UNDEFINED;
+  int alphaUniform_ = ID_UNDEFINED;
+  int xiUniform_ = ID_UNDEFINED;
+};
+
+}  // namespace gfx
+}  // namespace esp
+#endif

src/esp/gfx/FisheyeShader.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+#include "FisheyeShader.h"
+
+#include <Corrade/Utility/Assert.h>
+#include <Corrade/Utility/FormatStl.h>
+#include <Corrade/Utility/Resource.h>
+#include <Magnum/GL/Shader.h>
+#include <Magnum/GL/Texture.h>
+#include <Magnum/GL/Version.h>
+
+namespace Mn = Magnum;
+namespace Cr = Corrade;
+
+namespace esp {
+namespace gfx {
+FisheyeShader::FisheyeShader(Flags flags) : flags_(flags) {
+  CORRADE_ASSERT(flags != Flags{},
+                 "FisheyeShader::FisheyeShader(): shader "
+                 "flags cannot be empty.", );
+}
+FisheyeShader& FisheyeShader::bindColorTexture(
+    Mn::GL::CubeMapTexture& texture) {
+  CORRADE_ASSERT(flags_ & FisheyeShader::Flag::ColorTexture,
+                 "FisheyeShader::bindColorTexture(): the shader was not "
+                 "created with color texture enabled",
+                 *this);
+  texture.bind(fisheyeShaderTexUnitSpace::TextureUnit::Color);
+  return *this;
+}
+
+FisheyeShader& FisheyeShader::bindDepthTexture(
+    Mn::GL::CubeMapTexture& texture) {
+  CORRADE_ASSERT(flags_ & FisheyeShader::Flag::DepthTexture,
+                 "FisheyeShader::bindDepthTexture(): the shader was not "
+                 "created with depth texture enabled",
+                 *this);
+  texture.bind(fisheyeShaderTexUnitSpace::TextureUnit::Depth);
+  return *this;
+}
+
+}  // namespace gfx
+}  // namespace esp

src/esp/gfx/FisheyeShader.h

@@ -0,0 +1,97 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#ifndef ESP_GFX_FISHEYESHADER_H_
+#define ESP_GFX_FISHEYESHADER_H_
+
+#include <Corrade/Containers/EnumSet.h>
+#include <Corrade/Utility/Macros.h>
+#include <Magnum/GL/AbstractShaderProgram.h>
+#include <Magnum/GL/CubeMapTexture.h>
+#include <Magnum/Shaders/Generic.h>
+
+#include "esp/core/esp.h"
+
+namespace esp {
+namespace gfx {
+namespace fisheyeShaderTexUnitSpace {
+enum TextureUnit : uint8_t {
+  Color = 0,
+  Depth = 1,
+  // TODO
+  // ObjectId = 2,
+};
+}
+// Interface class for various fisheye camera shaders, such as "Double Sphere
+// Camera", "Field-of-View Camera", etc.
+class FisheyeShader : public Magnum::GL::AbstractShaderProgram {
+ public:
+  enum : Magnum::UnsignedInt {
+    /**
+     * Color shader output. @ref shaders-generic "Generic output",
+     * present always. Expects three- or four-component floating-point
+     * or normalized buffer attachment.
+     */
+    ColorOutput = Magnum::Shaders::Generic3D::ColorOutput,
+
+    // TODO
+    /**
+     * Object ID shader output. @ref shaders-generic "Generic output",
+     * present only if @ref Flag::ObjectId is set. Expects a
+     * single-component unsigned integral attachment. Writes the value
+     * set in @ref setObjectId() there.
+     */
+    // ObjectIdOutput = Magnum::Shaders::Generic3D::ObjectIdOutput,
+  };
+
+  /**
+   * @brief Flag
+   *
+   * @see @ref Flags, @ref flags()
+   */
+  enum class Flag : Magnum::UnsignedShort {
+    /**
+     * cubemap color texture
+     */
+    ColorTexture = 1 << 0,
+    /**
+     * cubemap depth texture
+     */
+    DepthTexture = 1 << 1,
+    // ObjectIdTexture = 1 << 2,
+  };
+
+  typedef Corrade::Containers::EnumSet<Flag> Flags;
+
+  /**
+   * @brief constructor
+   * @param[in] flags fisheye shader flags
+   */
+  explicit FisheyeShader(Flags flags = {Flag::ColorTexture});
+
+  ~FisheyeShader() override = default;
+
+  /** @brief Flags */
+  Flags flags() const { return flags_; }
+
+  /**
+   * @brief bind cubemap color texture
+   * @param[in] texture cubemap color texture
+   */
+  virtual FisheyeShader& bindColorTexture(Magnum::GL::CubeMapTexture& texture);
+  /**
+   * @brief bind cubemap depth texture
+   * @param[in] texture cubemap depth texture
+   */
+  virtual FisheyeShader& bindDepthTexture(Magnum::GL::CubeMapTexture& texture);
+  // virtual FisheyeShader& bindObjectIdTexture(Magnum::GL::Texture2D&
+  // texture);
+
+ protected:
+  Flags flags_;
+};
+CORRADE_ENUMSET_OPERATORS(FisheyeShader::Flags)
+}  // namespace gfx
+}  // namespace esp
+#endif