Merge pull request facebookresearch#598 from vauduong/modify-tests-sensor-spec

[Sensor] Fix hab-lab examples to work with Hierarchical SensorSpec changes

Author: jturner65

docs/pages/habitat-sim-demo.rst

@@ -46,34 +46,31 @@ Habitat Sim Demo
         sim_cfg.scene_id = settings["scene"]
 
         # Note: all sensors must have the same resolution
-        sensors = {
-            "color_sensor": {
-                "sensor_type": habitat_sim.SensorType.COLOR,
-                "resolution": [settings["height"], settings["width"]],
-                "position": [0.0, settings["sensor_height"], 0.0],
-            },
-            "depth_sensor": {
-                "sensor_type": habitat_sim.SensorType.DEPTH,
-                "resolution": [settings["height"], settings["width"]],
-                "position": [0.0, settings["sensor_height"], 0.0],
-            },
-            "semantic_sensor": {
-                "sensor_type": habitat_sim.SensorType.SEMANTIC,
-                "resolution": [settings["height"], settings["width"]],
-                "position": [0.0, settings["sensor_height"], 0.0],
-            },
-        }
-
         sensor_specs = []
-        for sensor_uuid, sensor_params in sensors.items():
-            if settings[sensor_uuid]:
-                sensor_spec = habitat_sim.SensorSpec()
-                sensor_spec.uuid = sensor_uuid
-                sensor_spec.sensor_type = sensor_params["sensor_type"]
-                sensor_spec.resolution = sensor_params["resolution"]
-                sensor_spec.position = sensor_params["position"]
-
-                sensor_specs.append(sensor_spec)
+
+        color_sensor_spec = habitat_sim.CameraSensorSpec()
+        color_sensor_spec.uuid = "color_sensor"
+        color_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
+        color_sensor_spec.resolution = [settings["height"], settings["width"]]
+        color_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
+        color_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
+        sensor_specs.append(color_sensor_spec)
+
+        depth_sensor_spec = habitat_sim.CameraSensorSpec()
+        depth_sensor_spec.uuid = "depth_sensor"
+        depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
+        depth_sensor_spec.resolution = [settings["height"], settings["width"]]
+        depth_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
+        depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
+        sensor_specs.append(depth_sensor_spec)
+
+        semantic_sensor_spec = habitat_sim.CameraSensorSpec()
+        semantic_sensor_spec.uuid = "semantic_sensor"
+        semantic_sensor_spec.sensor_type = habitat_sim.SensorType.SEMANTIC
+        semantic_sensor_spec.resolution = [settings["height"], settings["width"]]
+        semantic_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
+        semantic_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
+        sensor_specs.append(semantic_sensor_spec)
 
         # Here you can specify the amount of displacement in a forward action and the turn angle
         agent_cfg = habitat_sim.agent.AgentConfiguration()

examples/tutorials/colabs/Habitat_Interactive_Tasks.ipynb

@@ -261,29 +261,23 @@
     "    sim_cfg.physics_config_file = settings[\"physics_config_file\"]\n",
     "\n",
     "    # Note: all sensors must have the same resolution\n",
-    "    sensors = {\n",
-    "        \"rgb\": {\n",
-    "            \"sensor_type\": habitat_sim.SensorType.COLOR,\n",
-    "            \"resolution\": [settings[\"height\"], settings[\"width\"]],\n",
-    "            \"position\": [0.0, settings[\"sensor_height\"], 0.0],\n",
-    "        },\n",
-    "        \"depth\": {\n",
-    "            \"sensor_type\": habitat_sim.SensorType.DEPTH,\n",
-    "            \"resolution\": [settings[\"height\"], settings[\"width\"]],\n",
-    "            \"position\": [0.0, settings[\"sensor_height\"], 0.0],\n",
-    "        },\n",
-    "    }\n",
-    "\n",
     "    sensor_specs = []\n",
-    "    for sensor_uuid, sensor_params in sensors.items():\n",
-    "        if settings[sensor_uuid]:\n",
-    "            sensor_spec = habitat_sim.SensorSpec()\n",
-    "            sensor_spec.uuid = sensor_uuid\n",
-    "            sensor_spec.sensor_type = sensor_params[\"sensor_type\"]\n",
-    "            sensor_spec.resolution = sensor_params[\"resolution\"]\n",
-    "            sensor_spec.position = sensor_params[\"position\"]\n",
-    "\n",
-    "            sensor_specs.append(sensor_spec)\n",
+    "\n",
+    "    rgb_sensor_spec = habitat_sim.CameraSensorSpec()\n",
+    "    rgb_sensor_spec.uuid = \"rgb\"\n",
+    "    rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR\n",
+    "    rgb_sensor_spec.resolution = [settings[\"height\"], settings[\"width\"]]\n",
+    "    rgb_sensor_spec.position = [0.0, settings[\"sensor_height\"], 0.0]\n",
+    "    rgb_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE\n",
+    "    sensor_specs.append(rgb_sensor_spec)\n",
+    "\n",
+    "    depth_sensor_spec = habitat_sim.CameraSensorSpec()\n",
+    "    depth_sensor_spec.uuid = \"depth\"\n",
+    "    depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH\n",
+    "    depth_sensor_spec.resolution = [settings[\"height\"], settings[\"width\"]]\n",
+    "    depth_sensor_spec.position = [0.0, settings[\"sensor_height\"], 0.0]\n",
+    "    depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE\n",
+    "    sensor_specs.append(depth_sensor_spec)\n",
     "\n",
     "    # Here you can specify the amount of displacement in a forward action and the turn angle\n",
     "    agent_cfg = habitat_sim.agent.AgentConfiguration()\n",

examples/tutorials/nb_python/Habitat_Interactive_Tasks.py

@@ -258,29 +258,23 @@ def make_cfg(settings):
     sim_cfg.physics_config_file = settings["physics_config_file"]
 
     # Note: all sensors must have the same resolution
-    sensors = {
-        "rgb": {
-            "sensor_type": habitat_sim.SensorType.COLOR,
-            "resolution": [settings["height"], settings["width"]],
-            "position": [0.0, settings["sensor_height"], 0.0],
-        },
-        "depth": {
-            "sensor_type": habitat_sim.SensorType.DEPTH,
-            "resolution": [settings["height"], settings["width"]],
-            "position": [0.0, settings["sensor_height"], 0.0],
-        },
-    }
-
     sensor_specs = []
-    for sensor_uuid, sensor_params in sensors.items():
-        if settings[sensor_uuid]:
-            sensor_spec = habitat_sim.SensorSpec()
-            sensor_spec.uuid = sensor_uuid
-            sensor_spec.sensor_type = sensor_params["sensor_type"]
-            sensor_spec.resolution = sensor_params["resolution"]
-            sensor_spec.position = sensor_params["position"]
-
-            sensor_specs.append(sensor_spec)
+
+    rgb_sensor_spec = habitat_sim.CameraSensorSpec()
+    rgb_sensor_spec.uuid = "rgb"
+    rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR
+    rgb_sensor_spec.resolution = [settings["height"], settings["width"]]
+    rgb_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
+    rgb_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
+    sensor_specs.append(rgb_sensor_spec)
+
+    depth_sensor_spec = habitat_sim.CameraSensorSpec()
+    depth_sensor_spec.uuid = "depth"
+    depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH
+    depth_sensor_spec.resolution = [settings["height"], settings["width"]]
+    depth_sensor_spec.position = [0.0, settings["sensor_height"], 0.0]
+    depth_sensor_spec.sensor_subtype = habitat_sim.SensorSubType.PINHOLE
+    sensor_specs.append(depth_sensor_spec)
 
     # Here you can specify the amount of displacement in a forward action and the turn angle
     agent_cfg = habitat_sim.agent.AgentConfiguration()

habitat/sims/habitat_simulator/habitat_simulator.py

@@ -82,9 +82,11 @@ def if_config_to_lower(config):
 @registry.register_sensor
 class HabitatSimRGBSensor(RGBSensor):
     sim_sensor_type: habitat_sim.SensorType
+    sim_sensor_subtype: habitat_sim.SensorSubType
 
     def __init__(self, config: Config) -> None:
         self.sim_sensor_type = habitat_sim.SensorType.COLOR
+        self.sim_sensor_subtype = habitat_sim.SensorSubType.PINHOLE
         super().__init__(config=config)
 
     def _get_observation_space(self, *args: Any, **kwargs: Any) -> Box:
@@ -109,11 +111,13 @@ def get_observation(
 @registry.register_sensor
 class HabitatSimDepthSensor(DepthSensor):
     sim_sensor_type: habitat_sim.SensorType
+    sim_sensor_subtype: habitat_sim.SensorSubType
     min_depth_value: float
     max_depth_value: float
 
     def __init__(self, config: Config) -> None:
         self.sim_sensor_type = habitat_sim.SensorType.DEPTH
+        self.sim_sensor_subtype = habitat_sim.SensorSubType.PINHOLE
 
         if config.NORMALIZE_DEPTH:
             self.min_depth_value = 0
@@ -160,9 +164,11 @@ def get_observation(
 @registry.register_sensor
 class HabitatSimSemanticSensor(SemanticSensor):
     sim_sensor_type: habitat_sim.SensorType
+    sim_sensor_subtype: habitat_sim.SensorSubType
 
     def __init__(self, config):
         self.sim_sensor_type = habitat_sim.SensorType.SEMANTIC
+        self.sim_sensor_subtype = habitat_sim.SensorSubType.PINHOLE
         super().__init__(config=config)
 
     def _get_observation_space(self, *args: Any, **kwargs: Any):
@@ -257,8 +263,37 @@ def create_sim_config(
         )
 
         sensor_specifications = []
+        VisualSensorTypeSet = {
+            habitat_sim.SensorType.COLOR,
+            habitat_sim.SensorType.DEPTH,
+            habitat_sim.SensorType.SEMANTIC,
+        }
+        CameraSensorSubTypeSet = {
+            habitat_sim.SensorSubType.PINHOLE,
+            habitat_sim.SensorSubType.ORTHOGRAPHIC,
+        }
         for sensor in _sensor_suite.sensors.values():
-            sim_sensor_cfg = habitat_sim.SensorSpec()
+
+            # Check if type VisualSensorSpec, we know that Sensor is one of HabitatSimRGBSensor, HabitatSimDepthSensor, HabitatSimSemanticSensor
+            if (
+                getattr(sensor, "sim_sensor_type", [])
+                not in VisualSensorTypeSet
+            ):
+                raise ValueError(
+                    f"""{getattr(sensor, "sim_sensor_type", [])} is an illegal sensorType that is not implemented yet"""
+                )
+            # Check if type CameraSensorSpec
+            if (
+                getattr(sensor, "sim_sensor_subtype", [])
+                not in CameraSensorSubTypeSet
+            ):
+                raise ValueError(
+                    f"""{getattr(sensor, "sim_sensor_subtype", [])} is an illegal sensorSubType for a VisualSensor"""
+                )
+
+            # TODO: Implement checks for other types of SensorSpecs
+
+            sim_sensor_cfg = habitat_sim.CameraSensorSpec()
             # TODO Handle configs for custom VisualSensors that might need
             # their own ignore_keys. Maybe with special key / checking
             # SensorType
@@ -281,13 +316,13 @@ def create_sim_config(
             sim_sensor_cfg.resolution = list(
                 sensor.observation_space.shape[:2]
             )
-            sim_sensor_cfg.parameters["hfov"] = str(sensor.config.HFOV)
 
             # TODO(maksymets): Add configure method to Sensor API to avoid
             # accessing child attributes through parent interface
             # We know that the Sensor has to be one of these Sensors
             sensor = cast(HabitatSimVizSensors, sensor)
             sim_sensor_cfg.sensor_type = sensor.sim_sensor_type
+            sim_sensor_cfg.sensor_subtype = sensor.sim_sensor_subtype
             sim_sensor_cfg.gpu2gpu_transfer = (
                 self.habitat_config.HABITAT_SIM_V0.GPU_GPU
             )