[Impeller Scene] Add animation/PBR descriptions to ipscene (#38397)

Author: bdero

impeller/scene/geometry.cc

@@ -53,7 +53,17 @@ std::shared_ptr<VertexBufferGeometry> Geometry::MakeFromFlatbuffer(
       break;
   }
 
-  const size_t vertices_bytes = mesh.vertices()->size() * sizeof(fb::Vertex);
+  if (mesh.vertices_type() == fb::VertexBuffer::SkinnedVertexBuffer) {
+    VALIDATION_LOG << "Skinned meshes not yet supported.";
+    return nullptr;
+  }
+  if (mesh.vertices_type() != fb::VertexBuffer::UnskinnedVertexBuffer) {
+    VALIDATION_LOG << "Invalid vertex buffer type.";
+    return nullptr;
+  }
+
+  const auto* vertices = mesh.vertices_as_UnskinnedVertexBuffer()->vertices();
+  const size_t vertices_bytes = vertices->size() * sizeof(fb::Vertex);
   const size_t indices_bytes = mesh.indices()->data()->size();
   if (vertices_bytes == 0 || indices_bytes == 0) {
     return nullptr;
@@ -67,7 +77,7 @@ std::shared_ptr<VertexBufferGeometry> Geometry::MakeFromFlatbuffer(
   buffer->SetLabel("Mesh vertices+indices");
 
   const uint8_t* vertices_start =
-      reinterpret_cast<const uint8_t*>(mesh.vertices()->Get(0));
+      reinterpret_cast<const uint8_t*>(vertices->Get(0));
   const uint8_t* indices_start =
       reinterpret_cast<const uint8_t*>(mesh.indices()->data()->Data());
 

impeller/scene/importer/importer_gltf.cc

@@ -93,7 +93,7 @@ static bool ProcessMeshPrimitive(const tinygltf::Model& gltf,
                                      accessor.count);            // count
     }
 
-    builder.WriteFBVertices(mesh_primitive.vertices);
+    builder.WriteFBVertices(mesh_primitive);
   }
 
   //---------------------------------------------------------------------------
@@ -136,6 +136,9 @@ static bool ProcessMeshPrimitive(const tinygltf::Model& gltf,
 static void ProcessNode(const tinygltf::Model& gltf,
                         const tinygltf::Node& in_node,
                         fb::NodeT& out_node) {
+  out_node.name = in_node.name;
+  out_node.children = in_node.children;
+
   //---------------------------------------------------------------------------
   /// Transform.
   ///
@@ -184,16 +187,6 @@ static void ProcessNode(const tinygltf::Model& gltf,
       out_node.mesh_primitives.push_back(std::move(mesh_primitive));
     }
   }
-
-  //---------------------------------------------------------------------------
-  /// Children.
-  ///
-
-  for (size_t node_i = 0; node_i < in_node.children.size(); node_i++) {
-    auto child = std::make_unique<fb::NodeT>();
-    ProcessNode(gltf, gltf.nodes[in_node.children[node_i]], *child);
-    out_node.children.push_back(std::move(child));
-  }
 }
 
 bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
@@ -218,10 +211,12 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) {
   }
 
   const tinygltf::Scene& scene = gltf.scenes[gltf.defaultScene];
-  for (size_t node_i = 0; node_i < scene.nodes.size(); node_i++) {
+  out_scene.children = scene.nodes;
+
+  for (size_t node_i = 0; node_i < gltf.nodes.size(); node_i++) {
     auto node = std::make_unique<fb::NodeT>();
-    ProcessNode(gltf, gltf.nodes[scene.nodes[node_i]], *node);
-    out_scene.children.push_back(std::move(node));
+    ProcessNode(gltf, gltf.nodes[node_i], *node);
+    out_scene.nodes.push_back(std::move(node));
   }
 
   return true;

impeller/scene/importer/importer_unittests.cc

@@ -21,21 +21,23 @@ TEST(ImporterTest, CanParseGLTF) {
   ASSERT_TRUE(ParseGLTF(*mapping, scene));
 
   ASSERT_EQ(scene.children.size(), 1u);
-  auto& node = *scene.children[0];
+  auto& node = scene.nodes[scene.children[0]];
 
-  Matrix node_transform = ToMatrix(*node.transform);
+  Matrix node_transform = ToMatrix(*node->transform);
   ASSERT_MATRIX_NEAR(node_transform, Matrix());
 
-  ASSERT_EQ(node.mesh_primitives.size(), 1u);
-  auto& mesh = *node.mesh_primitives[0];
+  ASSERT_EQ(node->mesh_primitives.size(), 1u);
+  auto& mesh = *node->mesh_primitives[0];
   ASSERT_EQ(mesh.indices->count, 918u);
 
   uint16_t first_index =
       *reinterpret_cast<uint16_t*>(mesh.indices->data.data());
   ASSERT_EQ(first_index, 45u);
 
-  ASSERT_EQ(mesh.vertices.size(), 260u);
-  auto& vertex = mesh.vertices[0];
+  ASSERT_EQ(mesh.vertices.type, fb::VertexBuffer::UnskinnedVertexBuffer);
+  auto& vertices = mesh.vertices.AsUnskinnedVertexBuffer()->vertices;
+  ASSERT_EQ(vertices.size(), 260u);
+  auto& vertex = vertices[0];
 
   Vector3 position = ToVector3(vertex.position());
   ASSERT_VECTOR3_NEAR(position, Vector3(-0.0100185, -0.522907, -0.133178));

impeller/scene/importer/scene.fbs

@@ -4,6 +4,56 @@
 
 namespace impeller.fb;
 
+//-----------------------------------------------------------------------------
+/// Materials.
+///
+
+struct Color {
+  r: float;
+  g: float;
+  b: float;
+  a: float;
+}
+
+/// A compressed texture for Flutter to decode. The `bytes` field takes
+/// precedent over the `uri` field.
+table Texture {
+  /// A Flutter asset URI for an image file to import and decode.
+  uri: string;
+  /// Compressed image bytes for Flutter to decode. If this field is not empty,
+  /// it takes precedent over the `uri` field for sourcing the texture.
+  bytes: [ubyte];
+}
+
+enum MaterialType:byte {
+  kUnlit,
+  kPhysicallyBased,
+}
+
+/// The final color of each material component is the texture color multiplied
+/// by the factor of the component.
+/// Texture fields are indices into the `Scene`->`textures` array. All textures
+/// are optional -- a texture index value of -1 indicates no texture.
+table  Material {
+  // When the `MaterialType` is `kUnlit`, only the `base_color` fields are used.
+  type: MaterialType;
+
+  base_color_factor: Color;
+  base_color_texture: int = -1;
+
+  metallic_factor: float = 0;
+  roughness_factor: float = 0.5;
+  metallic_roughness_texture: int = -1;  // Red=Metallic, Green=Roughness.
+
+  normal_texture: int = -1;  // Tangent space normal map.
+
+  occlusion_texture: int = -1;
+}
+
+//-----------------------------------------------------------------------------
+/// Geometry.
+///
+
 struct Vec2 {
   x: float;
   y: float;
@@ -22,17 +72,6 @@ struct Vec4 {
   w: float;
 }
 
-struct Color {
-  r: float;
-  g: float;
-  b: float;
-  a: float;
-}
-
-struct Matrix {
-  m: [float:16];
-}
-
 // This attribute layout is expected to be identical to that within
 // `impeller/scene/shaders/geometry.vert`.
 struct Vertex {
@@ -43,6 +82,27 @@ struct Vertex {
   color: Color;
 }
 
+table UnskinnedVertexBuffer {
+  vertices: [Vertex];
+}
+
+struct SkinnedVertex {
+  vertex: Vertex;
+  /// Four joint indices corresponding to this mesh's skin transforms. These
+  /// are floats instead of ints because this vertex data is uploaded directly
+  /// to the GPU, and float attributes work for all Impeller backends.
+  joints: Vec4;
+  /// Four weight values that specify the influence of the corresponding
+  /// joints.
+  weights: Vec4;
+}
+
+table SkinnedVertexBuffer {
+  vertices: [SkinnedVertex];
+}
+
+union VertexBuffer { UnskinnedVertexBuffer, SkinnedVertexBuffer }
+
 enum IndexType:byte {
   k16Bit,
   k32Bit,
@@ -54,31 +114,69 @@ table Indices {
   type: IndexType;
 }
 
-table Texture {
-  // TODO(bdero): Allow optional image data embedding.
-  uri: string;
+table MeshPrimitive {
+  vertices: VertexBuffer;
+  indices: Indices;
+  material: Material;
 }
 
-table Material {
-  base_color_factor: Color;
-  base_color_texture: Texture;
-  // TODO(bdero): PBR textures.
+//-----------------------------------------------------------------------------
+/// Animations.
+///
+
+table TranslationKeyframes {
+  values: [Vec3];
 }
 
-table MeshPrimitive {
-  vertices: [Vertex];
-  indices: Indices;
-  material: Material;
+table RotationKeyframes {
+  values: [Vec4];
+}
+
+table ScaleKeyframes {
+  values: [Vec3];
+}
+
+union Keyframes { TranslationKeyframes, RotationKeyframes, ScaleKeyframes }
+
+table Channel {
+  node: int;  // Index into `Scene`->`nodes`.
+  timeline: [float];
+  keyframes: Keyframes;
+}
+
+table Animation {
+  name: string;
+  channels: [Channel];
+}
+
+table Skin {
+  joints: [int];  // Index into `Scene`->`nodes`.
+  inverse_bind_matrices: [Matrix];
+  /// The root joint of the skeleton.
+  skeleton: int;  // Index into `Scene`->`nodes`.
+}
+
+//-----------------------------------------------------------------------------
+/// Scene graph.
+///
+
+struct Matrix {
+  m: [float:16];
 }
 
 table Node {
-  children: [Node];
+  name: string;
+  children: [int];  // Index into `Scene`->`nodes`.
   transform: Matrix;
   mesh_primitives: [MeshPrimitive];
+  skin: Skin;
 }
 
 table Scene {
-  children: [Node];
+  children: [int];  // Index into `Scene`->`nodes`.
+  nodes: [Node];
+  textures: [Texture];  // Textures may be reused across different materials.
+  animations: [Animation];
 }
 
 root_type Scene;

impeller/scene/importer/vertices_builder.cc

@@ -6,6 +6,7 @@
 
 #include <cstring>
 #include <limits>
+#include <memory>
 #include <type_traits>
 
 #include "flutter/fml/logging.h"
@@ -18,13 +19,15 @@ namespace importer {
 
 VerticesBuilder::VerticesBuilder() = default;
 
-void VerticesBuilder::WriteFBVertices(std::vector<fb::Vertex>& vertices) const {
-  vertices.resize(0);
+void VerticesBuilder::WriteFBVertices(fb::MeshPrimitiveT& primitive) const {
+  auto vertex_buffer = fb::UnskinnedVertexBufferT();
+  vertex_buffer.vertices.resize(0);
   for (auto& v : vertices_) {
-    vertices.push_back(fb::Vertex(
+    vertex_buffer.vertices.push_back(fb::Vertex(
         ToFBVec3(v.position), ToFBVec3(v.normal), ToFBVec4(v.tangent),
         ToFBVec2(v.texture_coords), ToFBColor(v.color)));
   }
+  primitive.vertices.Set(std::move(vertex_buffer));
 }
 
 /// @brief  Reads a numeric component from `source` and returns a 32bit float.

impeller/scene/importer/vertices_builder.h

@@ -57,7 +57,7 @@ class VerticesBuilder {
 
   VerticesBuilder();
 
-  void WriteFBVertices(std::vector<fb::Vertex>& vertices) const;
+  void WriteFBVertices(fb::MeshPrimitiveT& primitive) const;
 
   void SetAttributeFromBuffer(AttributeType attribute,
                               ComponentType component_type,