In [0]:
#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
|
|
Non-rigid surface deformation is a technique that, among other things, can be used to interactively manipulate meshes or to deform a template mesh to fit to a point-cloud. When manipulating meshes, this can for instance allow users to move the hand of a character, and have the rest of the arm deform in a realistic manner. It is interesting to note that the deformation can also be performed over the scale of parts or the entire mesh.
This notebook illustrates how to use Tensorflow Graphics to perform deformations similiar to the one contained in the above image.
In [0]:
!pip install tensorflow_graphics
Now that Tensorflow Graphics is installed, let's import everything needed to run the demo contained in this notebook.
In [0]:
import numpy as np
import tensorflow as tf
from tensorflow_graphics.geometry.deformation_energy import as_conformal_as_possible
from tensorflow_graphics.geometry.representation.mesh import utils as mesh_utils
from tensorflow_graphics.geometry.transformation import quaternion
from tensorflow_graphics.math.optimizer import levenberg_marquardt
from tensorflow_graphics.notebooks import threejs_visualization
from tensorflow_graphics.notebooks.resources import triangulated_stripe
In this example, we build a mesh that corresponds to a flat and rectangular surface. Using the sliders, you can control the position of the deformation constraints applied to that surface, which respectively correspond to all the points along the left boundary, center, and right boundary of the mesh.
In [0]:
mesh_rest_pose = triangulated_stripe.mesh
connectivity = mesh_utils.extract_unique_edges_from_triangular_mesh(triangulated_stripe.mesh['faces'])
camera = threejs_visualization.build_perspective_camera(
field_of_view=40.0, position=(0.0, -5.0, 5.0))
width = 500
height = 500
_ = threejs_visualization.triangular_mesh_renderer([mesh_rest_pose],
width=width,
height=height,
camera=camera)
In [0]:
###############
# UI controls #
###############
#@title Constraints on the deformed pose { vertical-output: false, run: "auto" }
constraint_1_z = 0 #@param { type: "slider", min: -1, max: 1 , step: 0.05 }
constraint_2_z = -1 #@param { type: "slider", min: -1, max: 1 , step: 0.05 }
constraint_3_z = 0 #@param { type: "slider", min: -1, max: 1 , step: 0.05 }
vertices_rest_pose = tf.Variable(mesh_rest_pose['vertices'])
vertices_deformed_pose = np.copy(mesh_rest_pose['vertices'])
num_vertices = vertices_deformed_pose.shape[0]
# Adds the user-defined constraints
vertices_deformed_pose[0, 2] = constraint_1_z
vertices_deformed_pose[num_vertices // 2, 2] = constraint_1_z
vertices_deformed_pose[num_vertices // 4, 2] = constraint_2_z
vertices_deformed_pose[num_vertices // 2 + num_vertices // 4, 2] = constraint_2_z
vertices_deformed_pose[num_vertices // 2 - 1, 2] = constraint_3_z
vertices_deformed_pose[-1, 2] = constraint_3_z
mesh_deformed_pose = {
'vertices': vertices_deformed_pose,
'faces': mesh_rest_pose['faces']
}
vertices_deformed_pose = tf.Variable(vertices_deformed_pose)
# Builds a camera and render the mesh.
camera = threejs_visualization.build_perspective_camera(
field_of_view=40.0, position=(0.0, -5.0, 5.0))
_ = threejs_visualization.triangular_mesh_renderer([mesh_rest_pose],
width=width,
height=height,
camera=camera)
_ = threejs_visualization.triangular_mesh_renderer([mesh_deformed_pose],
width=width,
height=height,
camera=camera)
geometries = threejs_visualization.triangular_mesh_renderer(
[mesh_deformed_pose], width=width, height=height, camera=camera)
################
# Optimization #
################
def update_viewer_callback(iteration, objective_value, variables):
"""Callback to be called at each step of the optimization."""
geometries[0].getAttribute('position').copyArray(
variables[0].numpy().ravel().tolist())
geometries[0].getAttribute('position').needsUpdate = True
geometries[0].computeVertexNormals()
def deformation_energy(vertices_deformed_pose, rotation):
"""As conformal as possible deformation energy."""
return as_conformal_as_possible.energy(
vertices_rest_pose,
vertices_deformed_pose,
rotation,
connectivity,
aggregate_loss=False)
def soft_constraints(vertices_deformed_pose):
"""Soft constrains forcing results to obey the user-defined constraints."""
weight = 10.0
return (
weight * (vertices_deformed_pose[0, 2] - constraint_1_z),
weight * (vertices_deformed_pose[num_vertices // 2, 2] - constraint_1_z),
weight * (vertices_deformed_pose[num_vertices // 4, 2] - constraint_2_z),
weight * (vertices_deformed_pose[num_vertices // 2 + num_vertices // 4, 2] -
constraint_2_z),
weight *
(vertices_deformed_pose[num_vertices // 2 - 1, 2] - constraint_3_z),
weight * (vertices_deformed_pose[-1, 2] - constraint_3_z),
)
def fitting_energy(vertices_deformed_pose, rotation):
deformation = deformation_energy(vertices_deformed_pose, rotation)
constraints = soft_constraints(vertices_deformed_pose)
return tf.concat((deformation, constraints), axis=0)
rotations = tf.Variable(quaternion.from_euler(np.zeros((num_vertices, 3))))
max_iterations = 15 #@param { isTemplate: true, type: "integer" }
_ = levenberg_marquardt.minimize(
residuals=fitting_energy,
variables=(vertices_deformed_pose, rotations),
max_iterations=int(max_iterations),
callback=update_viewer_callback)