Skip to content

Commit

Permalink
blue face fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
@lilyjeon13
lilyjeon13 committed Jun 21, 2021
1 parent c7fc485 commit 81084d8
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 152 deletions.
150 changes: 1 addition & 149 deletions step1_1_manipulate_expression.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,131 +28,6 @@ def load_facerecon_graph(graph_filename):
output = graph.get_tensor_by_name('resnet/coeff:0')
return graph, input, output

def Texture_formation_block(tex_coeff, facemodel):
# print(f"tex_coeff: {tex_coeff.shape}") #(1, 80)
# print(f"facemodel.texBase: {facemodel.texBase.shape}") # (107127, 80)
# print(f"facemodel.meantex: {facemodel.meantex.shape}") # (1, 107127)
face_texture = np.einsum('ij,aj->ai',facemodel.texBase,tex_coeff) + facemodel.meantex
# reshape face texture to [batchsize,N,3], note that texture is in RGB order
face_texture = np.reshape(face_texture,[-1,3])

return face_texture

def Compute_norm(face_shape,facemodel):
shape = face_shape
face_id = facemodel.tri
point_id = facemodel.point_buf

## expand dim to [batch, N, 3] #TODO:
shape = tf.expand_dims(shape, axis = 0)

# face_id and point_id index starts from 1
face_id = tf.cast(face_id - 1,tf.int32)
point_id = tf.cast(point_id - 1,tf.int32)

#compute normal for each face
v1 = tf.gather(shape,face_id[:,0], axis = 1)
v2 = tf.gather(shape,face_id[:,1], axis = 1)
v3 = tf.gather(shape,face_id[:,2], axis = 1)
e1 = v1 - v2
e2 = v2 - v3
face_norm = tf.cross(e1,e2)

face_norm = tf.nn.l2_normalize(face_norm, dim = 2) # normalized face_norm first
## TODO:
face_norm = tf.cast(face_norm, tf.float32)
face_norm = tf.concat([face_norm,tf.zeros([tf.shape(shape)[0],1,3])], axis = 1) #TODO:

#compute normal for each vertex using one-ring neighborhood
v_norm = tf.reduce_sum(tf.gather(face_norm, point_id, axis = 1), axis = 2)
v_norm = tf.nn.l2_normalize(v_norm, dim = 2)

return v_norm


def Compute_rotation_matrix(angles):
n_data = tf.shape(angles)[0]

# compute rotation matrix for X-axis, Y-axis, Z-axis respectively
rotation_X = tf.concat([tf.ones([n_data,1]),
tf.zeros([n_data,3]),
tf.reshape(tf.cos(angles[:,0]),[n_data,1]),
-tf.reshape(tf.sin(angles[:,0]),[n_data,1]),
tf.zeros([n_data,1]),
tf.reshape(tf.sin(angles[:,0]),[n_data,1]),
tf.reshape(tf.cos(angles[:,0]),[n_data,1])],
axis = 1
)

rotation_Y = tf.concat([tf.reshape(tf.cos(angles[:,1]),[n_data,1]),
tf.zeros([n_data,1]),
tf.reshape(tf.sin(angles[:,1]),[n_data,1]),
tf.zeros([n_data,1]),
tf.ones([n_data,1]),
tf.zeros([n_data,1]),
-tf.reshape(tf.sin(angles[:,1]),[n_data,1]),
tf.zeros([n_data,1]),
tf.reshape(tf.cos(angles[:,1]),[n_data,1])],
axis = 1
)

rotation_Z = tf.concat([tf.reshape(tf.cos(angles[:,2]),[n_data,1]),
-tf.reshape(tf.sin(angles[:,2]),[n_data,1]),
tf.zeros([n_data,1]),
tf.reshape(tf.sin(angles[:,2]),[n_data,1]),
tf.reshape(tf.cos(angles[:,2]),[n_data,1]),
tf.zeros([n_data,3]),
tf.ones([n_data,1])],
axis = 1
)

rotation_X = tf.reshape(rotation_X,[n_data,3,3])
rotation_Y = tf.reshape(rotation_Y,[n_data,3,3])
rotation_Z = tf.reshape(rotation_Z,[n_data,3,3])

# R = RzRyRx
rotation = tf.matmul(tf.matmul(rotation_Z,rotation_Y),rotation_X)

rotation = tf.transpose(rotation, perm = [0,2,1])

return rotation

def Illumination_block(face_texture,norm_r,gamma):
n_data = tf.shape(gamma)[0]
n_point = tf.shape(norm_r)[1]
gamma = tf.reshape(gamma,[n_data,3,9])
# set initial lighting with an ambient lighting
init_lit = tf.constant([0.8,0,0,0,0,0,0,0,0])
gamma = gamma + tf.reshape(init_lit,[1,1,9])

# compute vertex color using SH function approximation
a0 = m.pi
a1 = 2*m.pi/tf.sqrt(3.0)
a2 = 2*m.pi/tf.sqrt(8.0)
c0 = 1/tf.sqrt(4*m.pi)
c1 = tf.sqrt(3.0)/tf.sqrt(4*m.pi)
c2 = 3*tf.sqrt(5.0)/tf.sqrt(12*m.pi)

Y = tf.concat([tf.tile(tf.reshape(a0*c0,[1,1,1]),[n_data,n_point,1]),
tf.expand_dims(-a1*c1*norm_r[:,:,1],2),
tf.expand_dims(a1*c1*norm_r[:,:,2],2),
tf.expand_dims(-a1*c1*norm_r[:,:,0],2),
tf.expand_dims(a2*c2*norm_r[:,:,0]*norm_r[:,:,1],2),
tf.expand_dims(-a2*c2*norm_r[:,:,1]*norm_r[:,:,2],2),
tf.expand_dims(a2*c2*0.5/tf.sqrt(3.0)*(3*tf.square(norm_r[:,:,2])-1),2),
tf.expand_dims(-a2*c2*norm_r[:,:,0]*norm_r[:,:,2],2),
tf.expand_dims(a2*c2*0.5*(tf.square(norm_r[:,:,0])-tf.square(norm_r[:,:,1])),2)],axis = 2)

color_r = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,0,:],2)),axis = 2)
color_g = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,1,:],2)),axis = 2)
color_b = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,2,:],2)),axis = 2)
face_texture = tf.expand_dims(face_texture, axis = 0)
#[batchsize,N,3] vertex color in RGB order
face_color = tf.stack([color_r*face_texture[:,:,0],color_g*face_texture[:,:,1],color_b*face_texture[:,:,2]],axis = 2)

return face_color


def face_recon(src_path, tgt_path, input_path, output_path, output, vis_path=None, s_factor=1.5, focal=1015, center=112, align_nums=10, degree=100):
# load BFM
facemodel = read_facemodel()
Expand Down Expand Up @@ -215,7 +90,7 @@ def face_recon(src_path, tgt_path, input_path, output_path, output, vis_path=Non
# transfer target expression to source expression
degree = degree/100
coeff[:, 80:144] = tgt_coeff[:, 80:144] * degree + coeff[:, 80:144] * (1-degree)
# from step1_3_manipulate_illumination
# transfer target illumination to source illumination
coeff[:, 227:254] = tgt_coeff[:, 227:254] * degree + tgt_coeff[:, 227:254]* (1-degree)

# preprocess input image for depth recon net
Expand Down Expand Up @@ -266,29 +141,6 @@ def face_recon(src_path, tgt_path, input_path, output_path, output, vis_path=Non
'face_mask': d_map[..., -1].squeeze(0),
'face_tri': facemodel.tri}, do_compression=True)

# # Get Texture from coefficient
# face_xyz = face_shape.astype(np.float32)
# tex_coeff = coeff[:,144:224]
# face_texture = Texture_formation_block(tex_coeff, facemodel)

# angles = coeff[:,224:227]
# rotation = Compute_rotation_matrix(angles)

# face_norm = Compute_norm(face_shape, facemodel)
# norm_r = tf.matmul(face_norm,rotation)

# gamma = coeff[:,227:254]
# face_color = Illumination_block(face_texture, norm_r, gamma)
# face_color = face_color.numpy()
# face_color = np.squeeze(face_color)
# face_tri = facemodel.tri

# with open(os.path.join(save_path, 'step1' + '.obj'), 'w') as f:
# face_color_ = np.clip(face_color,0,255)/255
# for i in range(face_xyz.shape[0]):
# f.write('v %f %f %f %f %f %f\n' %(*face_xyz[i, :], *face_color_[i, :]))
# for i in range(face_tri.shape[0]):
# f.write('f {} {} {}\n'.format(*face_tri[i, :]))

face_recon_sess.close()
render_sess.close()
Expand Down
5 changes: 2 additions & 3 deletions step3_1_modify_texture.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ def Illumination_block(face_texture,norm_r,gamma):
color_r = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,0,:],2)),axis = 2)
color_g = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,1,:],2)),axis = 2)
color_b = tf.squeeze(tf.matmul(Y,tf.expand_dims(gamma[:,2,:],2)),axis = 2)

face_texture = tf.expand_dims(face_texture, axis = 0)
#[batchsize,N,3] vertex color in RGB order
face_color = tf.stack([color_r*face_texture[:,:,0],color_g*face_texture[:,:,1],color_b*face_texture[:,:,2]],axis = 2)
Expand Down Expand Up @@ -221,7 +221,6 @@ def depth_recon(data_path, save_path):
input_uv: hairear_uv_input,
input_facedata: face3d_data_input
})


# render head depth
head_xyz = np.concatenate([
Expand All @@ -241,7 +240,7 @@ def depth_recon(data_path, save_path):
facemodel = read_facemodel()
coeff = face3d_data_input[:,3:260]
tex_coeff = coeff[:,144:224]
face_texture = Texture_formation_block(tex_coeff, facemodel)
face_texture = Texture_formation_block(tex_coeff, facemodel)[:, ::-1] # rgb to bgr

angles = coeff[:,224:227]
rotation = Compute_rotation_matrix(angles)
Expand Down

0 comments on commit 81084d8

Please sign in to comment.