因为之前正好看了CMU在CVPR2017上的论文《Realtime Multi-Person 2D Pose Estimation using Part Affinity Fields》，而且他们提供了训练好的模型。所以就直接用CMU训练的模型在AI challenge的数据集上做了测试。最后没有使用AI challenge训练集训练的模型在AI challenge上的得分是0.1667，可以看作是一个baseline。
以下是预处理的说明以及加入预处理程序的源代码。openpose的源代码使用#openpose ##openpose标注出来了，剩下的就是AI challenge的预处理程序。
在Google Cloud 上使用1片NVIDIA Tesla K80 跑完AI challenge的测试集大约需要24小时，4秒左右处理一副图。
AI challenge测试要求的关键点顺序是：1右肩，2右肘，3右腕，4左肩，5左肘，6左腕，7右髋，8右膝，9右踝，10左髋，11左膝，12左踝，13头顶，14脖子
openpose源码中subset输出的关键点顺序是：1鼻子，2脖子，3右肩，4右肘，5右腕，6左肩，7左肘，8左腕，9右髋，10右膝，11右踝，12左髋，13左膝，14左踝，15左眼，16右眼，17左耳，18右耳，19 pt19
函数 subset2AIsubset, all_peaks2all_peaks_1d, listMultiKeypoints 负责把openpose的关键点转换成AI challenge 的关键点。
当然还得按照官网上的要求输出特定格式的JSON文件,如下所示：
[
{
"image_id": "a0f6bdc065a602b7b84a67fb8d14ce403d902e0d",
"keypoint_annotations": {
"human1": [261, 294, 1, 281, 328, 1, 0, 0, 0, 213, 295, 1, 208, 346, 1, 192, 335, 1, 245, 375, 1, 255, 432, 1, 244, 494, 1, 221, 379, 1, 219, 442, 1, 226, 491, 1, 226, 256, 1, 231, 284, 1],
"human2": [313, 301, 1, 305, 337, 1, 321, 345, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 359, 1, 320, 409, 1, 311, 454, 1, 0, 0, 0, 330, 409, 1, 324, 446, 1, 337, 284, 1, 327, 302, 1],
"human3": [373, 304, 1, 346, 286, 1, 332, 263, 1, 0, 0, 0, 0, 0, 0, 345, 313, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 386, 1, 361, 424, 1, 361, 475, 1, 365, 273, 1, 369, 297, 1],
...
}
}
...
]
#import numpy as np
import json
import os
#openpose
import keras
from keras.models import Sequential
from keras.models import Model
from keras.layers import Input, Dense, Activation
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.merge import Concatenate
from config_reader import config_reader
import scipy import cv2
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
import util
import math
from numpy import ma
from scipy.ndimage.filters import gaussian_filter
##openpose
#openpose
def relu(x):
return Activation('relu')(x) def conv(x, nf, ks, name):
x1 = Conv2D(nf, (ks, ks), padding='same', name=name)(x)
return x1 def pooling(x, ks, st, name):
x = MaxPooling2D((ks, ks), strides=(st, st), name=name)(x)
return x def vgg_block(x): # Block 1
x = conv(x, 64, 3, "conv1_1")
x = relu(x)
x = conv(x, 64, 3, "conv1_2")
x = relu(x)
x = pooling(x, 2, 2, "pool1_1") # Block 2
x = conv(x, 128, 3, "conv2_1")
x = relu(x)
x = conv(x, 128, 3, "conv2_2")
x = relu(x)
x = pooling(x, 2, 2, "pool2_1") # Block 3
x = conv(x, 256, 3, "conv3_1")
x = relu(x)
x = conv(x, 256, 3, "conv3_2")
x = relu(x)
x = conv(x, 256, 3, "conv3_3")
x = relu(x)
x = conv(x, 256, 3, "conv3_4")
x = relu(x)
x = pooling(x, 2, 2, "pool3_1") # Block 4
x = conv(x, 512, 3, "conv4_1")
x = relu(x)
x = conv(x, 512, 3, "conv4_2")
x = relu(x) # Additional non vgg layers
x = conv(x, 256, 3, "conv4_3_CPM")
x = relu(x)
x = conv(x, 128, 3, "conv4_4_CPM")
x = relu(x) return x def stage1_block(x, num_p, branch): # Block 1
x = conv(x, 128, 3, "conv5_1_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_2_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_3_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 512, 1, "conv5_4_CPM_L%d" % branch)
x = relu(x)
x = conv(x, num_p, 1, "conv5_5_CPM_L%d" % branch) return x def stageT_block(x, num_p, stage, branch): # Block 1
x = conv(x, 128, 7, "Mconv1_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv2_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv3_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv4_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv5_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 1, "Mconv6_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, num_p, 1, "Mconv7_stage%d_L%d" % (stage, branch)) return x
##openpose def subset2AIsubset(t, numPersons):
AIsubset=[]
for j in xrange(numPersons):
tempsubset=[]
for i in xrange(12):
#
#print(i+2)
tempsubset.append(t[j][i+2]) tempsubset.append(t[j][0])
tempsubset.append(t[j][1])
#print(AIsubset)
AIsubset.append(tempsubset)
return AIsubset def all_peaks2all_peaks_1d(all_peaks):
all_peaks_1d=[]
for item in all_peaks:
for item1 in item:
all_peaks_1d.append(item1)
return all_peaks_1d def listMultiKeypoints(all_peaks_1d, numPersons):
multi_keypoints=[]
for i in xrange(numPersons):
sp_keypoints=[]
for j in xrange(14):
if(AIsubset[i][j]== -1.):
sp_keypoints.append(0)
sp_keypoints.append(0)
sp_keypoints.append(0)
else:
sp_keypoints.append(all_peaks_1d[int(AIsubset[i][j])][0])
sp_keypoints.append(all_peaks_1d[int(AIsubset[i][j])][1])
sp_keypoints.append(1)
#print(sp_keypoints)
multi_keypoints.append(sp_keypoints)
return multi_keypoints def nPersons(t):
return len(t) def listHuman(nPersons):
list_human=[]
for i in xrange(numPersons):
list_human.append('human'+str(i+1))
return list_human #openpose
weights_path = "model/keras/model.h5" input_shape = (None,None,3) img_input = Input(shape=input_shape) stages = 6
np_branch1 = 38
np_branch2 = 19 # VGG
stage0_out = vgg_block(img_input) # stage 1
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out]) # stage t >= 2
for sn in range(2, stages + 1):
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2)
if (sn < stages):
x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out]) model = Model(img_input, [stageT_branch1_out, stageT_branch2_out])
model.load_weights(weights_path)
##openpose #openpose
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
##openpose path = "./test0"
files = os.listdir(path)
list_image_names=[]
final_results=[]
num_processed_images=0.
total_images=30000.
for file in files:
num_processed_images+=1
print('file:',file)
print('number of image:',num_processed_images)
print('%.2f%%'%(num_processed_images/total_images*100))
list_image_names.append(str(file)[:-4])
#openpose
test_image = './test0/'+file
#test_image = 'sample_images/000a902c8674739c97f188157c63d709b45b7595.jpg'
oriImg = cv2.imread(test_image) param, model_params = config_reader()
multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'], model_params['padValue']) input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2))/256 - 0.5; # required shape (1, width, height, channels)
print("Input shape: " + str(input_img.shape)) output_blobs = model.predict(input_img)
print("Output shape (heatmap): " + str(output_blobs[1].shape)) # extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
##openpose
#openpose
U = paf_avg[:,:,16] * -1
V = paf_avg[:,:,17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M) all_peaks = []
peak_counter = 0 for part in range(19-1):
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:] peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
##openpose
#openpose
connection_all = []
special_k = []
mid_num = 10 for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
#
#print('vec0:',vec[0],'vec1:',vec[1])
# #
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1]+0.1)
vec = np.divide(vec, norm) startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
#
#print('norm',norm)
# #
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0) criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
##openpose
#openpose
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1 if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print ("found = 2")
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
##openpose
numPersons= nPersons(subset)
#print(subset2AIsubset(subset, numPersons))
AIsubset = subset2AIsubset(subset,numPersons)
#print(all_peaks[i][numPersons][3]==[int(AIsubset[0][0])])
#all_peaks->all_peaks_1d
all_peaks_1d=all_peaks2all_peaks_1d(all_peaks)
#print('numPersons:',numPersons)
#print('multi_keypoints:',listMultiKeypoints(all_peaks_1d, numPersons))
keys=['image_id','keypoint_annotations']
values=[]
image_id=str(file)[:-4] keypoint_annotations = dict(zip(listHuman(numPersons), listMultiKeypoints(all_peaks_1d, numPersons)))
values.append(image_id)
values.append(keypoint_annotations) d = dict(zip(keys, values)) final_results.append(d)
print(final_results)
with open('data.json', 'w') as f:
json.dump(final_results, f) print(list_image_names)
[
{
"image_id": "a0f6bdc065a602b7b84a67fb8d14ce403d902e0d",
"keypoint_annotations": {
"human1": [261, 294, 1, 281, 328, 1, 0, 0, 0, 213, 295, 1, 208, 346, 1, 192, 335, 1, 245, 375, 1, 255, 432, 1, 244, 494, 1, 221, 379, 1, 219, 442, 1, 226, 491, 1, 226, 256, 1, 231, 284, 1],
"human2": [313, 301, 1, 305, 337, 1, 321, 345, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 359, 1, 320, 409, 1, 311, 454, 1, 0, 0, 0, 330, 409, 1, 324, 446, 1, 337, 284, 1, 327, 302, 1],
"human3": [373, 304, 1, 346, 286, 1, 332, 263, 1, 0, 0, 0, 0, 0, 0, 345, 313, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 386, 1, 361, 424, 1, 361, 475, 1, 365, 273, 1, 369, 297, 1],
...
}
}
...
]