Note
Go to the end to download the full example code.
CSI human pose estimation Tutorial¶
# !pip install pysensing
In this tutorial, we will be implementing codes for CSI human pose estimation task
import sys
sys.path.append('../..')
import torch
import pysensing.csi.dataset.get_dataloader as get_dataloader
import pysensing.csi.model.get_model as get_model
import pysensing.csi.inference.predict as predict
import pysensing.csi.inference.embedding as embedding
import pysensing.csi.inference.train as train
from torch.utils.data import Subset, DataLoader
Load the data¶
# MMFi, the first multi-modal non-intrusive 4D human dataset with 27 daily or rehabilitation action categories, leveraging LiDAR, mmWave radar, and WiFi signals for device-free human sensing.. MM-Fi consists of over 320k synchronized frames of five modalities from 40 human subjects.
# WiPose consists of 166,600 packets of .mat format. These packets contain pose annotations and WiFi channel state information (CSI) of 12 different actions performed by 12 volunteers, including wave, walk, throw, run, push, pull, jump, crouch, circle, sit down, stand up, and bend.
train_loader, val_loader = get_dataloader.load_hpe_dataset(dataset_name='MMFi', protocol='protocol1', split_to_use='random_split', random_seed=0, random_ratio=0.8, batch_size=1, data_unit='frame')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
for i, data in enumerate(train_loader):
csi = data['input_wifi-csi'].type(torch.FloatTensor).to(device)
label = data['output'].to(device)
break
print('csi: ', csi)
print('label: ', label)
csi: tensor([[[[0.7791, 0.8007, 0.7848, ..., 0.7802, 0.7284, 0.6333],
[0.7943, 0.8174, 0.8042, ..., 0.7861, 0.7453, 0.6553],
[0.8086, 0.8345, 0.8122, ..., 0.8116, 0.7532, 0.6587],
...,
[0.6533, 0.6729, 0.6580, ..., 0.6221, 0.5164, 0.4519],
[0.6396, 0.6647, 0.6376, ..., 0.6217, 0.5251, 0.4410],
[0.6321, 0.6598, 0.6466, ..., 0.6211, 0.5307, 0.4314]],
[[0.8095, 0.7050, 0.7830, ..., 0.8129, 0.7579, 0.7874],
[0.8185, 0.7290, 0.8043, ..., 0.8239, 0.7769, 0.8036],
[0.8418, 0.7448, 0.8047, ..., 0.8430, 0.8126, 0.8191],
...,
[0.8644, 0.7684, 0.8265, ..., 0.8137, 0.8221, 0.8429],
[0.8474, 0.7495, 0.8169, ..., 0.7897, 0.8109, 0.8338],
[0.8418, 0.7304, 0.8070, ..., 0.7855, 0.7931, 0.8157]],
[[0.7520, 0.7984, 0.7713, ..., 0.7669, 0.7578, 0.7565],
[0.7677, 0.8183, 0.8004, ..., 0.7783, 0.7658, 0.7733],
[0.8009, 0.8357, 0.8051, ..., 0.8101, 0.7874, 0.7724],
...,
[0.8386, 0.8775, 0.8504, ..., 0.8853, 0.8804, 0.8851],
[0.8110, 0.8516, 0.8331, ..., 0.8607, 0.8634, 0.8613],
[0.7953, 0.8346, 0.8155, ..., 0.8428, 0.8434, 0.8419]]]],
device='cuda:0')
label: tensor([[[ 0.4694, -0.0193, 3.1069],
[ 0.3976, -0.0218, 3.1069],
[ 0.3420, 0.3704, 3.1069],
[ 0.3105, 0.7423, 3.1069],
[ 0.5412, -0.0168, 3.1069],
[ 0.5958, 0.3704, 3.1069],
[ 0.6151, 0.7423, 3.1069],
[ 0.4704, -0.2922, 3.0957],
[ 0.4713, -0.5652, 3.0844],
[ 0.4318, -0.6559, 3.0425],
[ 0.4427, -0.7158, 3.0788],
[ 0.6081, -0.5198, 3.1344],
[ 0.7167, -0.7992, 3.1323],
[ 0.7433, -1.0985, 3.1069],
[ 0.3742, -0.5198, 3.0763],
[ 0.3471, -0.2208, 3.0740],
[ 0.3486, 0.0785, 3.0717]]], device='cuda:0')
Load the model¶
For MMFi dataset, model zoo contains WPNet and WPFormer
model = get_model.load_hpe_model('MMFi', 'WPNet')
print(model)
WPNet(
(encoder_conv1): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(encoder_bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(encoder_relu): ReLU(inplace=True)
(encoder_maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
(encoder_layer1): Sequential(
(0): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(1): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(2): BasicBlock(
(conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(encoder_layer2): Sequential(
(0): BasicBlock(
(conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(downsample): Sequential(
(0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BasicBlock(
(conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(2): BasicBlock(
(conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(3): BasicBlock(
(conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(encoder_layer3): Sequential(
(0): BasicBlock(
(conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(downsample): Sequential(
(0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BasicBlock(
(conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(2): BasicBlock(
(conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(3): BasicBlock(
(conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(4): BasicBlock(
(conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(5): BasicBlock(
(conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(encoder_layer4): Sequential(
(0): BasicBlock(
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(downsample): Sequential(
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BasicBlock(
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(2): BasicBlock(
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(decode): Sequential(
(0): Conv2d(512, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): Tanh()
(3): Conv2d(32, 3, kernel_size=(1, 1), stride=(1, 1), bias=False)
)
(m): AvgPool2d(kernel_size=(1, 4), stride=(1, 4), padding=0)
(bn1): BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(rl): ReLU(inplace=True)
)
Model train¶
criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
epoch_num = 1
subset = Subset(train_loader.dataset, range(10))
subset_train_loader = DataLoader(subset, batch_size=train_loader.batch_size, shuffle=False)
train.hpe_train(subset_train_loader, model, epoch_num, optimizer, criterion, device)
Epoch:1, Loss:2.390793514
Model inference¶
model = get_model.load_pretrain(model, 'MMFi', 'WPNet', device=device)
output = predict.hpe_predict(csi, 'MMFi', model, device).to(device)
print('output: ', output)
output: tensor([[[ 0.0448, 0.0100, 3.2204],
[-0.0646, 0.0319, 3.2430],
[-0.0605, 0.3574, 3.1652],
[-0.0634, 0.7401, 3.2095],
[ 0.1069, -0.0335, 3.2347],
[ 0.1307, 0.3981, 3.2368],
[ 0.1460, 0.7888, 3.2247],
[ 0.0370, -0.2766, 3.1647],
[ 0.0684, -0.5362, 3.1459],
[ 0.1094, -0.6291, 3.1650],
[ 0.0895, -0.7127, 3.2038],
[ 0.2047, -0.5117, 3.2208],
[ 0.2679, -0.3496, 3.1751],
[ 0.2398, -0.2195, 3.1540],
[-0.1316, -0.4661, 3.1002],
[-0.2405, -0.3244, 3.0981],
[-0.2011, -0.2135, 3.0832]]], device='cuda:0')
Evaluate loss¶
criterion = torch.nn.MSELoss().to(device)
loss = criterion(output, label)
print(loss)
tensor(0.0911, device='cuda:0')
Generate embeddings¶
csi_embedding = embedding.hpe_csi_embedding(csi, 'MMFi', model, device)
print('csi_embedding: ', csi_embedding)
csi_embedding: tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.6898, 0.5873, 0.0000, 0.0000],
...,
[ 0.9009, 1.4870, 2.3145, 1.5277],
[ 2.9317, 6.7121, 8.0602, 5.2677],
[ 2.5851, 6.4446, 7.9997, 6.6200]],
[[ 5.0607, 6.1535, 3.8363, 1.6712],
[ 1.9554, 2.1506, 3.1182, 1.9268],
[ 0.6383, 0.0285, 0.1101, 0.0000],
...,
[ 0.8470, 1.6228, 1.7211, 0.7037],
[ 2.8147, 5.0181, 6.2785, 3.2779],
[ 0.3848, 1.7226, 3.3847, 2.3418]],
[[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.2151, 0.0000, 0.1223, 0.0294],
...,
[ 0.1593, 0.3754, 0.0983, 0.0000],
[ 3.0931, 5.3427, 6.8627, 4.9230],
[ 4.2251, 7.7868, 10.0394, 7.7943]],
...,
[[ 0.0814, 0.1308, 0.1284, 0.0000],
[ 0.2233, 0.3697, 0.0917, 0.0152],
[ 0.0000, 0.0000, 0.0000, 0.0000],
...,
[ 0.6894, 0.9976, 0.9500, 0.5045],
[ 3.7091, 6.4432, 7.3621, 4.6241],
[ 5.4806, 9.1068, 10.5166, 7.4853]],
[[ 0.8244, 0.8828, 0.8691, 0.8306],
[ 0.1502, 0.0000, 0.7248, 1.9703],
[ 1.9535, 2.6372, 1.9974, 1.9217],
...,
[ 0.6471, 0.4671, 0.8975, 1.0521],
[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000]],
[[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.5928, 0.3453, 0.1039, 0.1841],
...,
[ 1.1238, 1.3818, 1.8406, 1.2281],
[ 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000]]]], device='cuda:0',
grad_fn=<ReluBackward0>)
And that’s it. We’re done with our CSI human pose estimation tutorials. Thanks for reading.
Total running time of the script: (1 minutes 12.540 seconds)
