diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6c9b3f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +visual_res/* +frames/ +__pycache__ +*.pt +*.pth +*.mp4 +*.pyc +surgery_gesture_r2plus1d.py +surgery* \ No newline at end of file diff --git a/README.md b/README.md index 8b43444..b8e3ee7 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ # VideoVisual + +This is a PyTorch demo implemented several visualization methods for video classification networks. The target is to provide a toolkit (as [TorchRay](https://github.com/facebookresearch/TorchRay) to image) to interprete commonly utilized video classfication networks, such as I3D, R(2+1)D, TSM et al., which is also called *attribution* task, namely the problem of determining which part of the input video is responsible for the value computed by a neural network. + +The current version supports attribution methods and video classification models as following: + +#### Video classification models: +* **Pretrained on Kinetics-400**: I3D, R(2+1)D, R3D, MC3, TSM; +* **Pretrained on EPIC-Kitchens**: (noun & verb): TSM. + +#### Attribution methods: +* **Backprop-based**: Gradients, Gradients x Inputs, Integrated Gradients; +* **Activation-based**: GradCAM (does not support TSM now); +* **Perturbation-based**: Extremal Perturbation and Spatiotemporal Perturbation (An extension vision of extremal perturbation on video inputs). + +## Requirements + +* Python 3.6.5 or greater +* PyTorch 1.2.0 or greater +* matplotlib==2.2.3 +* numpy==1.14.3 +* opencv_python==4.1.2.30 +* torchvision==0.4.0a0 +* torchray==1.0.0.2 +* tqdm==4.45.0 +* pandas==0.23.3 +* scikit_image==0.15.0 +* Pillow==7.1.2 +* scikit_learn==0.22.2.post1 + +## Running the code + +### Examples + +#### Saptiotemporal Perturbation + I3D (pretrained on Kinetics-400) +`$ python main.py --videos_dir /home/acb11711tx/lzq/VideoVisual/test_data/kinetics/sampled_frames --model i3d --pretrain_dataset kinetics --vis_method perturb --num_iter 2000 --perturb_area 0.1` + +#### Spatiotemporal Perturbation + TSM (pretrained on EPIC-Kitchens-noun) +`$ python main.py --videos_dir /home/acb11711tx/lzq/VideoVisual/test_data/epic-kitchens-noun/sampled_frames --model tsm --pretrain_dataset epic-kitchens-noun --vis_method perturb --num_iter 2000 --perturb_area 0.05` + +#### Integrated Gradients + R(2+1)D (pretrained on Kinetics-400) +`$ python main.py --videos_dir /home/acb11711tx/lzq/VideoVisual/test_data/kinetics/sampled_frames --model r2plus1d --pretrain_dataset kinetics --vis_method integrated_grad` + +## License + +TorchRay is CC-BY-NC licensed, as found in the [LICENSE](LICENSE) file. diff --git a/datasets/universal_dataset.py b/datasets/universal_dataset.py new file mode 100644 index 0000000..88dc2ba --- /dev/null +++ b/datasets/universal_dataset.py @@ -0,0 +1,74 @@ +import torch +from torch import nn +from torchvision import transforms +from torch.utils.data import Dataset, DataLoader + +import os +import json +import numpy as np +from tqdm import tqdm +from PIL import Image + +import sys +sys.path.append(".") +sys.path.append("..") +from utils.LongRangeSample import long_range_sample + +class UniversalDataset (Dataset): + def __init__ (self, data_dir, model_name, class_namelist, clip_length=16): + self.data_dir = data_dir + self.model_name = model_name + self.class_namelist = class_namelist + self.clip_length = clip_length + + self.video_names = sorted(os.listdir(data_dir)) + assert len(self.video_names) > 0, f'Given directory contains no video.' + + if model_name == 'i3d': + self.transform = transforms.Compose([ + transforms.Resize((344, 256)), + transforms.CenterCrop((224, 224)), + transforms.ToTensor(), + transforms.Normalize([0.43216, 0.39467, 0.37645], [0.22803, 0.22145, 0.21699]), + ]) + elif model_name in ['r2plus1d', 'r3d', 'mc3']: + self.transform = transforms.Compose([ + transforms.Resize((172, 128)), + transforms.CenterCrop((112, 112)), + transforms.ToTensor(), + transforms.Normalize([0.43216, 0.39467, 0.37645], [0.22803, 0.22145, 0.21699]), + ]) + elif model_name in ['tsm', 'tsn']: # mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225] + self.transform = transforms.Compose([ + transforms.Resize((344, 256)), + transforms.CenterCrop((224, 224)), + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]) + + def __len__ (self): + return len(self.video_names) + + def __getitem__ (self, idx): + video_name = self.video_names[idx] + if '.mp4' not in video_name: + video_frames_dir = os.path.join(self.data_dir, video_name) + frame_names = sorted([f for f in os.listdir(video_frames_dir) if '.png' in f or '.jpg' in f]) + num_frame = len(frame_names) + assert num_frame > self.clip_length, \ + f"Number of frames should be larger than {self.clip_length}, given {num_frame}" + + clip_fidxs = long_range_sample(num_frame, self.clip_length, 'first') + clip_fidxs_tensor = torch.tensor(clip_fidxs).long() + + clip_frames = [Image.open(os.path.join(video_frames_dir, + f'{fidx+1:09d}.png')) for fidx in clip_fidxs] + clip_tensor = torch.stack([self.transform(frame) for frame in clip_frames], dim=1) + + label_name = video_name.split('-')[0] + label_idx = self.class_namelist.index(label_name) + return clip_tensor, label_idx, video_name, clip_fidxs_tensor + else: + raise Exception('Cannot process MP4 file yet.') + + diff --git a/main.py b/main.py new file mode 100644 index 0000000..afa9682 --- /dev/null +++ b/main.py @@ -0,0 +1,186 @@ +import argparse +import torch +import torchvision +from torch import nn +from torch.utils.data import Dataset, DataLoader + +import os +import time +import copy +from tqdm import tqdm +import pickle +import numpy as np +import pandas as pd +import csv +import json + +import warnings +warnings.filterwarnings("ignore") + +import sys +sys.path.append(".") +sys.path.append("..") + +from path_dict import PathDict +path_dict = PathDict() +proj_root = path_dict.proj_root + +from utils.ImageShow import * + +from visual_meth.integrated_grad import integrated_grad +from visual_meth.gradients import gradients +from visual_meth.perturbation import video_perturbation +from visual_meth.grad_cam import grad_cam + +parser = argparse.ArgumentParser() +parser.add_argument("--videos_dir", type=str, default='') +parser.add_argument("--model", type=str, default='r2plus1d', + choices=['r2plus1d', 'r3d', 'mc3', 'i3d', 'tsn', 'trn', 'tsm']) +parser.add_argument("--pretrain_dataset", type=str, default='kinetics', + choices=['', 'kinetics', 'epic-kitchens-verb', 'epic-kitchens-noun']) +parser.add_argument("--vis_method", type=str, default='integrated_grad', + choices=['grad', 'grad*input', 'integrated_grad', 'grad_cam', 'perturb']) +parser.add_argument("--save_label", type=str, default='') +parser.add_argument("--no_gpu", action='store_true') +parser.add_argument("--num_iter", type=int, default=1000) +parser.add_argument('--polarity', type=str, default='positive', + choices=['positive', 'negative']) +parser.add_argument('--perturb_area', type=float, default=0.1, + choices=[0.01, 0.02, 0.05, 0.1, 0.15, 0.2]) +args = parser.parse_args() + +# assert args.num_gpu >= -1 +# if args.num_gpu == 0: +# num_devices = 0 +# multi_gpu = False +# device = torch.device("cpu") +# elif args.num_gpu == 1: +# num_devices = 1 +# multi_gpu = False +# device = torch.device("cuda") +# elif args.num_gpu == -1: +# num_devices = torch.cuda.device_count() +# multi_gpu = (num_devices > 1) +# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +# else: +# num_devices = args.num_gpu +# assert torch.cuda.device_count() >= num_devices, \ +# f'Assign {args.num_gpu} GPUs, but only detected only {torch.cuda.device_count()} GPUs. Exiting...' +# multi_gpu = True +# device = torch.device("cuda") + +if args.no_gpu: + device = torch.device("cpu") + num_devices = 0 +else: + device = torch.device("cuda") + num_devices = 1 + +assert os.path.isdir(args.videos_dir), \ + f'Given directory of data does not exist: {args.videos_dir}.' + +if args.pretrain_dataset == 'kinetics': + if args.model == 'i3d': + from model_def.i3d import I3D as model + model_ft = model(num_classes=400) + i3d_pt_dir = os.path.join(proj_root, 'model_param/kinetics400_rgb_i3d.pth') + model_ft.load_state_dict(torch.load(i3d_pt_dir)) + clip_length = 16 + elif args.model == 'tsm': + from model_def.tsm import tsm as model + model_ft = model(400, segment_count=8, pretrained=args.pretrain_dataset) + clip_length = 8 + else: # Load pretrained models from PyTorch directly + clip_length = 16 + if args.model == 'r2plus1d': + from torchvision.models.video import r2plus1d_18 as model + elif args.model == 'mc3': + from torchvision.models.video import mc3_18 as model + elif args.model == 'r3d': + from torchvision.models.video import r3d_18 as model + else: + raise Exception(f'Given model of {args.model} has no pretrain on {args.pretrain_dataset}.') + model_ft = model(pretrained=True) + + model_ft = model_ft.to(device) + model_ft.eval() + # if multi_gpu: + # model_ft = nn.DataParallel(model_ft, device_ids=list(range(num_devices))) + + kinetics400_classes = os.path.join(proj_root, 'test_data/kinetics/classes.json') + class_namelist = json.load(open(kinetics400_classes)) + +elif 'epic-kitchens' in args.pretrain_dataset: + if 'noun' in args.pretrain_dataset: + epic_classes = os.path.join(proj_root, 'test_data/epic-kitchens-noun/EPIC_noun_classes.csv') + elif 'verb' in args.pretrain_dataset: + epic_classes = os.path.join(proj_root, 'test_data/epic-kitchens-verb/EPIC_verb_classes.csv') + else: + raise Exception(f'EPIC-Kitchens only supports two sub-tasks (noun & verb), given {args.pretrain_dataset}.') + class_namelist = [row['class_key'] for ridx, row in pd.read_csv(epic_classes).iterrows()] + class_num = len(class_namelist) + + if args.model == 'tsm': + from model_def.tsm import tsm as model + model_ft = model(class_num, segment_count=8, pretrained=args.pretrain_dataset) + clip_length = 16 + else: + raise Exception(f'{args.pretrain_dataset} has only pretrained TSM model. Given {args.model}.') + model_ft = model_ft.to(device) + model_ft.eval() + +from datasets.universal_dataset import UniversalDataset as dataset +test_dataset = dataset(args.videos_dir, args.model, class_namelist, clip_length=clip_length) +test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False) +print(f'Num of test samples:{len(test_dataset)}') + +for sample in tqdm(test_dataloader): + inp = sample[0].to(device) + label = sample[1].to(dtype=torch.long) + + inp_np = voxel_tensor_to_np(inp[0].detach().cpu()) # 3 x num_f x 224 224 + + if args.vis_method == 'integrated_grad': + res = integrated_grad(inp, label, model_ft, device, steps=50, polarity=args.polarity) + heatmap_np = res[0].numpy() + elif args.vis_method == 'grad': + res = gradients(inp, label, model_ft, device, polarity=args.polarity) + heatmap_np = res[0].numpy() + elif args.vis_method == 'grad*input': + res = gradients(inp, label, model_ft, device, multiply_input=True, polarity=args.polarity) + heatmap_np = res[0].numpy() + elif args.vis_method == 'grad_cam': + if args.model in ['i3d']: + layer_name = ['mixed_5c'] + elif args.model in ['r2plus1d', 'mc3', 'r3d']: # Load pretrained models from PyTorch directly + layer_name = ['layer4'] + # elif args.model in ['tsm', 'tsn']: + # layer_name = ['model', 'base_model', 'layer4'] + else: + raise Exception(f'Grad-CAM does not support {args.model} currently') + res = grad_cam(inp, label, model_ft, device, layer_name=layer_name, norm_vis=True) + heatmap_np = overlap_maps_on_voxel_np(inp_np, res[0,0].cpu().numpy(), norm_map=False) + elif args.vis_method == 'perturb': + sigma = 11 if inp.shape[-1] == 112 else 23 + res = video_perturbation( + model_ft, inp, label, areas=[args.perturb_area], sigma=sigma, + max_iter=args.num_iter, variant="preserve", + num_devices=num_devices, print_iter=100, perturb_type="fade")[0] + heatmap_np = overlap_maps_on_voxel_np(inp_np, res[0,0].cpu().numpy(), norm_map=False) + + sample_name = sample[2][0].split("/")[-1] + plot_save_name = f"{sample_name}.png" + if args.vis_method in ['grad', 'grad*input', 'integrated_grad']: + plot_save_name = plot_save_name.replace('.png', f'{args.polarity}.png') + plot_save_dir = os.path.join(proj_root, "visual_res", args.vis_method, args.model) + if args.save_label != '': + plot_save_dir = os.path.join(plot_save_dir, args.save_label) + os.makedirs(plot_save_dir, exist_ok=True) + + show_txt = f"{sample_name}" + plot_voxel_np(inp_np, heatmap_np, title=show_txt, + save_path=os.path.join(plot_save_dir, plot_save_name) ) + + + + diff --git a/model_def/i3d.py b/model_def/i3d.py new file mode 100644 index 0000000..7d0e8d9 --- /dev/null +++ b/model_def/i3d.py @@ -0,0 +1,424 @@ +import math +import os + +import numpy as np +import torch +from torch.nn import ReplicationPad3d + + +def get_padding_shape(filter_shape, stride): + def _pad_top_bottom(filter_dim, stride_val): + pad_along = max(filter_dim - stride_val, 0) + pad_top = pad_along // 2 + pad_bottom = pad_along - pad_top + return pad_top, pad_bottom + + padding_shape = [] + for filter_dim, stride_val in zip(filter_shape, stride): + pad_top, pad_bottom = _pad_top_bottom(filter_dim, stride_val) + padding_shape.append(pad_top) + padding_shape.append(pad_bottom) + depth_top = padding_shape.pop(0) + depth_bottom = padding_shape.pop(0) + padding_shape.append(depth_top) + padding_shape.append(depth_bottom) + + return tuple(padding_shape) + + +def simplify_padding(padding_shapes): + all_same = True + padding_init = padding_shapes[0] + for pad in padding_shapes[1:]: + if pad != padding_init: + all_same = False + return all_same, padding_init + + +class Unit3Dpy(torch.nn.Module): + def __init__(self, + in_channels, + out_channels, + kernel_size=(1, 1, 1), + stride=(1, 1, 1), + activation='relu', + padding='SAME', + use_bias=False, + use_bn=True): + super(Unit3Dpy, self).__init__() + + self.padding = padding + self.activation = activation + self.use_bn = use_bn + if padding == 'SAME': + padding_shape = get_padding_shape(kernel_size, stride) + simplify_pad, pad_size = simplify_padding(padding_shape) + self.simplify_pad = simplify_pad + elif padding == 'VALID': + padding_shape = 0 + else: + raise ValueError( + 'padding should be in [VALID|SAME] but got {}'.format(padding)) + + if padding == 'SAME': + if not simplify_pad: + self.pad = torch.nn.ConstantPad3d(padding_shape, 0) + self.conv3d = torch.nn.Conv3d( + in_channels, + out_channels, + kernel_size, + stride=stride, + bias=use_bias) + else: + self.conv3d = torch.nn.Conv3d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=pad_size, + bias=use_bias) + elif padding == 'VALID': + self.conv3d = torch.nn.Conv3d( + in_channels, + out_channels, + kernel_size, + padding=padding_shape, + stride=stride, + bias=use_bias) + else: + raise ValueError( + 'padding should be in [VALID|SAME] but got {}'.format(padding)) + + if self.use_bn: + self.batch3d = torch.nn.BatchNorm3d(out_channels) + + if activation == 'relu': + self.activation = torch.nn.functional.relu + + def forward(self, inp): + if self.padding == 'SAME' and self.simplify_pad is False: + inp = self.pad(inp) + out = self.conv3d(inp) + if self.use_bn: + out = self.batch3d(out) + if self.activation is not None: + out = torch.nn.functional.relu(out) + return out + + +class MaxPool3dTFPadding(torch.nn.Module): + def __init__(self, kernel_size, stride=None, padding='SAME'): + super(MaxPool3dTFPadding, self).__init__() + if padding == 'SAME': + padding_shape = get_padding_shape(kernel_size, stride) + self.padding_shape = padding_shape + self.pad = torch.nn.ConstantPad3d(padding_shape, 0) + self.pool = torch.nn.MaxPool3d(kernel_size, stride, ceil_mode=True) + + def forward(self, inp): + inp = self.pad(inp) + out = self.pool(inp) + return out + + +class Mixed(torch.nn.Module): + def __init__(self, in_channels, out_channels): + super(Mixed, self).__init__() + # Branch 0 + self.branch_0 = Unit3Dpy( + in_channels, out_channels[0], kernel_size=(1, 1, 1)) + + # Branch 1 + branch_1_conv1 = Unit3Dpy( + in_channels, out_channels[1], kernel_size=(1, 1, 1)) + branch_1_conv2 = Unit3Dpy( + out_channels[1], out_channels[2], kernel_size=(3, 3, 3)) + self.branch_1 = torch.nn.Sequential(branch_1_conv1, branch_1_conv2) + + # Branch 2 + branch_2_conv1 = Unit3Dpy( + in_channels, out_channels[3], kernel_size=(1, 1, 1)) + branch_2_conv2 = Unit3Dpy( + out_channels[3], out_channels[4], kernel_size=(3, 3, 3)) + self.branch_2 = torch.nn.Sequential(branch_2_conv1, branch_2_conv2) + + # Branch3 + branch_3_pool = MaxPool3dTFPadding( + kernel_size=(3, 3, 3), stride=(1, 1, 1), padding='SAME') + branch_3_conv2 = Unit3Dpy( + in_channels, out_channels[5], kernel_size=(1, 1, 1)) + self.branch_3 = torch.nn.Sequential(branch_3_pool, branch_3_conv2) + + def forward(self, inp): + out_0 = self.branch_0(inp) + out_1 = self.branch_1(inp) + out_2 = self.branch_2(inp) + out_3 = self.branch_3(inp) + out = torch.cat((out_0, out_1, out_2, out_3), 1) + return out + + +class I3D(torch.nn.Module): + def __init__(self, + num_classes, + modality='rgb', + dropout_prob=0, + name='inception'): + super(I3D, self).__init__() + + self.name = name + self.num_classes = num_classes + if modality == 'rgb': + in_channels = 3 + elif modality == 'flow': + in_channels = 2 + else: + raise ValueError( + '{} not among known modalities [rgb|flow]'.format(modality)) + self.modality = modality + + conv3d_1a_7x7 = Unit3Dpy( + out_channels=64, + in_channels=in_channels, + kernel_size=(7, 7, 7), + stride=(2, 2, 2), + padding='SAME') + # 1st conv-pool + self.conv3d_1a_7x7 = conv3d_1a_7x7 + self.maxPool3d_2a_3x3 = MaxPool3dTFPadding( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding='SAME') + # conv conv + conv3d_2b_1x1 = Unit3Dpy( + out_channels=64, + in_channels=64, + kernel_size=(1, 1, 1), + padding='SAME') + self.conv3d_2b_1x1 = conv3d_2b_1x1 + conv3d_2c_3x3 = Unit3Dpy( + out_channels=192, + in_channels=64, + kernel_size=(3, 3, 3), + padding='SAME') + self.conv3d_2c_3x3 = conv3d_2c_3x3 + self.maxPool3d_3a_3x3 = MaxPool3dTFPadding( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding='SAME') + + # Mixed_3b + self.mixed_3b = Mixed(192, [64, 96, 128, 16, 32, 32]) + self.mixed_3c = Mixed(256, [128, 128, 192, 32, 96, 64]) + + self.maxPool3d_4a_3x3 = MaxPool3dTFPadding( + kernel_size=(3, 3, 3), stride=(2, 2, 2), padding='SAME') + + # Mixed 4 + self.mixed_4b = Mixed(480, [192, 96, 208, 16, 48, 64]) + self.mixed_4c = Mixed(512, [160, 112, 224, 24, 64, 64]) + self.mixed_4d = Mixed(512, [128, 128, 256, 24, 64, 64]) + self.mixed_4e = Mixed(512, [112, 144, 288, 32, 64, 64]) + self.mixed_4f = Mixed(528, [256, 160, 320, 32, 128, 128]) + + self.maxPool3d_5a_2x2 = MaxPool3dTFPadding( + kernel_size=(2, 2, 2), stride=(2, 2, 2), padding='SAME') + + # Mixed 5 + self.mixed_5b = Mixed(832, [256, 160, 320, 32, 128, 128]) + self.mixed_5c = Mixed(832, [384, 192, 384, 48, 128, 128]) + + self.avg_pool = torch.nn.AvgPool3d((2, 7, 7), (1, 1, 1)) + self.dropout = torch.nn.Dropout(dropout_prob) + self.conv3d_0c_1x1 = Unit3Dpy( + in_channels=1024, + out_channels=self.num_classes, + kernel_size=(1, 1, 1), + activation=None, + use_bias=True, + use_bn=False) + self.softmax = torch.nn.Softmax(1) + + def forward(self, inp): + # Preprocessing + out = self.conv3d_1a_7x7(inp) + out = self.maxPool3d_2a_3x3(out) + out = self.conv3d_2b_1x1(out) + out = self.conv3d_2c_3x3(out) + out = self.maxPool3d_3a_3x3(out) + out = self.mixed_3b(out) + out = self.mixed_3c(out) + out = self.maxPool3d_4a_3x3(out) + out = self.mixed_4b(out) + out = self.mixed_4c(out) + out = self.mixed_4d(out) + out = self.mixed_4e(out) + out = self.mixed_4f(out) + out = self.maxPool3d_5a_2x2(out) + out = self.mixed_5b(out) + out = self.mixed_5c(out) + out = self.avg_pool(out) + out = self.dropout(out) + out = self.conv3d_0c_1x1(out) + out = out.squeeze(3) + out = out.squeeze(3) + out = out.mean(2) + # out_logits = out + # out = self.softmax(out_logits) + # return out, out_logits + return out + + def load_tf_weights(self, sess): + state_dict = {} + if self.modality == 'rgb': + prefix = 'RGB/inception_i3d' + elif self.modality == 'flow': + prefix = 'Flow/inception_i3d' + load_conv3d(state_dict, 'conv3d_1a_7x7', sess, + os.path.join(prefix, 'Conv3d_1a_7x7')) + load_conv3d(state_dict, 'conv3d_2b_1x1', sess, + os.path.join(prefix, 'Conv3d_2b_1x1')) + load_conv3d(state_dict, 'conv3d_2c_3x3', sess, + os.path.join(prefix, 'Conv3d_2c_3x3')) + + load_mixed(state_dict, 'mixed_3b', sess, + os.path.join(prefix, 'Mixed_3b')) + load_mixed(state_dict, 'mixed_3c', sess, + os.path.join(prefix, 'Mixed_3c')) + load_mixed(state_dict, 'mixed_4b', sess, + os.path.join(prefix, 'Mixed_4b')) + load_mixed(state_dict, 'mixed_4c', sess, + os.path.join(prefix, 'Mixed_4c')) + load_mixed(state_dict, 'mixed_4d', sess, + os.path.join(prefix, 'Mixed_4d')) + load_mixed(state_dict, 'mixed_4e', sess, + os.path.join(prefix, 'Mixed_4e')) + # Here goest to 0.1 max error with tf + load_mixed(state_dict, 'mixed_4f', sess, + os.path.join(prefix, 'Mixed_4f')) + + load_mixed( + state_dict, + 'mixed_5b', + sess, + os.path.join(prefix, 'Mixed_5b'), + fix_typo=True) + load_mixed(state_dict, 'mixed_5c', sess, + os.path.join(prefix, 'Mixed_5c')) + load_conv3d( + state_dict, + 'conv3d_0c_1x1', + sess, + os.path.join(prefix, 'Logits', 'Conv3d_0c_1x1'), + bias=True, + bn=False) + self.load_state_dict(state_dict) + + +def get_conv_params(sess, name, bias=False): + # Get conv weights + conv_weights_tensor = sess.graph.get_tensor_by_name( + os.path.join(name, 'w:0')) + if bias: + conv_bias_tensor = sess.graph.get_tensor_by_name( + os.path.join(name, 'b:0')) + conv_bias = sess.run(conv_bias_tensor) + conv_weights = sess.run(conv_weights_tensor) + conv_shape = conv_weights.shape + + kernel_shape = conv_shape[0:3] + in_channels = conv_shape[3] + out_channels = conv_shape[4] + + conv_op = sess.graph.get_operation_by_name( + os.path.join(name, 'convolution')) + padding_name = conv_op.get_attr('padding') + padding = _get_padding(padding_name, kernel_shape) + all_strides = conv_op.get_attr('strides') + strides = all_strides[1:4] + conv_params = [ + conv_weights, kernel_shape, in_channels, out_channels, strides, padding + ] + if bias: + conv_params.append(conv_bias) + return conv_params + + +def get_bn_params(sess, name): + moving_mean_tensor = sess.graph.get_tensor_by_name( + os.path.join(name, 'moving_mean:0')) + moving_var_tensor = sess.graph.get_tensor_by_name( + os.path.join(name, 'moving_variance:0')) + beta_tensor = sess.graph.get_tensor_by_name(os.path.join(name, 'beta:0')) + moving_mean = sess.run(moving_mean_tensor) + moving_var = sess.run(moving_var_tensor) + beta = sess.run(beta_tensor) + return moving_mean, moving_var, beta + + +def _get_padding(padding_name, conv_shape): + padding_name = padding_name.decode("utf-8") + if padding_name == "VALID": + return [0, 0] + elif padding_name == "SAME": + # return [math.ceil(int(conv_shape[0])/2), math.ceil(int(conv_shape[1])/2)] + return [ + math.floor(int(conv_shape[0]) / 2), + math.floor(int(conv_shape[1]) / 2), + math.floor(int(conv_shape[2]) / 2) + ] + else: + raise ValueError('Invalid padding name ' + padding_name) + + +def load_conv3d(state_dict, name_pt, sess, name_tf, bias=False, bn=True): + # Transfer convolution params + conv_name_tf = os.path.join(name_tf, 'conv_3d') + conv_params = get_conv_params(sess, conv_name_tf, bias=bias) + if bias: + conv_weights, kernel_shape, in_channels, out_channels, strides, padding, conv_bias = conv_params + else: + conv_weights, kernel_shape, in_channels, out_channels, strides, padding = conv_params + + conv_weights_rs = np.transpose( + conv_weights, (4, 3, 0, 1, + 2)) # to pt format (out_c, in_c, depth, height, width) + state_dict[name_pt + '.conv3d.weight'] = torch.from_numpy(conv_weights_rs) + if bias: + state_dict[name_pt + '.conv3d.bias'] = torch.from_numpy(conv_bias) + + # Transfer batch norm params + if bn: + conv_tf_name = os.path.join(name_tf, 'batch_norm') + moving_mean, moving_var, beta = get_bn_params(sess, conv_tf_name) + + out_planes = conv_weights_rs.shape[0] + state_dict[name_pt + '.batch3d.weight'] = torch.ones(out_planes) + state_dict[name_pt + + '.batch3d.bias'] = torch.from_numpy(beta.squeeze()) + state_dict[name_pt + + '.batch3d.running_mean'] = torch.from_numpy(moving_mean.squeeze()) + state_dict[name_pt + + '.batch3d.running_var'] = torch.from_numpy(moving_var.squeeze()) + + +def load_mixed(state_dict, name_pt, sess, name_tf, fix_typo=False): + # Branch 0 + load_conv3d(state_dict, name_pt + '.branch_0', sess, + os.path.join(name_tf, 'Branch_0/Conv3d_0a_1x1')) + + # Branch .1 + load_conv3d(state_dict, name_pt + '.branch_1.0', sess, + os.path.join(name_tf, 'Branch_1/Conv3d_0a_1x1')) + load_conv3d(state_dict, name_pt + '.branch_1.1', sess, + os.path.join(name_tf, 'Branch_1/Conv3d_0b_3x3')) + + # Branch 2 + load_conv3d(state_dict, name_pt + '.branch_2.0', sess, + os.path.join(name_tf, 'Branch_2/Conv3d_0a_1x1')) + if fix_typo: + load_conv3d(state_dict, name_pt + '.branch_2.1', sess, + os.path.join(name_tf, 'Branch_2/Conv3d_0a_3x3')) + else: + load_conv3d(state_dict, name_pt + '.branch_2.1', sess, + os.path.join(name_tf, 'Branch_2/Conv3d_0b_3x3')) + + # Branch 3 + load_conv3d(state_dict, name_pt + '.branch_3.1', sess, + os.path.join(name_tf, 'Branch_3/Conv3d_0b_1x1')) diff --git a/model_def/tsm.py b/model_def/tsm.py new file mode 100644 index 0000000..6fd637e --- /dev/null +++ b/model_def/tsm.py @@ -0,0 +1,31 @@ +import torch +from torch import nn + +class tsm (nn.Module): + def __init__ (self, num_classes, segment_count, pretrained): + super(tsm, self).__init__() + self.pretrained = pretrained + self.repo = 'epic-kitchens/action-models' + if 'epic-kitchens' in self.pretrained: + all_classes_num = (125, 352) + self.model = torch.hub.load(self.repo, 'TSM', all_classes_num, segment_count, 'RGB', + base_model='resnet50', pretrained='epic-kitchens') + elif 'kinetics' in self.pretrained: + kinetics_classes_num = 400 + self.model = torch.hub.load(self.repo, 'TSM', kinetics_classes_num, segment_count, 'RGB', + base_model='resnet50') + checkpoint_path = '/home/acb11711tx/lzq/ModelVisualization/model_param/TSM_kinetics_RGB_resnet50_shift8_blockres_avg_segment8_e100_dense.pth' + state_dict = torch.load(checkpoint_path)['state_dict'] + state_dict = {k[7:]: v for k, v in state_dict.items()} + self.model.load_state_dict(state_dict) + + def forward (self, inp): + if 'epic-kitchens' in self.pretrained: + feat = self.model.features(inp) + verb_logits, noun_logits = self.model.logits(feat) + if 'noun' in self.pretrained: + return noun_logits + elif 'verb' in self.pretrained: + return verb_logits + elif 'kinetics' in self.pretrained: + return self.model(inp) \ No newline at end of file diff --git a/path_dict.py b/path_dict.py new file mode 100644 index 0000000..43f8c6b --- /dev/null +++ b/path_dict.py @@ -0,0 +1,14 @@ +import socket +import os + +class PathDict (object): + def __init__ (self): + hostname = socket.gethostname() + if 'abci' in hostname: + self.surgery_ds_root = '/groups1/gcb50205/lzq/dataset/JIGSAWS' + self.proj_root = os.path.dirname(os.path.abspath(__file__)) + print(self.proj_root) + else: + # self.surgery_ds_root = '/home/shinkyo/dataset/JIGSAWS' + # self.proj_root = '/home/shinkyo/lzq/ModelVisualization' + self.proj_root = os.path.dirname(os.path.abspath(__file__)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2545b3f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +matplotlib==2.2.3 +opencv_python==4.1.2.30 +torchvision==0.4.0a0 +torchray==1.0.0.2 +tqdm==4.45.0 +pandas==0.23.3 +scikit_image==0.15.0 +numpy==1.14.3 +torch==1.2.0 +Pillow==7.1.2 +scikit_learn==0.22.2.post1 +skimage==0.0 diff --git a/test_data/epic-kitchens-noun/EPIC_noun_classes.csv b/test_data/epic-kitchens-noun/EPIC_noun_classes.csv new file mode 100644 index 0000000..9be7c63 --- /dev/null +++ b/test_data/epic-kitchens-noun/EPIC_noun_classes.csv @@ -0,0 +1,353 @@ +noun_id,class_key,nouns +0,Nothing,['Nothing'] +1,pan,"['pan', 'pan:sauce', 'pan:frying', 'pan:cake', 'saucepan', 'saucepan:empty', 'saucepan:small', 'wok', 'wok:scour', 'pan:content']" +2,pan:dust,"['pan:dust', 'dustpan']" +3,tap,"['tap', 'tap:water', 'nozzle', 'nozzle:coffee']" +4,plate,"['plate', 'plate:pasta', 'saucer', 'dish', 'plate:salad', 'plate:lasagna', 'food:plate', 'flatware', 'pate']" +5,knife,"['knife', 'knife:mincing', 'knife:using', 'blade', 'chopper', 'knife:mezzaluna', 'chopper:moon', 'mezzaluna']" +6,bowl,"['bowl', 'bowl:salad', 'bowl:glass', 'bowl:sugar', 'bowl:empty', 'bowl:cooker', 'bowl:washing', 'bowl:egg', 'bowl:processor']" +7,spoon,"['spoon', 'spoon:wooden', 'spoon:using', 'teaspoon', 'risotto:spoon', 'spade', 'spade:food', 'spoon:coffee', 'spoon:tea', 'measure', 'stirrer:coffee', 'scoop', 'stirrer', 'tablespoon']" +8,cupboard,"['cupboard', 'cabinet', 'locker', 'flap', 'door', 'door:cabinet', 'door:cupboard', 'closet']" +9,drawer,"['drawer', 'drawer:dishwasher', 'drawer:freezer', 'drawer:refrigerator', 'draw']" +10,fridge,"['fridge', 'refrigerator', 'door:refrigerator', 'door:fridge']" +11,lid,"['lid', 'lid:pot', 'lid:tin', 'lid:saucepan', 'lid:tupperware', 'lid:box', 'lid:pan', 'lid:bottle', 'lid:teapot', 'lid:kettle', 'lid:yoghurt', 'lid:chopper', 'seal']" +12,hand,"['hand', 'hand:using', 'hand:my']" +13,onion,"['onion', 'onion:cutting', 'piece:onion', 'bit:onion']" +14,onion:spring,['onion:spring'] +15,pot,"['pot', 'pot:coffee', 'pot:small', 'pot:white', 'pot:right', 'pot:left', 'pot:teapot', 'pot:second', 'teapot', 'part:teapot', 'pot:pressure', 'pot:pasta', 'pot:sauce', 'pot:third', 'pot:first']" +16,glass,"['glass', 'glass:wine', 'glass:empty']" +17,water,"['water', 'water:tap', 'water:empty', 'water:excess', 'water:hot', 'water:more', 'excess:water']" +18,fork,"['fork', 'fork:soap']" +19,board:chopping,"['board:chopping', 'board:cutting', 'board', 'board:serving', 'block:chopping', 'board:cut', 'block']" +20,bag,"['bag', 'bag:plastic', 'bag:pasta', 'bag:mozzarella', 'bag:cereal', 'bag:grape', 'bag:flour', 'bag:cheese', 'bag:nut', 'bag:rice', 'bag:oregano', 'bag:salad', 'bag:bin', 'bag:coffee', 'bag:spaghetti', 'bag:food', 'bag:trash', 'bag:onion', 'bag:carrot', 'bag:bread', 'bag:grapes', 'bag:sandwich', 'bag:kitchen']" +21,sponge,"['sponge', 'scrubber']" +22,spatula,"['spatula', 'spatula:wooden', 'spate']" +23,cup,"['cup', 'cup:coffee', 'cup:measuring', 'cup:small', 'mug', 'mug:rinse', 'mug:clean']" +24,oil,"['oil', 'oil:olive', 'oil:chilli', 'oil:sesame']" +25,bin,"['bin', 'bin:trash', 'bin:dust', 'bin:food', 'dustbin', 'bin:rubbish', 'bin:garbage', 'bin:recycling', 'basket:waste', 'wastebasket']" +26,meat,"['meat', 'meat:minced', 'piece:meat', 'mix:meat', 'meat:mix', 'mincemeat', 'meat:mince']" +27,potato,"['potato', 'potato:baby']" +28,bottle,"['bottle', 'bottle:oil', 'bottle:milk', 'bottle:water', 'bottle:sauce', 'bottle:pesto', 'bottle:vinegar']" +29,container,"['container', 'container:food', 'container:milk', 'container:sauce', 'container:rice', 'container:tofu', 'container:sugar', 'container:salt', 'container:tomato', 'container:pasta', 'container:cake', 'bin:bread', 'bucket']" +30,tomato,"['tomato', 'tomato:dried', 'tomato:cherry', 'tomato:chopped', 'sun']" +31,salt,"['salt', 'seasoning']" +32,cloth,"['cloth', 'cloth:table', 'rag', 'cloth:dish', 'cloth:cleaning', 'cloth:washing', 'dishrag']" +33,sink,"['sink', 'sink:kitchen', 'sink:water']" +34,door:kitchen,['door:kitchen'] +35,pasta,"['pasta', 'spaghetti', 'piece:pasta', 'lasagne']" +36,dish:soap,['dish:soap'] +37,food,"['food', 'food:scoop', 'residue:food', 'bit:food', 'grocery']" +38,kettle,['kettle'] +39,box,"['box', 'box:lunch', 'box:cereal', 'box:tea', 'box:salt', 'tupperware', 'lunchbox', 'case', 'box:mushroom', 'box:coffee', 'box:cheese', 'box:milk', 'box:rice', 'box:carrot', 'box:fruit', 'pot:plastic', 'tub', 'box:plastic', 'box:eggs', 'box:fajita']" +40,carrot,"['carrot', 'carrot:tail']" +41,sauce,"['sauce', 'sauce:soy', 'sauce:fish', 'sauce:pesto', 'sauce:tomato', 'salsa', 'tabasco', 'sauce:soya', 'soy', 'passata', 'mirin']" +42,colander,"['colander', 'strainer:pasta']" +43,milk,"['milk', 'milk:soy', 'milk:coconut', 'milk:soya']" +44,rice,"['rice', 'rice:scoop']" +45,garlic,"['garlic', 'garlic:chop', 'garlic:peel', 'piece:garlic', 'clove', 'bit:garlic', 'bulb']" +46,pepper,"['pepper', 'pepper:bell']" +47,hob,"['hob', 'stove', 'gas', 'burners', 'hop', 'hot', 'burner']" +48,dough,['dough'] +49,dishwasher,"['dishwasher', 'dishwater']" +50,egg,"['egg', 'flour:egg']" +51,cheese,"['cheese', 'slice:cheese', 'mixture:cheese', 'mozzarella', 'paneer', 'slice:mozzarella', 'parmesan']" +52,bread,"['bread', 'bread:white', 'bread:toast', 'loaf', 'rusk', 'toast', 'bun:hamburger', 'slice:bread', 'ciabatta', 'bread:sandwich']" +53,table,"['table', 'table:round']" +54,salad,"['salad', 'salad:cucumber']" +55,microwave,"['microwave', 'oven:microwave']" +56,oven,"['oven', 'oven:adjust', 'cooker', 'door:oven']" +57,cooker:slow,"['cooker:slow', 'cooker:rice']" +58,coffee,"['coffee', 'coffee:scoop', 'pod:coffee', 'bean:coffee', 'ground:coffee', 'scoop:coffee', 'spill:coffee', 'grounds', 'ground']" +59,filter,"['filter', 'filter:water', 'filter:coffee', 'filter:empty', 'filter:sink', 'paper:filter', 'catcher:sink', 'stud']" +60,jar,"['jar', 'jar:sugar', 'jar:coffee', 'jar:filter', 'jar:salt', 'jar:honey']" +61,rack:drying,"['rack:drying', 'rack', 'rack:dish', 'drainer']" +62,chicken,"['chicken', 'thigh:chicken', 'breast:chicken', 'piece:chicken', 'breast']" +63,tray,"['tray', 'tray:baking', 'pan:baking', 'tray:pizza', 'tin:baking', 'base:tin']" +64,mixture,"['mixture', 'mix']" +65,towel,"['towel', 'towel:tea']" +66,towel:kitchen,"['towel:kitchen', 'towel:paper', 'roll:kitchen']" +67,peach,"['peach', 'slice:peach']" +68,skin,"['skin', 'skin:onion', 'skin:potato', 'peel', 'peel:onion', 'peeling', 'peeling:onion', 'peeling:carrot', 'rind', 'skin:squash', 'skin:carrot', 'skin:banana']" +69,courgette,"['courgette', 'piece:courgette']" +70,liquid:washing,"['liquid:washing', 'liquid:cleaning', 'detergent', 'degreaser', 'cleanser', 'spray', 'cleaner', 'washing:dish', 'product']" +71,liquid,['liquid'] +72,leaf,"['leaf', 'leaf:lettuce', 'leaf:salad', 'leaf:spinach', 'leaf:bay']" +73,lettuce,"['lettuce', 'rocket']" +74,leaf:mint,"['leaf:mint', 'leaf:curry']" +75,cutlery,"['cutlery', 'knife:fork', 'silverware', 'tableware']" +76,scissors,"['scissors', 'scissor']" +77,package,"['package', 'package:cheese', 'package:bread', 'package:cooky', 'package:popcorn', 'package:mozzarella', 'package:parsley', 'package:salad', 'package:pasta', 'packaging', 'packaging:salad', 'packaging:bread', 'packet', 'pack:cheese', 'pack', 'pack:tortilla', 'pack:tea', 'carton:egg', 'wrapper', 'plastic', 'package:sausage', 'package:cookies', 'pack:soup', 'package:bacon', 'pack:oat', 'carton', 'packet:beef', 'packet:tea', 'package:meat', 'sachet', 'packet:cookies', 'packet:cheese', 'carton:milk', 'sachet:flavour', 'packet:sausage', 'package:cream', 'package:salt', 'pack:salad', 'pack:meat', 'pack:pasta', 'package:fajita', 'package:wrap', 'package:spice', 'package:egg', 'cardboard', 'wrapping']" +78,top,"['top', 'top:counter', 'top:stove', 'surface', 'surface:hob', 'surface:wipe', 'counter', 'counter:kitchen', 'side', 'top:kitchen', 'tile']" +79,spice,['spice'] +80,tortilla,['tortilla'] +81,paper,"['paper', 'paper:baking', 'paper:cooking', 'letter']" +82,machine:washing,"['machine:washing', 'machine', 'washer', 'machine:washer', 'door:machine']" +83,olive,['olive'] +84,sausage,"['sausage', 'chorizo']" +85,glove:oven,"['glove:oven', 'glove', 'mitt', 'mitt:oven', 'mitten']" +86,peeler:potato,"['peeler:potato', 'peeler', 'skinner:carrot', 'skinner', 'peeler:carrot']" +87,can,"['can', 'tin', 'tin:tomato']" +88,mat,"['mat', 'mat:place', 'board:eating', 'placemat']" +89,mat:sushi,['mat:sushi'] +90,vegetable,['vegetable'] +91,wrap:plastic,"['wrap:plastic', 'foil:transparent', 'film:cling', 'film']" +92,wrap,['wrap'] +93,flour,"['flour', 'mixture:flour', 'powder:flour']" +94,cucumber,['cucumber'] +95,curry,"['curry', 'curry:fish', 'masala:bean', 'paneer:saag', 'masala']" +96,cereal,"['cereal', 'shreddies', 'granola', 'bowl:cereal']" +97,napkin,['napkin'] +98,soap,['soap'] +99,squash,['squash'] +100,fish,"['fish', 'piece:fish', 'skin:fish', 'fragment:fish', 'skin:salmon']" +101,chilli,"['chilli', 'chilli:chop', 'chilli:red', 'chilli:slice', 'pepper:chilli']" +102,cover,"['cover', 'cover:plastic', 'cover:plate', 'cover:dish']" +103,sugar,['sugar'] +104,aubergine,['aubergine'] +105,jug,"['jug', 'jug:measuring', 'pitcher', 'carafe', 'carafe:rinse']" +106,heat,"['heat', 'temperature', 'temperature:gas', 'temperature:cooker', 'heat:cooker', 'temp']" +107,leek,"['leek', 'piece:leek']" +108,rubbish,"['rubbish', 'trash', 'garbage', 'waste']" +109,ladle,['ladle'] +110,mushroom,['mushroom'] +111,stock,"['stock', 'cube:stock']" +112,freezer,['freezer'] +113,light,['light'] +114,pizza,"['pizza', 'slice:pizza']" +115,ball,['ball'] +116,yoghurt,['yoghurt'] +117,chopstick,"['chopstick', 'stick:chop']" +118,grape,['grape'] +119,ginger,"['ginger', 'piece:ginger', 'galangal', 'root:galangal']" +120,banana,['banana'] +121,oregano,['oregano'] +122,tuna,"['tuna', 'mixture:tuna']" +123,kitchen,['kitchen'] +124,salmon,['salmon'] +125,basket,"['basket', 'hamper']" +126,maker:coffee,"['maker:coffee', 'v60', 'cafetiere', 'cafetiere:rinse', 'maker', 'machine:coffee', 'pot:moka', 'pot:mocha', 'moka']" +127,roll,['roll'] +128,brush,"['brush', 'brusher', 'brush:pastry', 'broom', 'wool', 'wool:steel']" +129,lemon,"['lemon', 'slice:lemon']" +130,clothes,"['clothes', 'laundry', 'washing:washing', 'washing']" +131,grater,"['grater', 'grater:cheese']" +132,strainer,"['strainer', 'strainer:mesh', 'sieve']" +133,bacon,['bacon'] +134,avocado,['avocado'] +135,blueberry,['blueberry'] +136,pesto,['pesto'] +137,utensil,"['utensil', 'skimmer']" +138,bean:green,"['bean:green', 'bean']" +139,floor,['floor'] +140,lime,"['lime', 'piece:lime']" +141,foil,"['foil', 'foil:kitchen', 'foil:aluminium']" +142,grill,['grill'] +143,ingredient,"['ingredient', 'ingredient:pan']" +144,scale,"['scale', 'system:weighing', 'scales', 'balance']" +145,paste:garlic,"['paste:garlic', 'paste:miso', 'paste', 'paste:shrimp', 'paste:pepper', 'paste:nut']" +146,processor:food,"['processor:food', 'processor', 'bit:processor']" +147,nut:pine,"['nut:pine', 'nut', 'peanut', 'pine']" +148,butter,['butter'] +149,butter:peanut,['butter:peanut'] +150,shelf,"['shelf', 'shelf:utensil']" +151,timer,"['timer', 'timer:oven']" +152,rinse,['rinse'] +153,tablecloth,['tablecloth'] +154,switch,['switch'] +155,powder:coconut,"['powder:coconut', 'powder:curry', 'powder:baking', 'powder', 'powder:garlic']" +156,powder:washing,"['powder:washing', 'persil']" +157,capsule,"['capsule', 'capsule:empty']" +158,oat,['oat'] +159,tofu,"['tofu', 'curd:bean', 'curd']" +160,lighter,['lighter'] +161,corn,"['corn', 'corn:sweet', 'cob:corn', 'cob', 'corncob', 'sweetcorn']" +162,vinegar,"['vinegar', 'reduction:balsamic', 'vinegar:wine', 'vinegar:rice', 'reduction']" +163,grinder,"['grinder', 'grinder:coffee']" +164,cap,"['cap', 'cap:sink']" +165,support,"['support', 'support:pan', 'trivet']" +166,cream,"['cream', 'fraiche:creme', 'fraiche']" +167,content,"['content', 'content:pan']" +168,tongs,"['tongs', 'tong']" +169,pie,"['pie', 'pastry']" +170,fan:extractor,"['fan:extractor', 'fan', 'ventilator', 'hood']" +171,raisin,['raisin'] +172,toaster,['toaster'] +173,broccoli,['broccoli'] +174,pin:rolling,"['pin:rolling', 'pin']" +175,plug,"['plug', 'socket']" +176,button,['button'] +177,tea,"['tea', 'bag:tea', 'teabag']" +178,parsley,['parsley'] +179,flame,['flame'] +180,herb,['herb'] +181,base,"['base', 'base:pizza']" +182,holder:filter,"['holder:filter', 'holder:pot', 'holder', 'holder:jug']" +183,thyme,['thyme'] +184,honey,['honey'] +185,celery,['celery'] +186,kiwi,['kiwi'] +187,tissue,['tissue'] +188,time,"['time', 'time:cooking']" +189,clip,"['clip', 'clip:bag']" +190,noodle,['noodle'] +191,yeast,['yeast'] +192,hummus,"['hummus', 'humus']" +193,coconut,['coconut'] +194,cabbage,['cabbage'] +195,spinach,['spinach'] +196,nutella,['nutella'] +197,fruit,['fruit'] +198,dressing:salad,"['dressing:salad', 'dressing']" +199,omelette,"['omelette', 'tamagoyaki', 'frittata']" +200,kale,['kale'] +201,paella,['paella'] +202,chip,['chip'] +203,opener:bottle,"['opener:bottle', 'opener:can', 'corkscrew', 'opener']" +204,shirt,"['shirt', 't', 'shirt:tee']" +205,chair,['chair'] +206,sandwich,['sandwich'] +207,burger:tuna,"['burger:tuna', 'patty', 'patty:tuna', 'burger', 'hamburger']" +208,pancake,['pancake'] +209,leftover,"['leftover', 'food:leftover']" +210,risotto,['risotto'] +211,pestle,['pestle'] +212,sock,['sock'] +213,pea,['pea'] +214,apron,['apron'] +215,juice,"['juice', 'juice:lemon']" +216,wine,['wine'] +217,dust,['dust'] +218,desk,['desk'] +219,mesh,['mesh'] +220,oatmeal,['oatmeal'] +221,artichoke,['artichoke'] +222,remover:spot,['remover:spot'] +223,coriander,"['coriander', 'cilantro']" +224,mocha,['mocha'] +225,quorn,['quorn'] +226,soup,['soup'] +227,turmeric,['turmeric'] +228,knob,"['knob', 'dial']" +229,seed,"['seed', 'pit', 'seed:squash']" +230,boxer,"['boxer', 'short:boxer']" +231,paprika,['paprika'] +232,juicer:lime,['juicer:lime'] +233,guard:hand,"['guard:hand', 'guard']" +234,apple,['apple'] +235,tahini,['tahini'] +236,finger,['finger'] +237,salami,"['salami', 'ham', 'prosciutto']" +238,mayonnaise,['mayonnaise'] +239,biscuit,"['biscuit', 'cookie', 'cooky', 'cracker']" +240,pear,['pear'] +241,mortar,"['mortar', 'bowl:mortar']" +242,berry,['berry'] +243,beef,"['beef', 'steak']" +244,squeezer:lime,['squeezer:lime'] +245,tail,['tail'] +246,stick:crab,['stick:crab'] +247,supplement,['supplement'] +248,phone,"['phone', 'mobile']" +249,shell:egg,"['shell:egg', 'eggshell', 'shell']" +250,pith,['pith'] +251,ring:onion,"['ring:onion', 'ring']" +252,cherry,['cherry'] +253,cake,['cake'] +254,sprout,['sprout'] +255,almond,['almond'] +256,mint,['mint'] +257,flake:chilli,"['flake:chilli', 'flake']" +258,cutter:pizza,"['cutter:pizza', 'cutter']" +259,nesquik,['nesquik'] +260,blender,"['blender', 'mixer']" +261,scrap,['scrap'] +262,backpack,['backpack'] +263,melon,['melon'] +264,breadcrumb,"['breadcrumb', 'crumb:bread']" +265,sticker,['sticker'] +266,shrimp,['shrimp'] +267,smoothie,['smoothie'] +268,grass:lemon,['grass:lemon'] +269,ketchup,['ketchup'] +270,slicer,"['slicer', 'device:cutting']" +271,stand,['stand'] +272,dumpling,['dumpling'] +273,watch,['watch'] +274,beer,['beer'] +275,power,['power'] +276,heater,['heater'] +277,basil,"['basil', 'plant:basil']" +278,cinnamon,['cinnamon'] +279,crisp,['crisp'] +280,asparagus,['asparagus'] +281,drink,"['drink', 'soda']" +282,fishcakes,"['fishcakes', 'cake:fish', 'fishcake']" +283,mustard,['mustard'] +284,caper,['caper'] +285,whetstone,"['whetstone', 'sharpener:knife', 'sharpener', 'stone:sharpening', 'stone']" +286,candle,['candle'] +287,control:remote,"['control:remote', 'control', 'remote']" +288,instruction,['instruction'] +289,cork,"['cork', 'stopper']" +290,tab,['tab'] +291,masher,['masher'] +292,part,['part'] +293,muffin,['muffin'] +294,shaker:pepper,['shaker:pepper'] +295,garni:bouquet,['garni:bouquet'] +296,popcorn,['popcorn'] +297,envelope,['envelope'] +298,chocolate,['chocolate'] +299,spot,['spot'] +300,window,['window'] +301,syrup,"['syrup', 'syrup:maple']" +302,bar:cereal,"['bar:cereal', 'bar']" +303,croissant,['croissant'] +304,coke,['coke'] +305,stereo,['stereo'] +306,alarm,['alarm'] +307,recipe,['recipe'] +308,handle,['handle'] +309,sleeve,['sleeve'] +310,cumin,"['cumin', 'jeera']" +311,wire,['wire'] +312,label,['label'] +313,fire,['fire'] +314,presser,"['presser', 'squeezer', 'press', 'press:garlic', 'juicer']" +315,air,['air'] +316,mouse,['mouse'] +317,boiler,['boiler'] +318,rest,"['rest', 'rest:spoon', 'holder:spoon']" +319,tablet,['tablet'] +320,poster,['poster'] +321,trousers,['trousers'] +322,form,['form'] +323,rubber,"['rubber', 'band:rubber']" +324,rug,['rug'] +325,sheets,"['sheets', 'sheet']" +326,pepper:cayenne,['pepper:cayenne'] +327,waffle,['waffle'] +328,pineapple,['pineapple'] +329,turkey,['turkey'] +330,alcohol,['alcohol'] +331,rosemary,['rosemary'] +332,lead,['lead'] +333,book,"['book', 'geographic:national']" +334,rim,['rim'] +335,gravy,['gravy'] +336,straw,['straw'] +337,hat,['hat'] +338,cd,['cd'] +339,slipper,['slipper'] +340,casserole,['casserole'] +341,ladder,['ladder'] +342,jambalaya,['jambalaya'] +343,wall,['wall'] +344,tube,['tube'] +345,lamp,['lamp'] +346,tarragon,['tarragon'] +347,heart,['heart'] +348,funnel,['funnel'] +349,whisk,['whisk'] +350,driver:screw,['driver:screw'] +351,trouser,['trouser'] diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000001.png new file mode 100644 index 0000000..727fb13 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000002.png new file mode 100644 index 0000000..34f7d5b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000003.png new file mode 100644 index 0000000..db19c6b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000004.png new file mode 100644 index 0000000..7ed091c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000005.png new file mode 100644 index 0000000..45ffd10 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000006.png new file mode 100644 index 0000000..ba9c0f2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000007.png new file mode 100644 index 0000000..97c69cf Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000008.png new file mode 100644 index 0000000..feca898 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000009.png new file mode 100644 index 0000000..8082397 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000010.png new file mode 100644 index 0000000..36261d4 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000011.png new file mode 100644 index 0000000..c224b71 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000012.png new file mode 100644 index 0000000..6626fc6 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000013.png new file mode 100644 index 0000000..1ba0e12 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000014.png new file mode 100644 index 0000000..e4d6c98 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000015.png new file mode 100644 index 0000000..2467d31 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000016.png new file mode 100644 index 0000000..9c4e7db Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cloth-take-P12_06_18977/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000001.png new file mode 100644 index 0000000..c2103b2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000002.png new file mode 100644 index 0000000..57c70fd Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000003.png new file mode 100644 index 0000000..3229398 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000004.png new file mode 100644 index 0000000..45bc937 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000005.png new file mode 100644 index 0000000..0939e68 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000006.png new file mode 100644 index 0000000..cea52ab Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000007.png new file mode 100644 index 0000000..d710fb6 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000008.png new file mode 100644 index 0000000..2ee937b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000009.png new file mode 100644 index 0000000..6b15a3f Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000010.png new file mode 100644 index 0000000..82eaf01 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000011.png new file mode 100644 index 0000000..d8c8aaf Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000012.png new file mode 100644 index 0000000..9e179fb Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000013.png new file mode 100644 index 0000000..fe9d762 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000014.png new file mode 100644 index 0000000..cc90830 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000015.png new file mode 100644 index 0000000..5aa576c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000016.png new file mode 100644 index 0000000..b67ce26 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P03_05_6777/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000001.png new file mode 100644 index 0000000..0dc28ab Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000002.png new file mode 100644 index 0000000..d4f5fe2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000003.png new file mode 100644 index 0000000..1803e6b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000004.png new file mode 100644 index 0000000..3a66539 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000005.png new file mode 100644 index 0000000..bec3d22 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000006.png new file mode 100644 index 0000000..f559795 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000007.png new file mode 100644 index 0000000..67e8af4 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000008.png new file mode 100644 index 0000000..8ce7274 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000009.png new file mode 100644 index 0000000..543ab66 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000010.png new file mode 100644 index 0000000..3965d1d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000011.png new file mode 100644 index 0000000..87e933a Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000012.png new file mode 100644 index 0000000..2fd00a8 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000013.png new file mode 100644 index 0000000..12c7ad9 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000014.png new file mode 100644 index 0000000..fa148e6 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000015.png new file mode 100644 index 0000000..b01bb80 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000016.png new file mode 100644 index 0000000..59d9967 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/cupboard-open-P30_06_37623/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000001.png new file mode 100644 index 0000000..d8bbe65 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000002.png new file mode 100644 index 0000000..8593f70 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000003.png new file mode 100644 index 0000000..e17dd07 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000004.png new file mode 100644 index 0000000..c60eb82 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000005.png new file mode 100644 index 0000000..7770efe Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000006.png new file mode 100644 index 0000000..6442364 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000007.png new file mode 100644 index 0000000..9f8ba81 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000008.png new file mode 100644 index 0000000..705a2df Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000009.png new file mode 100644 index 0000000..69ad206 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000010.png new file mode 100644 index 0000000..dbe4e0c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000011.png new file mode 100644 index 0000000..98ac6b3 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000012.png new file mode 100644 index 0000000..73937e5 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000013.png new file mode 100644 index 0000000..c7f0115 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000014.png new file mode 100644 index 0000000..30fcd88 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000015.png new file mode 100644 index 0000000..1e3f2d2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000016.png new file mode 100644 index 0000000..b04bfc0 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/food-stir-P03_05_6751/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000001.png new file mode 100644 index 0000000..0f9c443 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000002.png new file mode 100644 index 0000000..07af028 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000003.png new file mode 100644 index 0000000..f066079 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000004.png new file mode 100644 index 0000000..ca1d926 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000005.png new file mode 100644 index 0000000..34319fe Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000006.png new file mode 100644 index 0000000..eea1e63 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000007.png new file mode 100644 index 0000000..0ebbe45 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000008.png new file mode 100644 index 0000000..fb24913 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000009.png new file mode 100644 index 0000000..4de0737 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000010.png new file mode 100644 index 0000000..9f47783 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000011.png new file mode 100644 index 0000000..0023240 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000012.png new file mode 100644 index 0000000..9447fb9 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000013.png new file mode 100644 index 0000000..9fb7826 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000014.png new file mode 100644 index 0000000..323981c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000015.png new file mode 100644 index 0000000..ca60d4c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000016.png new file mode 100644 index 0000000..8184cd9 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/lid-put-P17_03_20421/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000001.png new file mode 100644 index 0000000..ef0a73c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000002.png new file mode 100644 index 0000000..deed2c2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000003.png new file mode 100644 index 0000000..9481098 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000004.png new file mode 100644 index 0000000..877d7ec Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000005.png new file mode 100644 index 0000000..911f141 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000006.png new file mode 100644 index 0000000..8bedf7b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000007.png new file mode 100644 index 0000000..2981877 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000008.png new file mode 100644 index 0000000..2481097 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000009.png new file mode 100644 index 0000000..2bc3e0b Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000010.png new file mode 100644 index 0000000..184c005 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000011.png new file mode 100644 index 0000000..5a90f1d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000012.png new file mode 100644 index 0000000..c19113c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000013.png new file mode 100644 index 0000000..a36ccae Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000014.png new file mode 100644 index 0000000..9915758 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000015.png new file mode 100644 index 0000000..6fd411f Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000016.png new file mode 100644 index 0000000..fab449a Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/package-tear-P17_03_20411/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000001.png new file mode 100644 index 0000000..ed6dddf Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000002.png new file mode 100644 index 0000000..f490d96 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000003.png new file mode 100644 index 0000000..4d5aafa Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000004.png new file mode 100644 index 0000000..259960f Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000005.png new file mode 100644 index 0000000..819c52d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000006.png new file mode 100644 index 0000000..8aa03b2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000007.png new file mode 100644 index 0000000..fb3af8a Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000008.png new file mode 100644 index 0000000..2187265 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000009.png new file mode 100644 index 0000000..31296b6 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000010.png new file mode 100644 index 0000000..9786f7f Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000011.png new file mode 100644 index 0000000..021a94d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000012.png new file mode 100644 index 0000000..aabc3ef Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000013.png new file mode 100644 index 0000000..5684f3d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000014.png new file mode 100644 index 0000000..4f3218e Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000015.png new file mode 100644 index 0000000..7a84e56 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000016.png new file mode 100644 index 0000000..c4c0f8a Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/pan-take-P26_21_32526/000000016.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000001.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000001.png new file mode 100644 index 0000000..539908c Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000001.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000002.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000002.png new file mode 100644 index 0000000..d13e584 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000002.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000003.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000003.png new file mode 100644 index 0000000..477615e Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000003.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000004.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000004.png new file mode 100644 index 0000000..97c404a Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000004.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000005.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000005.png new file mode 100644 index 0000000..b7c47ac Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000005.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000006.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000006.png new file mode 100644 index 0000000..61c0318 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000006.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000007.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000007.png new file mode 100644 index 0000000..23bbeab Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000007.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000008.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000008.png new file mode 100644 index 0000000..5e0ec32 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000008.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000009.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000009.png new file mode 100644 index 0000000..22319b4 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000009.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000010.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000010.png new file mode 100644 index 0000000..21b4e30 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000010.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000011.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000011.png new file mode 100644 index 0000000..ded4bfb Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000011.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000012.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000012.png new file mode 100644 index 0000000..1fb1de2 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000012.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000013.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000013.png new file mode 100644 index 0000000..13ed40d Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000013.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000014.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000014.png new file mode 100644 index 0000000..d57ef37 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000014.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000015.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000015.png new file mode 100644 index 0000000..c9c47b0 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000015.png differ diff --git a/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000016.png b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000016.png new file mode 100644 index 0000000..e363543 Binary files /dev/null and b/test_data/epic-kitchens-noun/sampled_frames/spoon-put-P30_06_37608/000000016.png differ diff --git a/test_data/epic-kitchens-verb/EPIC_verb_classes.csv b/test_data/epic-kitchens-verb/EPIC_verb_classes.csv new file mode 100644 index 0000000..eda5d9b --- /dev/null +++ b/test_data/epic-kitchens-verb/EPIC_verb_classes.csv @@ -0,0 +1,126 @@ +verb_id,class_key,verbs +0,take,"['take', 'grab', 'pick', 'draw', 'get', 'grab-up', 'collect-from', 'take-up', 'grab-down', 'pull-down', 'fetch', 'pick-up']" +1,put,"['put', 'pose', 'put-away', 'place-that', 'place-down', 'put', 'set-down', 'put-over', 'place-on', 'reture', 'put-on', 'place-away', 'swap', 'layer', 'position', 'put-down', 'put-back', 'place-back', 'return', 'pace', 'dose', 'put-up', 'leave', 'place', 'put-aside', 'lay', 'reposition', 'replace', 'put-onto']" +2,open,"['open', 'unzip', 'open-up']" +3,close,"['close', 'close-off', 'shut']" +4,wash,"['wash', 'sponge', 'lather', 'wash-with', 'rinse', 'rinse-off', 'soap-up', 'wash-off', 'soap', 'rise', 'wash-up', 'clean-around', 'clean-off', 'clean-with', 'clean', 'clean-up', 'wipe-down', 'wipe-of', 'wipe', 'wipe-off']" +5,cut,"['cut', 'chop', 'chop-off', 'cut-off', 'slice-into', 'slice-along', 'slice-off', 'stem', 'slice-up', 'cut-into', 'dice', 'half', 'halve', 'chop-up', 'snip', 'trim', 'slice']" +6,mix,"['mix', 'beat', 'mix-around', 'stir-with', 'whisk', 'stir', 'blend', 'mix-in', 'stir-in']" +7,pour,"['pour', 'pour-in', 'tip-in', 'pour-out', 'pour-on', 'pour-into', 'sieve']" +8,throw,"['throw', 'throw-out', 'recycle', 'dispose-of', 'throw-over', 'throw-away', 'throw-in', 'bin', 'trow', 'toss']" +9,move,"['move', 'transfer', 'move-around']" +10,remove,"['remove', 'extract', 'take-off', 'remove-out', 'take-out', 'get-out', 'remove-inside', 'remove-from', 'pick-out']" +11,dry,"['dry', 'dry-off', 'towel']" +12,turn-on,"['turn-on', 'start', 'begin', 'ignite', 'switch-on', 'activate', 'water-on', 'play', 'start-to', 'restart', 'light']" +13,turn,"['turn', 'rotate']" +14,shake,"['shake', 'shake-off', 'shake-out']" +15,turn-off,"['turn-off', 'switch-of', 'water-off', 'switch-off', 'switch-out', 'shut-off', 'turn-of']" +16,peel,"['peel', 'skin-from', 'skin', 'peel-off', 'peel-back']" +17,adjust,"['adjust', 'change', 'regulate']" +18,empty,['empty'] +19,scoop,"['scoop', 'spoon-in', 'spoon', 'scoop-out', 'scoop-up']" +20,check,"['check', 'ensure', 'test', 'look-in', 'watch', 'inspect', 'check-on']" +21,squeeze,"['squeeze', 'squidge', 'squidge-into', 'squash', 'squish-into', 'wring-out', 'wring', 'squish', 'squeeze-into']" +22,insert,"['insert', 'put-in', 'put-into', 'fit', 'place-in', 'put-inside']" +23,press,"['press', 'push-down', 'collapse', 'compress', 'push', 'press-on']" +24,fill,"['fill', 'fill-with', 'fill-up', 'stuff']" +25,add,"['add', 'combine', 'add-to']" +26,scrape,"['scrape', 'scrape-out', 'loose', 'scour', 'scrap', 'scrape-off']" +27,sharpen,"['sharpen', 'thin']" +28,wrap,"['wrap', 'top', 'lid', 'cover', 'seal', 'clamp', 'fasten', 'reseal', 'clip', 'wrap-up', 'tie']" +29,roll,"['roll', 'roll-up']" +30,sprinkle,"['sprinkle', 'sprinkle-on', 'drizzle', 'scatter', 'sprincle', 'crumble']" +31,break,"['break', 'snap', 'crack', 'break-up']" +32,flip,"['flip', 'overturn', 'turn-over']" +33,hang,"['hang', 'drape', 'hang-up']" +34,hold,['hold'] +35,sort,"['sort', 'rearrange', 'arrange', 'clear', 'tidy', 'line-up']" +36,apply,"['apply', 'spread']" +37,crush,"['crush', 'hit', 'grind', 'grate', 'mash', 'crush']" +38,search,"['search', 'search-for', 'search-in', 'look-for', 'find', 'locate']" +39,sample,"['sample', 'taste', 'smell', 'lick']" +40,knead,['knead'] +41,set,"['set', 'set-up', 'set-out']" +42,walk,"['walk', 'enter', 'enter-into', 'walk-into', 'walk-around', 'walk-to', 'walk-with']" +43,divide,"['divide', 'split', 'detach', 'separate', 'distribute']" +44,spray,"['spray', 'spay']" +45,use,"['use', 'used-to', 'used']" +46,fold,['fold'] +47,cook,"['cook', 'toast', 'fry-in', 'heat', 'fry', 'reduce', 'boil']" +48,filter,"['filter', 'strain', 'drain', 'dump-out']" +49,scrub,"['scrub', 'scrub-inside']" +50,look,"['look', 'stare-at', 'see', 'read-on', 'read', 'look-at']" +51,finish,"['finish', 'stop', 'end', 'do']" +52,soak,"['soak', 'submerge', 'immerse']" +53,brush,['brush'] +54,pull,"['pull', 'pull-out']" +55,pat,"['pat', 'pat-into', 'dab', 'tap', 'tap-on', 'poke', 'pat-down']" +56,form,"['form', 'shape', 'forge', 'make', 'shape-into']" +57,measure,['measure'] +58,drink,"['drink', 'drink-from']" +59,choose,"['choose', 'select']" +60,serve,"['serve', 'plate-on', 'plate', 'dish', 'plate-up']" +61,drop,"['drop', 'drop-on']" +62,wear,['wear'] +63,rip,"['rip', 'tear', 'tear-off', 'tear-down']" +64,tip,"['tip', 'tip-over', 'tip-out']" +65,turn-down,['turn-down'] +66,gather,"['gather', 'collect']" +67,eat,"['eat', 'chew-on', 'bite']" +68,stack,"['stack', 'stack-up']" +69,store,['store'] +70,switch,['switch'] +71,increase,"['increase', 'switch-up', 'turn-up']" +72,carry,"['carry', 'have', 'bring']" +73,lift,"['lift', 'raise', 'tilt', 'lift-up']" +74,twist,"['twist', 'screw', 'screw-in', 'screw-on', 'tighten']" +75,sweep,['sweep'] +76,rub,"['rub', 'rub-off']" +77,unwrap,"['unwrap', 'unpack', 'unseal']" +78,stab,['stab'] +79,attach,"['attach', 'connect', 'plug', 'assemble', 'plug-into', 'plug-in']" +80,stretch,"['stretch', 'unfold']" +81,lower,['lower'] +82,prepare,['prepare'] +83,unscrew,"['unscrew', 'untwist']" +84,season,"['season', 'sweeten', 'pepper', 'salt']" +85,video,['video'] +86,tap-off,['tap-off'] +87,set-off,['set-off'] +88,squirt,['squirt'] +89,load,['load'] +90,unroll,"['unroll', 'roll-out']" +91,water,"['water', 'wet']" +92,do,['do'] +93,flatten,"['flatten', 'flatten-with']" +94,uncover,"['uncover', 'lid-off']" +95,slide,"['slide', 'slide-out']" +96,unplug,['unplug'] +97,level,['level'] +98,tear-out,['tear-out'] +99,feel,"['feel', 'touch']" +100,fix,['fix'] +101,spill,['spill'] +102,pack,['pack'] +103,bake,['bake'] +104,blow,"['blow', 'blow-out']" +105,sit-on,['sit-on'] +106,count,['count'] +107,dip,['dip'] +108,cool,['cool'] +109,flush,['flush'] +110,knife,['knife'] +111,fork,['fork'] +112,swirl,['swirl'] +113,stick,['stick'] +114,pet-down,['pet-down'] +115,realize,['realize'] +116,weigh,['weigh'] +117,defoliate,['defoliate'] +118,deseed,['deseed'] +119,tessellate,['tessellate'] +120,unfreeze,['unfreeze'] +121,decide-if,['decide-if'] +122,let-out,['let-out'] +123,save,['save'] +124,reverse,['reverse'] diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000001.png new file mode 100644 index 0000000..c2103b2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000002.png new file mode 100644 index 0000000..57c70fd Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000003.png new file mode 100644 index 0000000..3229398 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000004.png new file mode 100644 index 0000000..45bc937 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000005.png new file mode 100644 index 0000000..0939e68 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000006.png new file mode 100644 index 0000000..cea52ab Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000007.png new file mode 100644 index 0000000..d710fb6 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000008.png new file mode 100644 index 0000000..2ee937b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000009.png new file mode 100644 index 0000000..6b15a3f Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000010.png new file mode 100644 index 0000000..82eaf01 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000011.png new file mode 100644 index 0000000..d8c8aaf Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000012.png new file mode 100644 index 0000000..9e179fb Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000013.png new file mode 100644 index 0000000..fe9d762 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000014.png new file mode 100644 index 0000000..cc90830 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000015.png new file mode 100644 index 0000000..5aa576c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000016.png new file mode 100644 index 0000000..b67ce26 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P03_05_6777/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000001.png new file mode 100644 index 0000000..0dc28ab Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000002.png new file mode 100644 index 0000000..d4f5fe2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000003.png new file mode 100644 index 0000000..1803e6b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000004.png new file mode 100644 index 0000000..3a66539 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000005.png new file mode 100644 index 0000000..bec3d22 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000006.png new file mode 100644 index 0000000..f559795 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000007.png new file mode 100644 index 0000000..67e8af4 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000008.png new file mode 100644 index 0000000..8ce7274 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000009.png new file mode 100644 index 0000000..543ab66 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000010.png new file mode 100644 index 0000000..3965d1d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000011.png new file mode 100644 index 0000000..87e933a Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000012.png new file mode 100644 index 0000000..2fd00a8 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000013.png new file mode 100644 index 0000000..12c7ad9 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000014.png new file mode 100644 index 0000000..fa148e6 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000015.png new file mode 100644 index 0000000..b01bb80 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000016.png new file mode 100644 index 0000000..59d9967 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/open-cupboard-P30_06_37623/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000001.png new file mode 100644 index 0000000..0f9c443 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000002.png new file mode 100644 index 0000000..07af028 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000003.png new file mode 100644 index 0000000..f066079 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000004.png new file mode 100644 index 0000000..ca1d926 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000005.png new file mode 100644 index 0000000..34319fe Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000006.png new file mode 100644 index 0000000..eea1e63 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000007.png new file mode 100644 index 0000000..0ebbe45 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000008.png new file mode 100644 index 0000000..fb24913 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000009.png new file mode 100644 index 0000000..4de0737 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000010.png new file mode 100644 index 0000000..9f47783 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000011.png new file mode 100644 index 0000000..0023240 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000012.png new file mode 100644 index 0000000..9447fb9 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000013.png new file mode 100644 index 0000000..9fb7826 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000014.png new file mode 100644 index 0000000..323981c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000015.png new file mode 100644 index 0000000..ca60d4c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000016.png new file mode 100644 index 0000000..8184cd9 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-lid-P17_03_20421/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000001.png new file mode 100644 index 0000000..539908c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000002.png new file mode 100644 index 0000000..d13e584 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000003.png new file mode 100644 index 0000000..477615e Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000004.png new file mode 100644 index 0000000..97c404a Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000005.png new file mode 100644 index 0000000..b7c47ac Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000006.png new file mode 100644 index 0000000..61c0318 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000007.png new file mode 100644 index 0000000..23bbeab Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000008.png new file mode 100644 index 0000000..5e0ec32 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000009.png new file mode 100644 index 0000000..22319b4 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000010.png new file mode 100644 index 0000000..21b4e30 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000011.png new file mode 100644 index 0000000..ded4bfb Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000012.png new file mode 100644 index 0000000..1fb1de2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000013.png new file mode 100644 index 0000000..13ed40d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000014.png new file mode 100644 index 0000000..d57ef37 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000015.png new file mode 100644 index 0000000..c9c47b0 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000016.png new file mode 100644 index 0000000..e363543 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/put-spoon-P30_06_37608/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000001.png new file mode 100644 index 0000000..d8bbe65 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000002.png new file mode 100644 index 0000000..8593f70 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000003.png new file mode 100644 index 0000000..e17dd07 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000004.png new file mode 100644 index 0000000..c60eb82 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000005.png new file mode 100644 index 0000000..7770efe Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000006.png new file mode 100644 index 0000000..6442364 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000007.png new file mode 100644 index 0000000..9f8ba81 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000008.png new file mode 100644 index 0000000..705a2df Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000009.png new file mode 100644 index 0000000..69ad206 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000010.png new file mode 100644 index 0000000..dbe4e0c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000011.png new file mode 100644 index 0000000..98ac6b3 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000012.png new file mode 100644 index 0000000..73937e5 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000013.png new file mode 100644 index 0000000..c7f0115 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000014.png new file mode 100644 index 0000000..30fcd88 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000015.png new file mode 100644 index 0000000..1e3f2d2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000016.png new file mode 100644 index 0000000..b04bfc0 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/stir-food-P03_05_6751/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000001.png new file mode 100644 index 0000000..727fb13 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000002.png new file mode 100644 index 0000000..34f7d5b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000003.png new file mode 100644 index 0000000..db19c6b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000004.png new file mode 100644 index 0000000..7ed091c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000005.png new file mode 100644 index 0000000..45ffd10 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000006.png new file mode 100644 index 0000000..ba9c0f2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000007.png new file mode 100644 index 0000000..97c69cf Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000008.png new file mode 100644 index 0000000..feca898 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000009.png new file mode 100644 index 0000000..8082397 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000010.png new file mode 100644 index 0000000..36261d4 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000011.png new file mode 100644 index 0000000..c224b71 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000012.png new file mode 100644 index 0000000..6626fc6 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000013.png new file mode 100644 index 0000000..1ba0e12 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000014.png new file mode 100644 index 0000000..e4d6c98 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000015.png new file mode 100644 index 0000000..2467d31 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000016.png new file mode 100644 index 0000000..9c4e7db Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-cloth-P12_06_18977/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000001.png new file mode 100644 index 0000000..ed6dddf Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000002.png new file mode 100644 index 0000000..f490d96 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000003.png new file mode 100644 index 0000000..4d5aafa Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000004.png new file mode 100644 index 0000000..259960f Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000005.png new file mode 100644 index 0000000..819c52d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000006.png new file mode 100644 index 0000000..8aa03b2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000007.png new file mode 100644 index 0000000..fb3af8a Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000008.png new file mode 100644 index 0000000..2187265 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000009.png new file mode 100644 index 0000000..31296b6 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000010.png new file mode 100644 index 0000000..9786f7f Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000011.png new file mode 100644 index 0000000..021a94d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000012.png new file mode 100644 index 0000000..aabc3ef Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000013.png new file mode 100644 index 0000000..5684f3d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000014.png new file mode 100644 index 0000000..4f3218e Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000015.png new file mode 100644 index 0000000..7a84e56 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000016.png new file mode 100644 index 0000000..c4c0f8a Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/take-pan-P26_21_32526/000000016.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000001.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000001.png new file mode 100644 index 0000000..ef0a73c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000001.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000002.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000002.png new file mode 100644 index 0000000..deed2c2 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000002.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000003.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000003.png new file mode 100644 index 0000000..9481098 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000003.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000004.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000004.png new file mode 100644 index 0000000..877d7ec Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000004.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000005.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000005.png new file mode 100644 index 0000000..911f141 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000005.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000006.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000006.png new file mode 100644 index 0000000..8bedf7b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000006.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000007.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000007.png new file mode 100644 index 0000000..2981877 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000007.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000008.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000008.png new file mode 100644 index 0000000..2481097 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000008.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000009.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000009.png new file mode 100644 index 0000000..2bc3e0b Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000009.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000010.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000010.png new file mode 100644 index 0000000..184c005 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000010.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000011.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000011.png new file mode 100644 index 0000000..5a90f1d Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000011.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000012.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000012.png new file mode 100644 index 0000000..c19113c Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000012.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000013.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000013.png new file mode 100644 index 0000000..a36ccae Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000013.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000014.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000014.png new file mode 100644 index 0000000..9915758 Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000014.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000015.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000015.png new file mode 100644 index 0000000..6fd411f Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000015.png differ diff --git a/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000016.png b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000016.png new file mode 100644 index 0000000..fab449a Binary files /dev/null and b/test_data/epic-kitchens-verb/sampled_frames/tear-package-P17_03_20411/000000016.png differ diff --git a/test_data/frame_process.py b/test_data/frame_process.py new file mode 100644 index 0000000..a097549 --- /dev/null +++ b/test_data/frame_process.py @@ -0,0 +1,50 @@ +import os +from PIL import Image +import cv2 + +# basedir = '/home/acb11711tx/lzq/ModelVisualization/test_data/epic-kitchens-noun/frames' + +# # for video_name in os.listdir(basedir): +# # video_dir = os.path.join(basedir, video_name) +# # frame_names = sorted(os.listdir(video_dir)) +# # for frame_index, frame_name in enumerate(frame_names): +# # frame_index = int(frame_name[6:-4]) +# # os.system(f'mv {video_dir}/{frame_name} {video_dir}/{frame_index+1:09d}.jpg') +# # print(f'{video_name} is finished.') + +# for video_name in os.listdir(basedir): +# video_dir = os.path.join(basedir, video_name) +# frame_names = sorted(os.listdir(video_dir)) +# num_frame = len(frame_names) +# for frame_index in range(num_frame): +# img = Image.open(f'{video_dir}/{frame_index+1:09d}.jpg') +# img.save(f'{video_dir}/{frame_index+1:09d}.png') +# os.system(f'rm {video_dir}/{frame_index+1:09d}.jpg') +# print(f'{video_name} is finished.') + +# import os +import sys +sys.path.append(".") +sys.path.append("..") +from utils.LongRangeSample import long_range_sample + +basedir = '/home/acb11711tx/lzq/VideoVisual/test_data' +dataset_names = ['epic-kitchens-noun', 'epic-kitchens-verb', 'kinetics'] +for dataset in dataset_names: + ds_path = os.path.join(basedir, dataset, 'frames') + video_names = sorted(os.listdir(ds_path)) + for video_name in video_names: + video_dir = os.path.join(ds_path, video_name) + frame_names = sorted(os.listdir(video_dir)) + num_frame = len(frame_names) + sampled_fidxs = long_range_sample(num_frame, 16, 'first') + + new_video_dir = video_dir.replace('frames', 'sampled_frames') + os.makedirs(new_video_dir, exist_ok=True) + for idx, fidx in enumerate(sampled_fidxs): + img = Image.open(f'{video_dir}/{fidx+1:09d}.png') + img = img.resize((344, 256)) + img.save(f'{new_video_dir}/{idx+1:09d}.png') + print(f'{video_dir} finished!') + + \ No newline at end of file diff --git a/test_data/kinetics/classes.json b/test_data/kinetics/classes.json new file mode 100644 index 0000000..47e6bcf --- /dev/null +++ b/test_data/kinetics/classes.json @@ -0,0 +1,402 @@ +[ + "abseiling", + "air drumming", + "answering questions", + "applauding", + "applying cream", + "archery", + "arm wrestling", + "arranging flowers", + "assembling computer", + "auctioning", + "baby waking up", + "baking cookies", + "balloon blowing", + "bandaging", + "barbequing", + "bartending", + "beatboxing", + "bee keeping", + "belly dancing", + "bench pressing", + "bending back", + "bending metal", + "biking through snow", + "blasting sand", + "blowing glass", + "blowing leaves", + "blowing nose", + "blowing out candles", + "bobsledding", + "bookbinding", + "bouncing on trampoline", + "bowling", + "braiding hair", + "breading or breadcrumbing", + "breakdancing", + "brush painting", + "brushing hair", + "brushing teeth", + "building cabinet", + "building shed", + "bungee jumping", + "busking", + "canoeing or kayaking", + "capoeira", + "carrying baby", + "cartwheeling", + "carving pumpkin", + "catching fish", + "catching or throwing baseball", + "catching or throwing frisbee", + "catching or throwing softball", + "celebrating", + "changing oil", + "changing wheel", + "checking tires", + "cheerleading", + "chopping wood", + "clapping", + "clay pottery making", + "clean and jerk", + "cleaning floor", + "cleaning gutters", + "cleaning pool", + "cleaning shoes", + "cleaning toilet", + "cleaning windows", + "climbing a rope", + "climbing ladder", + "climbing tree", + "contact juggling", + "cooking chicken", + "cooking egg", + "cooking on campfire", + "cooking sausages", + "counting money", + "country line dancing", + "cracking neck", + "crawling baby", + "crossing river", + "crying", + "curling hair", + "cutting nails", + "cutting pineapple", + "cutting watermelon", + "dancing ballet", + "dancing charleston", + "dancing gangnam style", + "dancing macarena", + "deadlifting", + "decorating the christmas tree", + "digging", + "dining", + "disc golfing", + "diving cliff", + "dodgeball", + "doing aerobics", + "doing laundry", + "doing nails", + "drawing", + "dribbling basketball", + "drinking", + "drinking beer", + "drinking shots", + "driving car", + "driving tractor", + "drop kicking", + "drumming fingers", + "dunking basketball", + "dying hair", + "eating burger", + "eating cake", + "eating carrots", + "eating chips", + "eating doughnuts", + "eating hotdog", + "eating ice cream", + "eating spaghetti", + "eating watermelon", + "egg hunting", + "exercising arm", + "exercising with an exercise ball", + "extinguishing fire", + "faceplanting", + "feeding birds", + "feeding fish", + "feeding goats", + "filling eyebrows", + "finger snapping", + "fixing hair", + "flipping pancake", + "flying kite", + "folding clothes", + "folding napkins", + "folding paper", + "front raises", + "frying vegetables", + "garbage collecting", + "gargling", + "getting a haircut", + "getting a tattoo", + "giving or receiving award", + "golf chipping", + "golf driving", + "golf putting", + "grinding meat", + "grooming dog", + "grooming horse", + "gymnastics tumbling", + "hammer throw", + "headbanging", + "headbutting", + "high jump", + "high kick", + "hitting baseball", + "hockey stop", + "holding snake", + "hopscotch", + "hoverboarding", + "hugging", + "hula hooping", + "hurdling", + "hurling (sport)", + "ice climbing", + "ice fishing", + "ice skating", + "ironing", + "javelin throw", + "jetskiing", + "jogging", + "juggling balls", + "juggling fire", + "juggling soccer ball", + "jumping into pool", + "jumpstyle dancing", + "kicking field goal", + "kicking soccer ball", + "kissing", + "kitesurfing", + "knitting", + "krumping", + "laughing", + "laying bricks", + "long jump", + "lunge", + "making a cake", + "making a sandwich", + "making bed", + "making jewelry", + "making pizza", + "making snowman", + "making sushi", + "making tea", + "marching", + "massaging back", + "massaging feet", + "massaging legs", + "massaging person's head", + "milking cow", + "mopping floor", + "motorcycling", + "moving furniture", + "mowing lawn", + "news anchoring", + "opening bottle", + "opening present", + "paragliding", + "parasailing", + "parkour", + "passing American football (in game)", + "passing American football (not in game)", + "peeling apples", + "peeling potatoes", + "petting animal (not cat)", + "petting cat", + "picking fruit", + "planting trees", + "plastering", + "playing accordion", + "playing badminton", + "playing bagpipes", + "playing basketball", + "playing bass guitar", + "playing cards", + "playing cello", + "playing chess", + "playing clarinet", + "playing controller", + "playing cricket", + "playing cymbals", + "playing didgeridoo", + "playing drums", + "playing flute", + "playing guitar", + "playing harmonica", + "playing harp", + "playing ice hockey", + "playing keyboard", + "playing kickball", + "playing monopoly", + "playing organ", + "playing paintball", + "playing piano", + "playing poker", + "playing recorder", + "playing saxophone", + "playing squash or racquetball", + "playing tennis", + "playing trombone", + "playing trumpet", + "playing ukulele", + "playing violin", + "playing volleyball", + "playing xylophone", + "pole vault", + "presenting weather forecast", + "pull ups", + "pumping fist", + "pumping gas", + "punching bag", + "punching person (boxing)", + "push up", + "pushing car", + "pushing cart", + "pushing wheelchair", + "reading book", + "reading newspaper", + "recording music", + "riding a bike", + "riding camel", + "riding elephant", + "riding mechanical bull", + "riding mountain bike", + "riding mule", + "riding or walking with horse", + "riding scooter", + "riding unicycle", + "ripping paper", + "robot dancing", + "rock climbing", + "rock scissors paper", + "roller skating", + "running on treadmill", + "sailing", + "salsa dancing", + "sanding floor", + "scrambling eggs", + "scuba diving", + "setting table", + "shaking hands", + "shaking head", + "sharpening knives", + "sharpening pencil", + "shaving head", + "shaving legs", + "shearing sheep", + "shining shoes", + "shooting basketball", + "shooting goal (soccer)", + "shot put", + "shoveling snow", + "shredding paper", + "shuffling cards", + "side kick", + "sign language interpreting", + "singing", + "situp", + "skateboarding", + "ski jumping", + "skiing (not slalom or crosscountry)", + "skiing crosscountry", + "skiing slalom", + "skipping rope", + "skydiving", + "slacklining", + "slapping", + "sled dog racing", + "smoking", + "smoking hookah", + "snatch weight lifting", + "sneezing", + "sniffing", + "snorkeling", + "snowboarding", + "snowkiting", + "snowmobiling", + "somersaulting", + "spinning poi", + "spray painting", + "spraying", + "springboard diving", + "squat", + "sticking tongue out", + "stomping grapes", + "stretching arm", + "stretching leg", + "strumming guitar", + "surfing crowd", + "surfing water", + "sweeping floor", + "swimming backstroke", + "swimming breast stroke", + "swimming butterfly stroke", + "swing dancing", + "swinging legs", + "swinging on something", + "sword fighting", + "tai chi", + "taking a shower", + "tango dancing", + "tap dancing", + "tapping guitar", + "tapping pen", + "tasting beer", + "tasting food", + "testifying", + "texting", + "throwing axe", + "throwing ball", + "throwing discus", + "tickling", + "tobogganing", + "tossing coin", + "tossing salad", + "training dog", + "trapezing", + "trimming or shaving beard", + "trimming trees", + "triple jump", + "tying bow tie", + "tying knot (not on a tie)", + "tying tie", + "unboxing", + "unloading truck", + "using computer", + "using remote controller (not gaming)", + "using segway", + "vault", + "waiting in line", + "walking the dog", + "washing dishes", + "washing feet", + "washing hair", + "washing hands", + "water skiing", + "water sliding", + "watering plants", + "waxing back", + "waxing chest", + "waxing eyebrows", + "waxing legs", + "weaving basket", + "welding", + "whistling", + "windsurfing", + "wrapping present", + "wrestling", + "writing", + "yawning", + "yoga", + "zumba" +] \ No newline at end of file diff --git a/test_data/kinetics/frame_extract.py b/test_data/kinetics/frame_extract.py new file mode 100644 index 0000000..a583904 --- /dev/null +++ b/test_data/kinetics/frame_extract.py @@ -0,0 +1,14 @@ +import os + +basedir = '/home/acb11711tx/lzq/ModelVisualization/test_data/kinetics' +video_names = [name[:-4] for name in os.listdir(basedir) if '.mp4' in name] +for video_name in video_names: + video_name = video_name + savedir = os.path.join(basedir, video_name) + os.makedirs(savedir, exist_ok=True) + + savedir_ = savedir.replace(' ', '\ ') + video_name_ = video_name.replace(' ', '\ ') + print(video_name_) + os.system(f'ffmpeg -i {video_name_}.mp4 -vf fps=10 {video_name_}/%09d.png') + print(f'{video_name} finished!') \ No newline at end of file diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000001.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000001.png new file mode 100644 index 0000000..b300937 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000002.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000002.png new file mode 100644 index 0000000..361bcba Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000003.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000003.png new file mode 100644 index 0000000..787899a Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000004.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000004.png new file mode 100644 index 0000000..35ebb83 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000005.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000005.png new file mode 100644 index 0000000..5cc80f9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000006.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000006.png new file mode 100644 index 0000000..e0590c3 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000007.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000007.png new file mode 100644 index 0000000..a0f8978 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000008.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000008.png new file mode 100644 index 0000000..2e27258 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000009.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000009.png new file mode 100644 index 0000000..a4a6ed2 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000010.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000010.png new file mode 100644 index 0000000..f61395b Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000011.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000011.png new file mode 100644 index 0000000..3ecc8bc Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000012.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000012.png new file mode 100644 index 0000000..fd6a993 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000013.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000013.png new file mode 100644 index 0000000..051eebf Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000014.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000014.png new file mode 100644 index 0000000..1577a50 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000015.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000015.png new file mode 100644 index 0000000..8ad771e Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000016.png b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000016.png new file mode 100644 index 0000000..e4e89d0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-4lFmhj4x43c_66_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000001.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000001.png new file mode 100644 index 0000000..5e95170 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000002.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000002.png new file mode 100644 index 0000000..8b02c18 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000003.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000003.png new file mode 100644 index 0000000..e6a44ee Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000004.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000004.png new file mode 100644 index 0000000..ffa7203 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000005.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000005.png new file mode 100644 index 0000000..82c6dee Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000006.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000006.png new file mode 100644 index 0000000..6dadf78 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000007.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000007.png new file mode 100644 index 0000000..91aa57c Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000008.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000008.png new file mode 100644 index 0000000..233f995 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000009.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000009.png new file mode 100644 index 0000000..5ca2cc7 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000010.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000010.png new file mode 100644 index 0000000..582a41b Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000011.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000011.png new file mode 100644 index 0000000..556269d Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000012.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000012.png new file mode 100644 index 0000000..906b2b3 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000013.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000013.png new file mode 100644 index 0000000..dd8302a Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000014.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000014.png new file mode 100644 index 0000000..7e101d9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000015.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000015.png new file mode 100644 index 0000000..0f6f53d Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000016.png b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000016.png new file mode 100644 index 0000000..0207c97 Binary files /dev/null and b/test_data/kinetics/sampled_frames/abseiling-BkIhp07ge9E_22_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000001.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000001.png new file mode 100644 index 0000000..c4b91ee Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000002.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000002.png new file mode 100644 index 0000000..80ad00e Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000003.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000003.png new file mode 100644 index 0000000..6ee04af Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000004.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000004.png new file mode 100644 index 0000000..929eab4 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000005.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000005.png new file mode 100644 index 0000000..01e3dd9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000006.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000006.png new file mode 100644 index 0000000..efb85de Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000007.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000007.png new file mode 100644 index 0000000..fcb719a Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000008.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000008.png new file mode 100644 index 0000000..9654220 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000009.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000009.png new file mode 100644 index 0000000..a91d3cd Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000010.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000010.png new file mode 100644 index 0000000..5f7f021 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000011.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000011.png new file mode 100644 index 0000000..1a3cd50 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000012.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000012.png new file mode 100644 index 0000000..68decb5 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000013.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000013.png new file mode 100644 index 0000000..0659dd9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000014.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000014.png new file mode 100644 index 0000000..176b977 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000015.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000015.png new file mode 100644 index 0000000..01cc30e Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000016.png b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000016.png new file mode 100644 index 0000000..01a0118 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-3sEk_w2T8Fw_15_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000001.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000001.png new file mode 100644 index 0000000..3d712fa Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000002.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000002.png new file mode 100644 index 0000000..fe6bd95 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000003.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000003.png new file mode 100644 index 0000000..9138994 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000004.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000004.png new file mode 100644 index 0000000..9c40a44 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000005.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000005.png new file mode 100644 index 0000000..83356a8 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000006.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000006.png new file mode 100644 index 0000000..5cff18c Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000007.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000007.png new file mode 100644 index 0000000..cbe813f Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000008.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000008.png new file mode 100644 index 0000000..bc162bb Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000009.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000009.png new file mode 100644 index 0000000..197ced5 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000010.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000010.png new file mode 100644 index 0000000..e5e3cee Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000011.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000011.png new file mode 100644 index 0000000..11df013 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000012.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000012.png new file mode 100644 index 0000000..f08d7f4 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000013.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000013.png new file mode 100644 index 0000000..9ab5653 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000014.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000014.png new file mode 100644 index 0000000..ce5300f Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000015.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000015.png new file mode 100644 index 0000000..60455ca Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000016.png b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000016.png new file mode 100644 index 0000000..f3ecdb2 Binary files /dev/null and b/test_data/kinetics/sampled_frames/ironing-K2COe93hgqM_96_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000001.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000001.png new file mode 100644 index 0000000..e1d8866 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000002.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000002.png new file mode 100644 index 0000000..566cb0b Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000003.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000003.png new file mode 100644 index 0000000..3f109e0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000004.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000004.png new file mode 100644 index 0000000..a5e2582 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000005.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000005.png new file mode 100644 index 0000000..bb92ea9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000006.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000006.png new file mode 100644 index 0000000..5fffc37 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000007.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000007.png new file mode 100644 index 0000000..e27fe26 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000008.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000008.png new file mode 100644 index 0000000..8732e93 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000009.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000009.png new file mode 100644 index 0000000..30fe4fa Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000010.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000010.png new file mode 100644 index 0000000..8f78ef0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000011.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000011.png new file mode 100644 index 0000000..46e7a11 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000012.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000012.png new file mode 100644 index 0000000..b98c9c9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000013.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000013.png new file mode 100644 index 0000000..a9439a9 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000014.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000014.png new file mode 100644 index 0000000..e43690e Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000015.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000015.png new file mode 100644 index 0000000..7de505a Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000016.png b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000016.png new file mode 100644 index 0000000..212bee8 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-JVAwfvQUlwA_46_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000001.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000001.png new file mode 100644 index 0000000..256a67d Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000002.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000002.png new file mode 100644 index 0000000..9a0896c Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000003.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000003.png new file mode 100644 index 0000000..70135f6 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000004.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000004.png new file mode 100644 index 0000000..aec1a90 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000005.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000005.png new file mode 100644 index 0000000..3658850 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000006.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000006.png new file mode 100644 index 0000000..4b43714 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000007.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000007.png new file mode 100644 index 0000000..74db3a2 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000008.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000008.png new file mode 100644 index 0000000..3c50d9c Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000009.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000009.png new file mode 100644 index 0000000..d083fcf Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000010.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000010.png new file mode 100644 index 0000000..b258ed7 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000011.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000011.png new file mode 100644 index 0000000..cde84d1 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000012.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000012.png new file mode 100644 index 0000000..f63547f Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000013.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000013.png new file mode 100644 index 0000000..97266b3 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000014.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000014.png new file mode 100644 index 0000000..bf07003 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000015.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000015.png new file mode 100644 index 0000000..9c28e0f Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000016.png b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000016.png new file mode 100644 index 0000000..6162d07 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing bass guitar-fUbFAXY-vxQ_2_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000001.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000001.png new file mode 100644 index 0000000..1b00e4f Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000002.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000002.png new file mode 100644 index 0000000..572ceea Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000003.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000003.png new file mode 100644 index 0000000..8eaae80 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000004.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000004.png new file mode 100644 index 0000000..aeebc95 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000005.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000005.png new file mode 100644 index 0000000..1eafd89 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000006.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000006.png new file mode 100644 index 0000000..19b95c0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000007.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000007.png new file mode 100644 index 0000000..cd2d8fb Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000008.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000008.png new file mode 100644 index 0000000..deba515 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000009.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000009.png new file mode 100644 index 0000000..e29576a Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000010.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000010.png new file mode 100644 index 0000000..9894454 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000011.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000011.png new file mode 100644 index 0000000..f1ee165 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000012.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000012.png new file mode 100644 index 0000000..0d85753 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000013.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000013.png new file mode 100644 index 0000000..f6bfed0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000014.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000014.png new file mode 100644 index 0000000..6c7557e Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000015.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000015.png new file mode 100644 index 0000000..8500cef Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000016.png b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000016.png new file mode 100644 index 0000000..2840447 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-2enXX9xrIp0_7_10/000000016.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000001.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000001.png new file mode 100644 index 0000000..5b72dbd Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000001.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000002.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000002.png new file mode 100644 index 0000000..d74a42b Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000002.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000003.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000003.png new file mode 100644 index 0000000..abdf647 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000003.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000004.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000004.png new file mode 100644 index 0000000..45b1bf0 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000004.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000005.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000005.png new file mode 100644 index 0000000..e6cfbd8 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000005.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000006.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000006.png new file mode 100644 index 0000000..4e797e4 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000006.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000007.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000007.png new file mode 100644 index 0000000..b005e71 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000007.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000008.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000008.png new file mode 100644 index 0000000..f88cc9a Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000008.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000009.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000009.png new file mode 100644 index 0000000..439cb6a Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000009.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000010.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000010.png new file mode 100644 index 0000000..5e02f9c Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000010.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000011.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000011.png new file mode 100644 index 0000000..9ef2aa6 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000011.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000012.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000012.png new file mode 100644 index 0000000..2a16429 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000012.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000013.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000013.png new file mode 100644 index 0000000..f6eb754 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000013.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000014.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000014.png new file mode 100644 index 0000000..43bc602 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000014.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000015.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000015.png new file mode 100644 index 0000000..2d354a3 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000015.png differ diff --git a/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000016.png b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000016.png new file mode 100644 index 0000000..28859d8 Binary files /dev/null and b/test_data/kinetics/sampled_frames/playing guitar-wFWTW0q57ms_140_10/000000016.png differ diff --git a/utils/CalAcc.py b/utils/CalAcc.py new file mode 100644 index 0000000..070bf75 --- /dev/null +++ b/utils/CalAcc.py @@ -0,0 +1,72 @@ +import torch + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + +# output: batch_size x num_class(1000) +# target: batch_size x 1 (LongTensor) + + +def accuracy(output, target, topk=(1, 5)): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def process_activations(activations, targets, softmaxed=True): + assert activations.shape[0] % targets.shape[0] == 0, \ + f"Check the batch size of activations and targets!" + + b_repeat = activations.shape[0] // targets.shape[0] + if b_repeat > 1: + targets = targets.repeat(b_repeat) + + if not softmaxed: + soft_act = torch.nn.functional.softmax(activations, dim=1) + else: + soft_act = activations + + row_idx = torch.arange( + soft_act.shape[0], dtype=torch.long, device=activations.device) + probs = soft_act[row_idx, targets] # batch_size + pred_label_probs, pred_labels = torch.max(soft_act, dim=1) + + return probs, pred_labels, pred_label_probs + +if __name__ == "__main__": + output = torch.tensor([[0.01, 0.1, 0.05, 0.9], [0.99, 0.4, 0.2, 0.3]]) + target = torch.LongTensor([[3], [1]]) + + acc = accuracy(output, target, topk=(1, 3)) + print(acc) diff --git a/utils/ConfusionMatrix.py b/utils/ConfusionMatrix.py new file mode 100644 index 0000000..a3f6d74 --- /dev/null +++ b/utils/ConfusionMatrix.py @@ -0,0 +1,69 @@ +import numpy as np +# import matplotlib.pyplot as plt + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from sklearn import svm, datasets +from sklearn.model_selection import train_test_split +from sklearn.metrics import confusion_matrix +from sklearn.utils.multiclass import unique_labels + +import pickle + + +def plot_confusion_matrix(y_true, y_pred, classes, + normalize=False, + title=None, + cmap=plt.cm.Blues): + """ + This function prints and plots the confusion matrix. + Normalization can be applied by setting `normalize=True`. + """ + if not title: + if normalize: + title = 'Normalized confusion matrix' + else: + title = 'Confusion matrix, without normalization' + + # Compute confusion matrix + cm = confusion_matrix(y_true, y_pred) + # Only use the labels that appear in the data + classes = classes[unique_labels(y_true, y_pred)] + if normalize: + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + print("Normalized confusion matrix") + else: + print('Confusion matrix, without normalization') + + # print(cm) + + fig, ax = plt.subplots(figsize=(10,10)) + im = ax.imshow(cm, interpolation='nearest', cmap=plt.get_cmap('Blues')) + # ax.figure.colorbar(im, ax=ax) + plt.colorbar(im,fraction=0.046, pad=0.04) + # We want to show all ticks... + ax.set(xticks=np.arange(cm.shape[1]), + yticks=np.arange(cm.shape[0]), + # ... and label them with the respective list entries + xticklabels=classes, yticklabels=classes, + title=title, + ylabel='True label', + xlabel='Predicted label') + + # Rotate the tick labels and set their alignment. + plt.setp(ax.get_xticklabels(), rotation=45, ha="right", + rotation_mode="anchor", fontsize=6) + plt.setp(ax.get_yticklabels(), fontsize=6) + + # Loop over data dimensions and create text annotations. + # fmt = '.2f' if normalize else 'd' + # thresh = cm.max() / 2. + # for i in range(cm.shape[0]): + # for j in range(cm.shape[1]): + # ax.text(j, i, format(cm[i, j], fmt), + # ha="center", va="center", + # color="white" if cm[i, j] > thresh else "black") + fig.tight_layout() + return fig, ax \ No newline at end of file diff --git a/utils/ImageShow.py b/utils/ImageShow.py new file mode 100644 index 0000000..8dec631 --- /dev/null +++ b/utils/ImageShow.py @@ -0,0 +1,268 @@ +import torch +import torchvision.transforms.functional as TF + +import numpy as np +from PIL import Image +from skimage import transform, filters +import math +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +def pil_to_tensor(pil_image): + r"""Convert a PIL image to a tensor. + Args: + pil_image (:class:`PIL.Image`): PIL image. + Returns: + :class:`torch.Tensor`: the image as a :math:`3\times H\times W` tensor + in the [0, 1] range. + """ + pil_image = np.array(pil_image) + if len(pil_image.shape) == 2: + pil_image = pil_image[:, :, None] + return torch.tensor(pil_image, dtype=torch.float32).permute(2, 0, 1) / 255 + +def img_tensor_to_pil(img_tensor): + lim = [img_tensor.min(), img_tensor.max()] + img_tensor = img_tensor - lim[0] # also makes a copy + img_tensor.mul_(1 / (lim[1] - lim[0])) + img_tensor = torch.clamp(img_tensor, min=0, max=1) + img_pil = TF.to_pil_image(img_tensor) + return img_pil + +def img_tensor_to_np(img_tensor): + lim = [img_tensor.min(), img_tensor.max()] + img_tensor = img_tensor - lim[0] # also makes a copy + img_tensor.mul_(1 / (lim[1] - lim[0])) + img_tensor = torch.clamp(img_tensor, min=0, max=1) + img_np = img_tensor.numpy() + return img_np + +def voxel_tensor_to_np(voxel_tensor): + # voxel_tensor: CxTxHxW + voxel_np = [] + for t in range(voxel_tensor.shape[1]): + img_np = img_tensor_to_np(voxel_tensor[:,t,:,:]) + voxel_np.append(img_np) + voxel_np = np.stack(voxel_np, axis=1) + return voxel_np + +def map_to_colormap(attMap, resize=(), norm_map=False, blur=False): + attMap = attMap.copy() + if norm_map: + attMap -= attMap.min() + if attMap.max() > 0: + attMap /= attMap.max() + + if resize != (): + attMap = transform.resize(attMap, resize, order=3) + + if blur: + attMap = filters.gaussian(attMap, 0.02*max(resize)) + attMap -= attMap.min() + attMap /= attMap.max() + + cmap = plt.get_cmap('jet') + colormap = cmap(attMap) + colormap = np.delete(colormap, 3, 2) + colormap = np.transpose(colormap, (2,0,1)) #3x224x224 + return attMap, colormap + +def overlap_map_on_img(img_np, attMap, norm_map=False, blur=False): + # img_np: CxHxW, attMap: h x w + plt.axis('off') + resized_map, colormap = map_to_colormap(attMap, resize=(img_np.shape[1:]), norm_map=False, blur=blur) + resized_map = 1*(1-resized_map**0.8)*img_np + (resized_map**0.8)*colormap + return resized_map + +def overlap_maps_on_voxel_np(voxel_np, attMaps, norm_map=False, blur=False): + overlaps = [] + for t in range(voxel_np.shape[1]): + img_np = voxel_np[:,t,:,:] + attMap = attMaps[t,:,:] + overlap = overlap_map_on_img(img_np, attMap, norm_map, blur) + overlaps.append(overlap) + overlaps = np.stack(overlaps, axis=1) # 3x16x224x224 + return overlaps + +def img_np_show (img_np, interpolation='lanczos'): + bitmap = np.transpose(img_np, (1,2,0)) # HxWxC + handle = plt.imshow( + bitmap, interpolation=interpolation, vmin=0, vmax=1) + curr_ax = plt.gca() + curr_ax.axis('off') + return handle + +def imsc(img, *args, lim=None, quiet=False, interpolation='lanczos', **kwargs): + r"""Rescale and displays an image represented as a img. + The function scales the img :attr:`im` to the [0 ,1] range. + The img is assumed to have shape :math:`3\times H\times W` (RGB) + :math:`1\times H\times W` (grayscale). + Args: + img (:class:`torch.Tensor` or :class:`PIL.Image`): image. + quiet (bool, optional): if False, do not display image. + Default: ``False``. + lim (list, optional): maximum and minimum intensity value for + rescaling. Default: ``None``. + interpolation (str, optional): The interpolation mode to use with + :func:`matplotlib.pyplot.imshow` (e.g. ``'lanczos'`` or + ``'nearest'``). Default: ``'lanczos'``. + Returns: + :class:`torch.Tensor`: Rescaled image img. + """ + if isinstance(img, Image.Image): + img = pil_to_tensor(img) + handle = None + with torch.no_grad(): + if not lim: + lim = [img.min(), img.max()] + img = img - lim[0] # also makes a copy + img.mul_(1 / (lim[1] - lim[0])) + img = torch.clamp(img, min=0, max=1) + if not quiet: + bitmap = img.expand(3, *img.shape[1:]).permute(1, 2, 0).cpu().numpy() + handle = plt.imshow( + bitmap, *args, interpolation=interpolation, **kwargs) + curr_ax = plt.gca() + curr_ax.axis('off') + return img, handle + +def plot_voxel(voxel, saliency, show_plot=False, save_path=None): + # batch_size = len(input) + num_frame = voxel.shape[1] + num_row = 2 * num_frame//8 + + plt.clf() + fig = plt.figure(figsize=(16,num_row*2)) + for i in range(num_frame): + plt.subplot(num_row, 8, (i//8)*16+i%8+1) + imsc(voxel[:,i,:,:]) + plt.title(i, fontsize=8) + + plt.subplot(num_row, 8, (i//8)*16+i%8+8+1) + imsc(saliency[:,i,:,:], interpolation='none') + + # Save figure if path is specified. + if save_path: + save_dir = os.path.dirname(os.path.abspath(save_path)) + # Create directory if necessary. + if not os.path.exists(save_dir): + os.makedirs(save_dir) + ext = os.path.splitext(save_path)[1].strip('.') + plt.savefig(save_path, format=ext, bbox_inches='tight') + + # Show plot if desired. + if show_plot: + plt.show() + +def plot_voxel_wbbox(voxel, saliency, bbox_tensor, + show_plot=False, save_path=None): + # batch_size = len(input) + num_frame = voxel.shape[1] + num_row = 2 * num_frame//8 + + for idx in range(num_frame): + x0, y0, x1, y1 = bbox_tensor[idx,:].tolist() + voxel[1, idx, y0:y1+1, x0] = 1.0 + voxel[1, idx, y0:y1+1, x1] = 1.0 + voxel[1, idx, y0, x0:x1+1] = 1.0 + voxel[1, idx, y1, x0:x1+1] = 1.0 + + plt.clf() + fig = plt.figure(figsize=(16,num_row*2)) + for i in range(num_frame): + plt.subplot(num_row, 8, (i//8)*16+i%8+1) + imsc(voxel[:,i,:,:]) + plt.title(i, fontsize=8) + + plt.subplot(num_row, 8, (i//8)*16+i%8+8+1) + imsc(saliency[:,i,:,:], interpolation='none') + + # Save figure if path is specified. + if save_path: + save_dir = os.path.dirname(os.path.abspath(save_path)) + # Create directory if necessary. + if not os.path.exists(save_dir): + os.makedirs(save_dir) + ext = os.path.splitext(save_path)[1].strip('.') + plt.savefig(save_path, format=ext, bbox_inches='tight') + + # Show plot if desired. + if show_plot: + plt.show() + +def plot_voxel_np(voxel_np, saliency_np, title=None, + show_plot=False, save_path=None): + # batch_size = len(input) + num_frame = voxel_np.shape[1] + num_row = 2 * num_frame//8 + + plt.clf() + fig = plt.figure(figsize=(16,num_row*2)) + for i in range(num_frame): + plt.subplot(num_row, 8, (i//8)*16+i%8+1) + img_np_show(voxel_np[:,i,:,:]) + plt.title(i, fontsize=8) + + plt.subplot(num_row, 8, (i//8)*16+i%8+8+1) + img_np_show(saliency_np[:,i,:,:], interpolation='none') + # fig.close() + + if title is not None: + fig.suptitle(title, fontsize=14) + + # Save figure if path is specified. + if save_path: + save_dir = os.path.dirname(os.path.abspath(save_path)) + # Create directory if necessary. + if not os.path.exists(save_dir): + os.makedirs(save_dir) + ext = os.path.splitext(save_path)[1].strip('.') + plt.savefig(save_path, format=ext, bbox_inches='tight') + + # Show plot if desired. + if show_plot: + plt.show() + +def plot_voxel_wbbox_np(voxel_np, saliency_np, bbox_tensor, title=None, + show_plot=False, save_path=None): + # batch_size = len(input) + num_frame = voxel_np.shape[1] + num_row = 2 * num_frame//8 + + for idx in range(num_frame): + x0, y0, x1, y1 = bbox_tensor[idx,:].tolist() + voxel_np[1, idx, y0:y1+1, x0] = 1.0 + voxel_np[1, idx, y0:y1+1, x1] = 1.0 + voxel_np[1, idx, y0, x0:x1+1] = 1.0 + voxel_np[1, idx, y1, x0:x1+1] = 1.0 + + plt.clf() + fig = plt.figure(figsize=(16,num_row*2)) + for i in range(num_frame): + plt.subplot(num_row, 8, (i//8)*16+i%8+1) + img_np_show(voxel_np[:,i,:,:]) + plt.title(i, fontsize=8) + + plt.subplot(num_row, 8, (i//8)*16+i%8+8+1) + img_np_show(saliency_np[:,i,:,:], interpolation='none') + + if title is not None: + fig.suptitle(title, fontsize=14) + + # Save figure if path is specified. + if save_path: + save_dir = os.path.dirname(os.path.abspath(save_path)) + # Create directory if necessary. + if not os.path.exists(save_dir): + os.makedirs(save_dir) + ext = os.path.splitext(save_path)[1].strip('.') + plt.savefig(save_path, format=ext, bbox_inches='tight') + + # Show plot if desired. + if show_plot: + plt.show() + + plt.close(fig) \ No newline at end of file diff --git a/utils/LongRangeSample.py b/utils/LongRangeSample.py new file mode 100644 index 0000000..812d9d2 --- /dev/null +++ b/utils/LongRangeSample.py @@ -0,0 +1,45 @@ +import math +import random + + +def long_range_rand(seq_len, num_seg): + r = int(seq_len / num_seg) + real_num_seg = int(math.ceil(seq_len / r)) + + frame_ind = [] + for i in range(0, real_num_seg-1): + frame_ind.append(random.randint(i*r, (i+1)*r-1)) + frame_ind.append(random.randint((real_num_seg-1)*r, seq_len-1)) + + frame_ind = frame_ind[len(frame_ind)-num_seg:] + return frame_ind + +def long_range_first(seq_len, num_seg): + r = int(seq_len / num_seg) + + frame_ind = [] + for i in range(0, seq_len, r): + frame_ind.append(i) + frame_ind = frame_ind[len(frame_ind)-num_seg:] + return frame_ind + +def long_range_last(seq_len, num_seg): + r = int(seq_len / num_seg) + + frame_ind = [] + for i in range(seq_len-1, -1, -r): + frame_ind.append(i) + + frame_ind = frame_ind[0:num_seg] + frame_ind.reverse() + return frame_ind + +def long_range_sample(seq_len, num_seg, mode): + if mode == "random": + return long_range_rand(seq_len, num_seg) + elif mode == "first": + return long_range_first(seq_len, num_seg) + elif mode == "last": + return long_range_last(seq_len, num_seg) + else: + raise Exception(f"Given mode is unacceptable!") diff --git a/visual_meth/grad_cam.py b/visual_meth/grad_cam.py new file mode 100644 index 0000000..972928c --- /dev/null +++ b/visual_meth/grad_cam.py @@ -0,0 +1,65 @@ +import torch +import torch.nn.functional as F +import numpy as np + +# Backward hook +observ_grad_ = [] +def backward_hook(m, i_grad, o_grad): + global observ_grad_ + observ_grad_.insert(0, o_grad[0].detach()) + +# Forward hook +observ_actv_ = [] +def forward_hook(m, i, o): + global observ_actv_ + observ_actv_.append(o[0].detach()) + +def grad_cam (inputs, labels, model, device, layer_name, norm_vis=True): + model.eval() # Set model to evaluate mode + + bs, ch, nt, h, w = inputs.shape + assert ch == 3 + assert labels.shape[0] == bs + + # layer_dict = dict(model.module.named_children()) + # assert layer_name in layer_dict, \ + # f'Given layer ({layer_name}) is not in model. {model}' + # observ_layer = layer_dict[layer_name] + + observ_layer = model + for name in layer_name: + print(dict(observ_layer.named_children()).keys()) + observ_layer = dict(observ_layer.named_children())[name] + + observ_layer.register_backward_hook(backward_hook) + observ_layer.register_forward_hook(forward_hook) + + inputs = inputs.to(device) + labels = labels.to(dtype=torch.long) + + # Forward pass + outputs = model(inputs) + _, preds = torch.max(outputs, 1) + + observ_actv = observ_actv_[0] # 1 x C x num_f/8 x 56 x 56 + observ_actv = torch.repeat_interleave(observ_actv, int(nt/observ_actv.shape[2]), dim=2) + + # backward pass + backward_signals = torch.zeros_like(outputs, device=device) + for bidx in range(bs): + backward_signals[bidx, labels[bidx].cpu().item()] = 1.0 + outputs.backward(backward_signals) + + observ_grad = observ_grad_[0] # 1 x C x num_f/8 x 56 x 56 + observ_grad = torch.repeat_interleave(observ_grad, int(nt/observ_grad.shape[2]), dim=2) + + observ_grad_w = observ_grad.mean(dim=4, keepdim=True).mean(dim=3, keepdim=True) # 1 x 512 x num_f x 1x1 + out_masks = F.relu( (observ_grad_w*observ_actv).sum(dim=1, keepdim=True) ) # 1 x 1 x num_f x 14x14 + + if norm_vis: + out_masks = (out_masks - out_masks.min()) / (out_masks.max() - out_masks.min()) + + return out_masks + + + diff --git a/visual_meth/gradients.py b/visual_meth/gradients.py new file mode 100644 index 0000000..e310de8 --- /dev/null +++ b/visual_meth/gradients.py @@ -0,0 +1,51 @@ +import torch +import numpy as np + +def gradients (inputs, labels, model, device, multiply_input=False, + polarity='both', show_gray=False): + model.eval() # Set model to evaluate mode + + bs, ch, nt, h, w = inputs.shape + assert ch == 3 + assert labels.shape[0] == bs + + inputs = inputs.to(device) + # labels = labels.to(device) + labels = labels.to(dtype=torch.long) + + # Get model outputs and calculate loss + inputs.requires_grad = True + with torch.set_grad_enabled(True): + outputs = model(inputs) + _, preds = torch.max(outputs, 1) + + backward_signals = torch.zeros_like(outputs, device=device) + for bidx in range(bs): + backward_signals[bidx, labels[bidx].cpu().item()] = 1.0 + outputs.backward(backward_signals) + inputs_grad = inputs.grad.cpu() + if multiply_input: + inputs_grad *= inputs.detach().cpu() + + if polarity == 'positive': + inputs_grad = torch.clamp(inputs_grad, min=0.0) + elif polarity == 'negative': + inputs_grad = -1.0 * torch.clamp(inputs_grad, max=0.0) + + if show_gray: + inputs_grad = inputs_grad.mean(dim=1, keepdim=True).repeat(1,ch,1,1,1) + + grad_sorted = inputs_grad.view(-1).sort()[0] + length = grad_sorted.shape[0] + grad_min = 0.0 + grad_max = grad_sorted[int(0.999 * length)] + grad_show = inputs_grad.clamp(min=grad_min, max=grad_max) + grad_show = (grad_show - grad_min) / (grad_max - grad_min) + # print(grad_show.max(), grad_show.min()) + + if polarity == 'positive': + grad_show[:,1:,...] = 0.0 + elif polarity == 'negative': + grad_show[:,:2,...] = 0.0 + + return grad_show diff --git a/visual_meth/integrated_grad.py b/visual_meth/integrated_grad.py new file mode 100644 index 0000000..f6c3892 --- /dev/null +++ b/visual_meth/integrated_grad.py @@ -0,0 +1,57 @@ +import torch +import numpy as np + +def integrated_grad (inputs, labels, model, device, steps, + polarity='positive', show_gray=True): + model.eval() # Set model to evaluate mode + + bs, ch, nt, h, w = inputs.shape + assert ch == 3 + assert labels.shape[0] == bs + + inputs = inputs.to(device) + # labels = labels.to(device) + labels = labels.to(dtype=torch.long) + + baseline = torch.zeros_like(inputs, device=device) + baseline_out = model(baseline) + + backward_signals = torch.zeros_like(baseline_out, device=device) + for bidx in range(bs): + backward_signals[bidx, labels[bidx].cpu().item()] = 1.0 + + intg_grads = 0 + for i in range(steps): + scaled_inputs = baseline + (float(i) / steps) * (inputs - baseline) + scaled_inputs.requires_grad = True + + # Forward + outputs = model(scaled_inputs) + _, preds = torch.max(outputs, dim=1) + + # Backward + outputs.backward(backward_signals) + intg_grads += scaled_inputs.grad.cpu() / steps + intg_grads *= (inputs - baseline).detach().cpu() + + if polarity == 'positive': + intg_grads = torch.clamp(intg_grads, min=0.0) + elif polarity == 'negative': + intg_grads = -1.0 * torch.clamp(intg_grads, max=0.0) + + if show_gray: + intg_grads = intg_grads.mean(dim=1, keepdim=True).repeat(1,ch,1,1,1) + + grad_sorted = intg_grads.view(-1).sort()[0] + length = grad_sorted.shape[0] + grad_min = 0.0 + grad_max = grad_sorted[int(0.999 * length)] + grad_show = intg_grads.clamp(min=grad_min, max=grad_max) + grad_show = (grad_show - grad_min) / (grad_max - grad_min) # NxCxTxHxW + + if polarity == 'positive': + intg_grads[:,1:,...] = 0.0 + elif polarity == 'negative': + intg_grads[:,:2,...] = 0.0 + return grad_show + diff --git a/visual_meth/perturbation.py b/visual_meth/perturbation.py new file mode 100644 index 0000000..e1ee734 --- /dev/null +++ b/visual_meth/perturbation.py @@ -0,0 +1,414 @@ +import math +import matplotlib.pyplot as plt +import numpy as np + +import torch +import torch.nn.functional as F +import torch.optim as optim + +import sys +sys.path.append(".") +sys.path.append("..") +from utils.CalAcc import process_activations +from utils.ImageShow import * + +from torchray.utils import imsmooth, imsc +from torchray.attribution.common import resize_saliency + +BLUR_PERTURBATION = "blur" +FADE_PERTURBATION = "fade" + +PRESERVE_VARIANT = "preserve" +DELETE_VARIANT = "delete" +DUAL_VARIANT = "dual" + +def simple_log_reward(activation, target, variant): + N = target.shape[0] + bs = activation.shape[0] + b_repeat = int( bs // N ) + device = activation.device + + col_idx = target.repeat(b_repeat) # batch_size + row_idx = torch.arange(activation.shape[0], dtype=torch.long, device=device) # batch_size + prob = activation[row_idx, col_idx] # batch_size + + if variant == DELETE_VARIANT: + reward = -torch.log(1-prob) + elif variant == PRESERVE_VARIANT: + reward = -torch.log(prob) + elif variant == DUAL_VARIANT: + reward = (-torch.log(1-prob[N:])) + (-torch.log(prob[:N])) + else: + assert False + return reward + +class MaskGenerator: + def __init__(self, shape, step, sigma, batch_size=1, clamp=True, pooling_method='softmax'): + self.shape = shape + self.step = step + self.sigma = sigma + self.coldness = 20 + self.batch_size = batch_size + self.clamp = clamp + self.pooling_method = pooling_method + + assert int(step) == step + + # self.kernel = lambda z: (z < 1).float() + self.kernel = lambda z: torch.exp(-2 * ((z - .5).clamp(min=0)**2)) + + self.margin = self.sigma + self.padding = 1 + math.ceil((self.margin + sigma) / step) + self.radius = 1 + math.ceil(sigma / step) + self.shape_in = [math.ceil(z / step) for z in self.shape] + self.shape_mid = [ + z + 2 * self.padding - (2 * self.radius + 1) + 1 + for z in self.shape_in + ] + self.shape_up = [self.step * z for z in self.shape_mid] + self.shape_out = [z - step + 1 for z in self.shape_up] + + step_inv = [ + torch.tensor(zm, dtype=torch.float32) / + torch.tensor(zo, dtype=torch.float32) + for zm, zo in zip(self.shape_mid, self.shape_up) + ] + + # Generate kernel weights for smoothing mask_in + self.weight = torch.zeros(( + 1, + (2 * self.radius + 1)**2, + self.shape_out[0], + self.shape_out[1] + )) + + for ky in range(2 * self.radius + 1): + for kx in range(2 * self.radius + 1): + uy, ux = torch.meshgrid( + torch.arange(self.shape_out[0], dtype=torch.float32), + torch.arange(self.shape_out[1], dtype=torch.float32) + ) + iy = torch.floor(step_inv[0] * uy) + ky - self.padding + ix = torch.floor(step_inv[1] * ux) + kx - self.padding + + delta = torch.sqrt( + (uy - (self.margin + self.step * iy))**2 + + (ux - (self.margin + self.step * ix))**2 + ) + + k = ky * (2 * self.radius + 1) + kx + + self.weight[0, k] = self.kernel(delta / sigma) + + def generate(self, mask_in): + # mask_in: Nx1xHxW --> mask: Nx1xS_outxS_out + mask = F.unfold(mask_in, + (2 * self.radius + 1,) * 2, + padding=(self.padding,) * 2) + mask = mask.reshape( + mask_in.shape[0], -1, self.shape_mid[0], self.shape_mid[1]) + mask = F.interpolate(mask, size=self.shape_up, mode='nearest') + mask = F.pad(mask, (0, -self.step + 1, 0, -self.step + 1)) + mask = self.weight * mask + + if self.pooling_method == 'sigmoid': + if self.coldness == float('+Inf'): + mask = (mask.sum(dim=1, keepdim=True) - 5 > 0).float() + else: + mask = torch.sigmoid( + self.coldness * mask.sum(dim=1, keepdim=True) - 3 + ) + elif self.pooling_method == 'softmax': + if self.coldness == float('+Inf'): # max normalization + mask = mask.max(dim=1, keepdim=True)[0] + else: # smax normalization + mask = ( + mask * F.softmax(self.coldness * mask, dim=1) + ).sum(dim=1, keepdim=True) + elif self.pooling_method == 'sum': + mask = mask.sum(dim=1, keepdim=True) + else: + assert False, f"Unknown pooling method {self.pooling_method}" + + m = round(self.margin) + if self.clamp: + mask = mask.clamp(min=0, max=1) + cropped = mask[:, :, m:m + self.shape[0], m:m + self.shape[1]] + return cropped, mask + + def to(self, dev): + """Switch to another device. + Args: + dev: PyTorch device. + Returns: + MaskGenerator: self. + """ + self.weight = self.weight.to(dev) + return self + +class Perturbation: + def __init__(self, input, num_levels=8, max_blur=20, type=BLUR_PERTURBATION): + self.type = type + self.num_levels = num_levels + self.pyramid = [] + assert num_levels >= 2 + assert max_blur > 0 + with torch.no_grad(): + for sigma in torch.linspace(0, 1, self.num_levels): + if type == BLUR_PERTURBATION: + # input could be a batched tensor with size of NxCxHxW + y = imsmooth(input, sigma=(1 - sigma) * max_blur) + # ouput y has size of NxCxHxW + elif type == FADE_PERTURBATION: + y = input * sigma + else: + assert False + self.pyramid.append(y) + # self.pyramid = torch.cat(self.pyramid, dim=0) + self.pyramid = torch.stack(self.pyramid, dim=1) # NxLxCxHxW, L=num_levels + + def apply(self, mask): + # mask: Nx1xHxW + n = mask.shape[0] + # starred expression: unpack a list to separated numbers + w = mask.reshape(n, 1, *mask.shape[1:]) # Nx1x1xHxW, mask.unsqueeze(1) + w = w * (self.num_levels - 1) # w = 7*w + k = w.floor() # Integral part of w + w = w - k # Fractional part of w + k = k.long() # Transfer k to long int + + # y = self.pyramid[None, :] #1xLxCxHxW + # y = y.expand(n, *y.shape[1:]) #nxLxCxHxW + # k = k.expand(n, 1, *y.shape[2:]) #nx1xCxHxW + y = self.pyramid # NxLxCxHxW + k = k.expand(n, 1, *y.shape[2:]) #Nx1xCxHxW + y0 = torch.gather(y, 1, k) # select low level, Nx1xCxHxW + y1 = torch.gather(y, 1, torch.clamp(k + 1, max=self.num_levels - 1)) # select high level, Nx1xCxHxW + + # return ((1 - w) * y0 + w * y1).squeeze(dim=1) + perturb_x = ((1 - w) * y0 + w * y1) #Nx1xCxHxW + return perturb_x + + def to(self, dev): + """Switch to another device. + Args: + dev: PyTorch device. + Returns: + Perturbation: self. + """ + self.pyramid.to(dev) + return self + + def __str__(self): + return ( + f"Perturbation:\n" + f"- type: {self.type}\n" + f"- num_levels: {self.num_levels}\n" + f"- pyramid shape: {list(self.pyramid.shape)}" + ) + +def video_perturbation(model, input, target, + areas=[0.1], method='spatiotemporal', perturb_type=FADE_PERTURBATION, + max_iter=2000, num_levels=8, step=7, sigma=11, variant=PRESERVE_VARIANT, + print_iter=None, debug=False, reward_func="simple_log", resize=False, + resize_mode='bilinear', smooth=0, num_devices=1): + + if isinstance(areas, float): + areas = [areas] + momentum = 0.9 + learning_rate = 0.05 + regul_weight = 300 + reward_weight = 100 + device = input.device + + iter_period = 2000 + + regul_weight_last = max(regul_weight / 2, 1) + + # input shape: NxCxTxHxW (1x3x16x112x112) + batch_size = input.shape[0] + num_frame = input.shape[2] #16 + num_areas = len(areas) + + if debug: + print( + f"spatiotemporal_perturbation:\n" + f"- target: {target}\n" + f"- areas: {areas}\n" + f"- variant: {variant}\n" + f"- max_iter: {max_iter}\n" + f"- step/sigma: {step}, {sigma}\n" + f"- voxel size: {list(input.shape)}\n" + f"- reward function: {reward_func}" + ) + print(f"- Target: {target.detach().cpu().tolist()}") + + # Disable gradients for model parameters. + for p in model.parameters(): + p.requires_grad_(False) + model.eval() + + ori_y = model(input) + ori_prob, ori_pred_label, _ = process_activations(ori_y, target, softmaxed=True) + + # NxCxTxHxW --> N*T x CxHxW + pmt_inp = input.transpose(1,2).contiguous() # NxTxCxHxW + pmt_inp = pmt_inp.view(batch_size*num_frame, *pmt_inp.shape[2:]) # N*T x CxHxW + + # Get the perturbation operator. + # perturbation.pyramid: T*N x LxCxHxW + perturbation = Perturbation(pmt_inp, num_levels=num_levels, + type=perturb_type).to(device) + + # Prepare the mask generator (generating mask(134x134) from pmask(16x16)). + shape = perturbation.pyramid.shape[3:] # 112x112 + mask_generator = MaskGenerator(shape, step, sigma, pooling_method='softmax').to(device) + h, w = mask_generator.shape_in # h=112/step, w=112/step, 16x16 + pmasks = torch.ones(batch_size*num_frame, 1, h, w).to(device) #N*T x 1x16x16 + + max_area = np.prod(mask_generator.shape_out) + max_volume = np.prod(mask_generator.shape_out) * num_frame + + # Prepare reference area vector. + if method == 'spatiotemporal': + vref = torch.ones(batch_size, max_volume).to(device) + vref[:, :int(max_volume * (1 - areas[0]))] = 0 + elif method == 'extremal': + aref = torch.ones(batch_size, num_frame, max_area).to(device) + aref[:, :, :int(max_area * (1 - areas[0]))] = 0 + + # Initialize optimizer. + optimizer = optim.SGD([pmasks], + lr=learning_rate, + momentum=momentum, + dampening=momentum) + hist = torch.zeros((batch_size, 2, 0)) + + for t in range(max_iter): + pmasks.requires_grad_(True) + masks, padded_masks = mask_generator.generate(pmasks) + + if variant == DELETE_VARIANT: + perturb_x = perturbation.apply(1 - masks) # N*T x 1xCxHxW + elif variant == PRESERVE_VARIANT: + perturb_x = perturbation.apply(masks) # N*T x 1xCxHxW + elif variant == DUAL_VARIANT: + perturb_x = torch.cat(( + perturbation.apply(masks), #preserve + perturbation.apply(1 - masks), #delete + ), dim = 1) # N*T x 2xCxHxW + else: + assert False + + perturb_x = perturb_x.view(batch_size, num_frame, *perturb_x.shape[1:]) # NxTx2xCxHxW + perturb_x = perturb_x.permute(2, 0, 3, 1, 4, 5).contiguous() # 2xNxCxTxHxW + perturb_x = perturb_x.view(perturb_x.shape[0]*perturb_x.shape[1], *perturb_x.shape[2:]) # 2*N x CxTxHxW + + masks = masks.view(batch_size, num_frame, *masks.shape[1:]).transpose(1,2) # Nx1xTxHxW + padded_masks = padded_masks.view(batch_size, num_frame, \ + *padded_masks.shape[1:]).transpose(1,2) # Nx1xTx S_out x S_out + + # Evaluate the model on the masked data + # The input of model should have size of NxCxTxHxW + y = model(perturb_x) # 2*N x num_classes + y = F.softmax(y, dim=1) + + # Cal probability + prob, pred_label, pred_label_prob = process_activations(y, target, softmaxed=True) + + # Get reward. + if reward_func == "simple_log": + reward = simple_log_reward(y, target, variant=variant) + else: + raise Exception("Do not support other reward function now.") + # print(f"Reward shape: {reward.shape}") + reward = reward.view(batch_size, -1).mean(dim=1) * reward_weight # batch_size + + # Area regularization. + # padded_masks: N x 1 x T x S_out x S_out + if method == 'spatiotemporal': + mask_sorted = padded_masks.squeeze(1).reshape(batch_size, -1).sort(dim=1)[0] + regul = ((mask_sorted - vref)**2).mean(dim=1) * regul_weight # batch_size + elif method == 'extremal': + mask_sorted = padded_masks.squeeze(1).reshape(batch_size, num_frame, -1).sort(dim=2)[0] + regul = ((mask_sorted - aref)**2).mean(dim=2).mean(dim=1) * regul_weight # batch_size + + # Energy summary + energy = (reward + regul).sum() + + # Gradient step. + optimizer.zero_grad() + energy.backward() + optimizer.step() + + pmasks.data = pmasks.data.clamp(0, 1) + + # Record energy + # hist: batch_size x 2 x num_iter + hist_item = torch.cat( + ( + reward.detach().cpu().view(-1, 1, 1), + regul.detach().cpu().view(-1, 1, 1) + ), dim=1) + # print(f"Item shape: {hist_item.shape}") + hist = torch.cat((hist,hist_item), dim=2) + + # Diagnostics. + # debug_this_iter = debug and (regul_weight / regul_weight_last >= 2) + + # if (print_iter is not None and t % print_iter == 0) or debug_this_iter: + if (print_iter != None) and (t % print_iter == 0): + print("[{:04d}/{:04d}]".format(t + 1, max_iter), end="\n") + for i in range(batch_size): + if variant == "dual": + print(" [area:{:.2f} loss:{:.2f} reg:{:.2f} presv:{:.2f}/{} del:{:.2f}/{}]".format( + areas[0], hist[i, 0, -1], hist[i, 1, -1], + prob[i], pred_label[i], + prob[i+batch_size], pred_label[i+batch_size]), end="") + else: + print(" [area:{:.2f} loss:{:.2f} reg:{:.2f} {}:{:.2f}/{}]".format( + areas[0], hist[i, 0, -1], hist[i, 1, -1], + variant, prob[i], pred_label[i]), end="") + print() + + # if debug_this_iter: + # regul_weight_last = regul_weight + # # for i, a in enumerate(areas): + # for i in range(batch_size): + # plt.figure(i, figsize=(20, 6)) + # plt.clf() + # ncols = 4 if variant == DUAL_VARIANT else 3 + # plt.subplot(1, ncols, 1) + # plt.plot(hist[i, 0].numpy()) + # plt.plot(hist[i, 1].numpy()) + # plt.plot(hist[i].sum(dim=0).numpy()) + # plt.legend(('energy', 'regul', 'both')) + # plt.title(f'target area:{areas[0]:.2f}') + # mask = padded_masks[i,:,7,:,:] + # plt.subplot(1, ncols, 2) + # imsc(mask, lim=[0, 1]) + # plt.title( + # f"min:{mask.min().item():.2f}" + # f" max:{mask.max().item():.2f}" + # f" area:{mask.sum() / mask.numel():.2f}") + # plt.subplot(1, ncols, 3) + # imsc(perturb_x[i,:,7,:,:]) + # if variant == DUAL_VARIANT: + # plt.subplot(1, ncols, 4) + # imsc(perturb_x[i + batch_size,:,7,:,:]) + # plt.pause(0.001) + + masks = masks.detach() + # Resize saliency map. + list_mask = [] + for frame_idx in range(num_frame): + mask = masks[:,:,frame_idx,:,:] + mask = resize_saliency(pmt_inp, mask, resize, mode=resize_mode) + # Smooth saliency map. + if smooth > 0: + mask = imsmooth(mask, sigma=smooth * min(mask.shape[2:]), padding_mode='constant') + list_mask.append(mask) + masks = torch.stack(list_mask, dim=2) # NxCxTxHxW + + # masks: NxCxTxHxW; hist: Nx3xmax_iter + return masks, hist, perturb_x \ No newline at end of file