 View on TensorFlow.org
|
 Run in Google Colab
|
 View on GitHub
|
 Download notebook
|
 See TF Hub model
|
This Colab demonstrates recognizing actions in video data using the tfhub.dev/deepmind/i3d-kinetics-400/1 module. More models to detect actions in videos can be found here.
The underlying model is described in the paper "Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset" by Joao Carreira and Andrew Zisserman. The paper was posted on arXiv in May 2017, and was published as a CVPR 2017 conference paper. The source code is publicly available on github.
"Quo Vadis" introduced a new architecture for video classification, the Inflated 3D Convnet or I3D. This architecture achieved state-of-the-art results on the UCF101 and HMDB51 datasets from fine-tuning these models. I3D models pre-trained on Kinetics also placed first in the CVPR 2017 Charades challenge.
The original module was trained on the kinetics-400 dateset and knows about 400 different actions. Labels for these actions can be found in the label map file.
In this Colab we will use it recognize activities in videos from a UCF101 dataset.
Setup
pip install -q imageiopip install -q opencv-pythonpip install -q git+https://github.com/tensorflow/docs
Import the necessary modules
# TensorFlow and TF-Hub modules.
from absl import logging
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed
logging.set_verbosity(logging.ERROR)
# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np
# Some modules to display an animation using imageio.
import imageio
from IPython import display
from urllib import request # requires python3
Helper functions for the UCF101 dataset
# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()
def list_ucf_videos():
"""Lists videos available in UCF101 dataset."""
global _VIDEO_LIST
if not _VIDEO_LIST:
index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
videos = re.findall("(v_[\w_]+\.avi)", index)
_VIDEO_LIST = sorted(set(videos))
return list(_VIDEO_LIST)
def fetch_ucf_video(video):
"""Fetches a video and cache into local filesystem."""
cache_path = os.path.join(_CACHE_DIR, video)
if not os.path.exists(cache_path):
urlpath = request.urljoin(UCF_ROOT, video)
print("Fetching %s => %s" % (urlpath, cache_path))
data = request.urlopen(urlpath, context=unverified_context).read()
open(cache_path, "wb").write(data)
return cache_path
# Utilities to open video files using CV2
def crop_center_square(frame):
y, x = frame.shape[0:2]
min_dim = min(y, x)
start_x = (x // 2) - (min_dim // 2)
start_y = (y // 2) - (min_dim // 2)
return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]
def load_video(path, max_frames=0, resize=(224, 224)):
cap = cv2.VideoCapture(path)
frames = []
try:
while True:
ret, frame = cap.read()
if not ret:
break
frame = crop_center_square(frame)
frame = cv2.resize(frame, resize)
frame = frame[:, :, [2, 1, 0]]
frames.append(frame)
if len(frames) == max_frames:
break
finally:
cap.release()
return np.array(frames) / 255.0
def to_gif(images):
converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
imageio.mimsave('./animation.gif', converted_images, duration=40)
return embed.embed_file('./animation.gif')
Get the kinetics-400 labels
# Get the kinetics-400 action labels from the GitHub repository.
KINETICS_URL = "https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt"
with request.urlopen(KINETICS_URL) as obj:
labels = [line.decode("utf-8").strip() for line in obj.readlines()]
print("Found %d labels." % len(labels))
Found 400 labels.
Using the UCF101 dataset
# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()
categories = {}
for video in ucf_videos:
category = video[2:-12]
if category not in categories:
categories[category] = []
categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))
for category, sequences in categories.items():
summary = ", ".join(sequences[:2])
print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))
Found 13320 videos in 101 categories. ApplyEyeMakeup 145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...) ApplyLipstick 114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...) Archery 145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...) BabyCrawling 132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...) BalanceBeam 108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...) BandMarching 155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...) BaseballPitch 150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...) BasketballDunk 131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...) Basketball 134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...) BenchPress 160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...) Biking 134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...) Billiards 150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...) BlowDryHair 131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...) BlowingCandles 109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...) BodyWeightSquats 112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...) Bowling 155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...) BoxingPunchingBag 163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...) BoxingSpeedBag 134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...) BreastStroke 101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...) BrushingTeeth 131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...) CleanAndJerk 112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...) CliffDiving 138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...) CricketBowling 139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...) CricketShot 167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...) CuttingInKitchen 110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...) Diving 150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...) Drumming 161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...) Fencing 111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...) FieldHockeyPenalty 126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...) FloorGymnastics 125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...) FrisbeeCatch 126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...) FrontCrawl 137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...) GolfSwing 139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...) Haircut 130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...) HammerThrow 150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...) Hammering 140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...) HandstandPushups 128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...) HandstandWalking 111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...) HeadMassage 147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...) HighJump 123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...) HorseRace 124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...) HorseRiding 164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...) HulaHoop 125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...) IceDancing 158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...) JavelinThrow 117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...) JugglingBalls 121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...) JumpRope 144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...) JumpingJack 123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...) Kayaking 141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...) Knitting 123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...) LongJump 131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...) Lunges 127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...) MilitaryParade 125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...) Mixing 136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...) MoppingFloor 110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...) Nunchucks 132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...) ParallelBars 114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...) PizzaTossing 113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...) PlayingCello 164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...) PlayingDaf 151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...) PlayingDhol 164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...) PlayingFlute 155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...) PlayingGuitar 160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...) PlayingPiano 105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...) PlayingSitar 157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...) PlayingTabla 111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...) PlayingViolin 100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...) PoleVault 149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...) PommelHorse 123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...) PullUps 100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...) Punch 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...) PushUps 102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...) Rafting 111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...) RockClimbingIndoor 144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...) RopeClimbing 119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...) Rowing 137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...) SalsaSpin 133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...) ShavingBeard 161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...) Shotput 144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...) SkateBoarding 120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...) Skiing 135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...) Skijet 100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...) SkyDiving 110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...) SoccerJuggling 147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...) SoccerPenalty 137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...) StillRings 112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...) SumoWrestling 116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...) Surfing 126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...) Swing 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...) TableTennisShot 140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...) TaiChi 100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...) TennisSwing 166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...) ThrowDiscus 130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...) TrampolineJumping 119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...) Typing 136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...) UnevenBars 104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...) VolleyballSpiking 116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...) WalkingWithDog 123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...) WallPushups 130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...) WritingOnBoard 152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...) YoYo 128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)
# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)
Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmpfs/tmp/tmpmfdu65l5/v_CricketShot_g04_c02.avi
sample_video.shape
(116, 224, 224, 3)
i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']
2024-03-09 14:56:58.199382: E external/local_xla/xla/stream_executor/cuda/cuda_driver.cc:282] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected
Run the id3 model and print the top-5 action predictions.
def predict(sample_video):
# Add a batch axis to the sample video.
model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]
logits = i3d(model_input)['default'][0]
probabilities = tf.nn.softmax(logits)
print("Top 5 actions:")
for i in np.argsort(probabilities)[::-1][:5]:
print(f" {labels[i]:22}: {probabilities[i] * 100:5.2f}%")
predict(sample_video)
Top 5 actions: playing cricket : 97.77% skateboarding : 0.71% robot dancing : 0.56% roller skating : 0.56% golf putting : 0.13%
Now try a new video, from: https://commons.wikimedia.org/wiki/Category:Videos_of_sports
How about this video by Patrick Gillett:
curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 55.0M 100 55.0M 0 0 26.7M 0 0:00:02 0:00:02 --:--:-- 26.7M
video_path = "End_of_a_jam.ogv"
sample_video = load_video(video_path)[:100]
sample_video.shape
(100, 224, 224, 3)
to_gif(sample_video)
predict(sample_video)
Top 5 actions: roller skating : 96.85% playing volleyball : 1.63% skateboarding : 0.21% playing ice hockey : 0.20% playing basketball : 0.16%