این صفحه به‌وسیله ‏Cloud Translation API‏ ترجمه شده است.

تشخیص عملکرد با یک CNN تورم 3D

مشاهده در TensorFlow.org

در Google Colab اجرا شود

در GitHub مشاهده کنید

دانلود دفترچه یادداشت

مدل TF Hub را ببینید

این نشان می دهد COLAB اقدامات شناخت در داده های ویدئویی با استفاده از tfhub.dev/deepmind/i3d-kinetics-400/1 ماژول. مدل های بیشتر برای تشخیص اقدامات در فیلم را می توان یافت در اینجا .

مدل اساسی در مقاله شرح " موجود VADIS، تشخیص اکشن؟ مدل جدید و سینتیک مجموعه داده " توسط Joao Carreira و اندرو Zisserman. این مقاله در می 2017 در arXiv پست شد و به عنوان مقاله کنفرانس CVPR 2017 منتشر شد. کد منبع در دسترس عموم است در گیتهاب .

«Quo Vadis» یک معماری جدید برای طبقه‌بندی ویدیویی معرفی کرد که Inflated 3D Convnet یا I3D است. این معماری با تنظیم دقیق این مدل‌ها به نتایج پیشرفته‌ای در مجموعه داده‌های UCF101 و HMDB51 دست یافت. مدل I3D قبل آموزش دیده در سینتیک همچنین برای اولین بار در CVPR 2017 قرار داده شده باکلمات چالش .

ماژول اصلی در آموزش دیده بود سینتیک-400 dateset و حدود 400 عملیات های مختلف می داند. برچسب برای این اقدامات را می توان در یافت فایل بر روی نقشه برچسب .

در این Colab از آن استفاده خواهیم کرد تا فعالیت‌های موجود در ویدیوها را از مجموعه داده UCF101 شناسایی کند.

برپایی

pip install -q imageio
pip install -q opencv-python
pip install -q git+https://github.com/tensorflow/docs

ماژول های لازم را وارد کنید

# TensorFlow and TF-Hub modules.
from absl import logging

import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed

logging.set_verbosity(logging.ERROR)

# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np

# Some modules to display an animation using imageio.
import imageio
from IPython import display

from urllib import request  # requires python3

توابع کمکی برای مجموعه داده UCF101

# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()

def list_ucf_videos():
  """Lists videos available in UCF101 dataset."""
  global _VIDEO_LIST
  if not _VIDEO_LIST:
    index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
    videos = re.findall("(v_[\w_]+\.avi)", index)
    _VIDEO_LIST = sorted(set(videos))
  return list(_VIDEO_LIST)

def fetch_ucf_video(video):
  """Fetchs a video and cache into local filesystem."""
  cache_path = os.path.join(_CACHE_DIR, video)
  if not os.path.exists(cache_path):
    urlpath = request.urljoin(UCF_ROOT, video)
    print("Fetching %s => %s" % (urlpath, cache_path))
    data = request.urlopen(urlpath, context=unverified_context).read()
    open(cache_path, "wb").write(data)
  return cache_path

# Utilities to open video files using CV2
def crop_center_square(frame):
  y, x = frame.shape[0:2]
  min_dim = min(y, x)
  start_x = (x // 2) - (min_dim // 2)
  start_y = (y // 2) - (min_dim // 2)
  return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]

def load_video(path, max_frames=0, resize=(224, 224)):
  cap = cv2.VideoCapture(path)
  frames = []
  try:
    while True:
      ret, frame = cap.read()
      if not ret:
        break
      frame = crop_center_square(frame)
      frame = cv2.resize(frame, resize)
      frame = frame[:, :, [2, 1, 0]]
      frames.append(frame)

      if len(frames) == max_frames:
        break
  finally:
    cap.release()
  return np.array(frames) / 255.0

def to_gif(images):
  converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
  imageio.mimsave('./animation.gif', converted_images, fps=25)
  return embed.embed_file('./animation.gif')

برچسب های kinetics-400 را دریافت کنید

# Get the kinetics-400 action labels from the GitHub repository.
KINETICS_URL = "https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt"
with request.urlopen(KINETICS_URL) as obj:
  labels = [line.decode("utf-8").strip() for line in obj.readlines()]
print("Found %d labels." % len(labels))

Found 400 labels.

با استفاده از مجموعه داده UCF101

# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()

categories = {}
for video in ucf_videos:
  category = video[2:-12]
  if category not in categories:
    categories[category] = []
  categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))

for category, sequences in categories.items():
  summary = ", ".join(sequences[:2])
  print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))

Found 13320 videos in 101 categories.
ApplyEyeMakeup        145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...)
ApplyLipstick         114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...)
Archery               145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...)
BabyCrawling          132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...)
BalanceBeam           108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...)
BandMarching          155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...)
BaseballPitch         150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...)
BasketballDunk        131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...)
Basketball            134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...)
BenchPress            160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...)
Biking                134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...)
Billiards             150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...)
BlowDryHair           131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...)
BlowingCandles        109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...)
BodyWeightSquats      112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...)
Bowling               155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...)
BoxingPunchingBag     163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...)
BoxingSpeedBag        134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...)
BreastStroke          101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...)
BrushingTeeth         131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...)
CleanAndJerk          112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...)
CliffDiving           138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...)
CricketBowling        139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...)
CricketShot           167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...)
CuttingInKitchen      110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...)
Diving                150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...)
Drumming              161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...)
Fencing               111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...)
FieldHockeyPenalty    126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...)
FloorGymnastics       125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...)
FrisbeeCatch          126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...)
FrontCrawl            137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...)
GolfSwing             139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...)
Haircut               130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...)
HammerThrow           150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...)
Hammering             140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...)
HandstandPushups      128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...)
HandstandWalking      111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...)
HeadMassage           147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...)
HighJump              123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...)
HorseRace             124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...)
HorseRiding           164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...)
HulaHoop              125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...)
IceDancing            158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...)
JavelinThrow          117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...)
JugglingBalls         121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...)
JumpRope              144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...)
JumpingJack           123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...)
Kayaking              141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...)
Knitting              123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...)
LongJump              131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...)
Lunges                127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...)
MilitaryParade        125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...)
Mixing                136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...)
MoppingFloor          110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...)
Nunchucks             132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...)
ParallelBars          114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...)
PizzaTossing          113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...)
PlayingCello          164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...)
PlayingDaf            151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...)
PlayingDhol           164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...)
PlayingFlute          155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...)
PlayingGuitar         160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...)
PlayingPiano          105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...)
PlayingSitar          157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...)
PlayingTabla          111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...)
PlayingViolin         100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...)
PoleVault             149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...)
PommelHorse           123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...)
PullUps               100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...)
Punch                 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...)
PushUps               102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...)
Rafting               111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...)
RockClimbingIndoor    144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...)
RopeClimbing          119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...)
Rowing                137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...)
SalsaSpin             133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...)
ShavingBeard          161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...)
Shotput               144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...)
SkateBoarding         120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...)
Skiing                135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...)
Skijet                100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...)
SkyDiving             110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...)
SoccerJuggling        147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...)
SoccerPenalty         137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...)
StillRings            112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...)
SumoWrestling         116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...)
Surfing               126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...)
Swing                 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...)
TableTennisShot       140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...)
TaiChi                100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...)
TennisSwing           166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...)
ThrowDiscus           130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...)
TrampolineJumping     119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...)
Typing                136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...)
UnevenBars            104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...)
VolleyballSpiking     116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...)
WalkingWithDog        123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...)
WallPushups           130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...)
WritingOnBoard        152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...)
YoYo                  128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)

# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)

Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmp/tmp4wns8m2b/v_CricketShot_g04_c02.avi

sample_video.shape

(116, 224, 224, 3)

i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']

مدل id3 را اجرا کنید و 5 پیش بینی عمل برتر را چاپ کنید.

def predict(sample_video):
  # Add a batch axis to the sample video.
  model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]

  logits = i3d(model_input)['default'][0]
  probabilities = tf.nn.softmax(logits)

  print("Top 5 actions:")
  for i in np.argsort(probabilities)[::-1][:5]:
    print(f"  {labels[i]:22}: {probabilities[i] * 100:5.2f}%")

predict(sample_video)

Top 5 actions:
  playing cricket       : 97.77%
  skateboarding         :  0.71%
  robot dancing         :  0.56%
  roller skating        :  0.56%
  golf putting          :  0.13%

: در حال حاضر یک ویدیوی جدید، از سعی https://commons.wikimedia.org/wiki/Category : Videos_of_sports

چگونه در مورد این فیلم توسط Patrick ژیلت:

curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv

% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 55.0M  100 55.0M    0     0  21.9M      0  0:00:02  0:00:02 --:--:-- 21.9M

video_path = "End_of_a_jam.ogv"

sample_video = load_video(video_path)[:100]
sample_video.shape

(100, 224, 224, 3)

to_gif(sample_video)

gif

predict(sample_video)

Top 5 actions:
  roller skating        : 96.85%
  playing volleyball    :  1.63%
  skateboarding         :  0.21%
  playing ice hockey    :  0.20%
  playing basketball    :  0.16%