import cv2 import torch as th import os import numpy as np from decord import VideoReader, cpu import ffmpeg class Normalize(object): def __init__(self, mean, std): self.mean = th.FloatTensor(mean).view(1, 3, 1, 1) self.std = th.FloatTensor(std).view(1, 3, 1, 1) def __call__(self, tensor): tensor = (tensor - self.mean) / (self.std + 1e-8) return tensor class Preprocessing(object): def __init__(self): self.norm = Normalize( mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711], ) def __call__(self, tensor): tensor = tensor / 255.0 tensor = self.norm(tensor) return tensor class VideoLoader: """Pytorch video loader.""" def __init__( self, framerate=1, size=224, centercrop=True, ): self.centercrop = centercrop self.size = size self.framerate = framerate self.preprocess = Preprocessing() self.max_feats = 10 self.features_dim = 768 # def _get_video_dim(self, video_path): # vr = VideoReader(video_path, ctx=cpu(0)) # height, width, _ = vr[0].shape # frame_rate = vr.get_avg_fps() # return height, width, frame_rate def _get_output_dim(self, h, w): if isinstance(self.size, tuple) and len(self.size) == 2: return self.size elif h >= w: return int(h * self.size / w), self.size else: return self.size, int(w * self.size / h) def _get_video_dim(self, video_path): probe = ffmpeg.probe(video_path) video_stream = next( (stream for stream in probe["streams"] if stream["codec_type"] == "video"), None, ) width = int(video_stream["width"]) height = int(video_stream["height"]) num, denum = video_stream["avg_frame_rate"].split("/") frame_rate = int(num) / int(denum) return height, width, frame_rate def _getvideo(self, video_path): if os.path.isfile(video_path): print("Decoding video: {}".format(video_path)) try: h, w, fr = self._get_video_dim(video_path) except: print("ffprobe failed at: {}".format(video_path)) return { "video": th.zeros(1), "input": video_path } if fr < 1: print("Corrupted Frame Rate: {}".format(video_path)) return { "video": th.zeros(1), "input": video_path } height, width = self._get_output_dim(h, w) # resize ## try: cmd = ( ffmpeg.input(video_path) .filter("fps", fps=self.framerate) .filter("scale", width, height) ) if self.centercrop: x = int((width - self.size) / 2.0) y = int((height - self.size) / 2.0) cmd = cmd.crop(x, y, self.size, self.size) out, _ = cmd.output("pipe:", format="rawvideo", pix_fmt="rgb24").run( capture_stdout=True, quiet=True ) # try: # vr = VideoReader(video_path, ctx=cpu(0)) # video = vr.get_batch(range(0, len(vr), int(fr))).asnumpy() # video = np.array([cv2.resize(frame, (width, height)) for frame in video]) # if self.centercrop: # x = int((width - self.size) / 2.0) # y = int((height - self.size) / 2.0) # video = video[:, y:y+self.size, x:x+self.size, :] except: print("ffmpeg error at: {}".format(video_path)) return { "video": th.zeros(1), "input": video_path, } if self.centercrop and isinstance(self.size, int): height, width = self.size, self.size video = np.frombuffer(out, np.uint8).reshape([-1, height, width, 3]) video = th.from_numpy(video.astype("float32")) video = video.permute(0, 3, 1, 2) # t,c,h,w else: video = th.zeros(1) return {"video": video, "input": video_path} def __call__(self, video_path): video = self._getvideo(video_path)['video'] if len(video) > self.max_feats: sampled = [] for j in range(self.max_feats): sampled.append(video[(j * len(video)) // self.max_feats]) video = th.stack(sampled) video_len = self.max_feats elif len(video) < self.max_feats: video_len = len(video) video = th.cat( [video, th.zeros(self.max_feats - video_len, self.features_dim)], 0 ) video = self.preprocess(video) return video, video_len