1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
mod chunk_decoder;
mod player;
use std::{collections::hash_map::Entry, sync::Arc};
use ahash::HashMap;
use parking_lot::Mutex;
use crate::{
resource_managers::{GpuTexture2D, SourceImageDataFormat},
RenderContext,
};
use re_video::{decode::DecodeSettings, VideoData};
/// Error that can occur during playing videos.
#[derive(thiserror::Error, Debug, Clone)]
pub enum VideoPlayerError {
#[error("The decoder is lagging behind")]
EmptyBuffer,
#[error("Video seems to be empty, no segments have beem found.")]
EmptyVideo,
/// e.g. unsupported codec
#[error("Failed to create video chunk: {0}")]
CreateChunk(String),
/// e.g. unsupported codec
#[error("Failed to decode video chunk: {0}")]
DecodeChunk(String),
/// e.g. unsupported codec
#[error("Failed to decode video: {0}")]
Decoding(#[from] re_video::decode::Error),
#[error("The timestamp passed was negative.")]
NegativeTimestamp,
/// e.g. bad mp4, or bug in mp4 parse
#[error("Bad data.")]
BadData,
#[error("Failed to create gpu texture from decoded video data: {0}")]
ImageDataToTextureError(#[from] crate::resource_managers::ImageDataToTextureError),
}
pub type FrameDecodingResult = Result<VideoFrameTexture, VideoPlayerError>;
/// Information about the status of a frame decoding.
pub struct VideoFrameTexture {
/// The texture to show.
pub texture: GpuTexture2D,
/// If true, the texture is outdated. Keep polling for a fresh one.
pub is_pending: bool,
/// If true, this texture is so out-dated that it should have a loading spinner on top of it.
pub show_spinner: bool,
/// Format information about the original data from the video decoder.
///
/// The texture is already converted to something the renderer can use directly.
pub source_pixel_format: SourceImageDataFormat,
/// Meta information about the decoded frame.
pub frame_info: Option<re_video::decode::FrameInfo>,
}
/// Identifier for an independent video decoding stream.
///
/// A single video may use several decoders at a time to simultaneously decode frames at different timestamps.
/// The id does not need to be globally unique, just unique enough to distinguish streams of the same video.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct VideoPlayerStreamId(pub u64);
struct PlayerEntry {
player: player::VideoPlayer,
frame_index: u64,
}
/// Video data + decoder(s).
///
/// Supports asynchronously decoding video into GPU textures via [`Video::frame_at`].
pub struct Video {
debug_name: String,
data: Arc<re_video::VideoData>,
players: Mutex<HashMap<VideoPlayerStreamId, PlayerEntry>>,
decode_settings: DecodeSettings,
}
impl Video {
/// Loads a video from the given data.
///
/// Currently supports the following media types:
/// - `video/mp4`
pub fn load(debug_name: String, data: Arc<VideoData>, decode_settings: DecodeSettings) -> Self {
let players = Mutex::new(HashMap::default());
Self {
debug_name,
data,
players,
decode_settings,
}
}
/// The video data
#[inline]
pub fn data(&self) -> &Arc<re_video::VideoData> {
&self.data
}
/// Natural width of the video.
#[inline]
pub fn width(&self) -> u32 {
self.data.width()
}
/// Natural height of the video.
#[inline]
pub fn height(&self) -> u32 {
self.data.height()
}
/// Returns a texture with the latest frame at the given time since video start.
///
/// If the time is negative, a zeroed texture is returned.
///
/// This API is _asynchronous_, meaning that the decoder may not yet have decoded the frame
/// at the given timestamp. If the frame is not yet available, the returned texture will be
/// empty.
///
/// The time is specified in seconds since the start of the video.
pub fn frame_at(
&self,
render_context: &RenderContext,
player_stream_id: VideoPlayerStreamId,
time_since_video_start_in_seconds: f64,
video_data: &[u8],
) -> FrameDecodingResult {
re_tracing::profile_function!();
let global_frame_idx = render_context.active_frame_idx();
// We could protect this hashmap by a RwLock and the individual decoders by a Mutex.
// However, dealing with the RwLock efficiently is complicated:
// Upgradable-reads exclude other upgradable-reads which means that if an element is not found,
// we have to drop the unlock and relock with a write lock, during which new elements may be inserted.
// This can be overcome by looping until successful, or instead we can just use a single Mutex lock and leave it there.
let mut players = self.players.lock();
let decoder_entry = match players.entry(player_stream_id) {
Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
Entry::Vacant(vacant_entry) => {
let new_player = player::VideoPlayer::new(
&self.debug_name,
render_context,
self.data.clone(),
&self.decode_settings,
)?;
vacant_entry.insert(PlayerEntry {
player: new_player,
frame_index: global_frame_idx,
})
}
};
decoder_entry.frame_index = render_context.active_frame_idx();
decoder_entry.player.frame_at(
render_context,
time_since_video_start_in_seconds,
video_data,
)
}
/// Removes all decoders that have been unused in the last frame.
///
/// Decoders are very memory intensive, so they should be cleaned up as soon they're no longer needed.
pub fn purge_unused_decoders(&self, active_frame_idx: u64) {
if active_frame_idx == 0 {
return;
}
let mut players = self.players.lock();
players.retain(|_, decoder| decoder.frame_index >= active_frame_idx - 1);
}
}