1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
mod chunk_decoder;
mod player;

use std::{collections::hash_map::Entry, sync::Arc};

use ahash::HashMap;
use parking_lot::Mutex;

use crate::{
    resource_managers::{GpuTexture2D, SourceImageDataFormat},
    RenderContext,
};
use re_video::{decode::DecodeSettings, VideoData};

/// Error that can occur during playing videos.
#[derive(thiserror::Error, Debug, Clone)]
pub enum VideoPlayerError {
    #[error("The decoder is lagging behind")]
    EmptyBuffer,

    #[error("Video seems to be empty, no segments have beem found.")]
    EmptyVideo,

    /// e.g. unsupported codec
    #[error("Failed to create video chunk: {0}")]
    CreateChunk(String),

    /// e.g. unsupported codec
    #[error("Failed to decode video chunk: {0}")]
    DecodeChunk(String),

    /// e.g. unsupported codec
    #[error("Failed to decode video: {0}")]
    Decoding(#[from] re_video::decode::Error),

    #[error("The timestamp passed was negative.")]
    NegativeTimestamp,

    /// e.g. bad mp4, or bug in mp4 parse
    #[error("Bad data.")]
    BadData,

    #[error("Failed to create gpu texture from decoded video data: {0}")]
    ImageDataToTextureError(#[from] crate::resource_managers::ImageDataToTextureError),
}

pub type FrameDecodingResult = Result<VideoFrameTexture, VideoPlayerError>;

/// Information about the status of a frame decoding.
pub struct VideoFrameTexture {
    /// The texture to show.
    pub texture: GpuTexture2D,

    /// If true, the texture is outdated. Keep polling for a fresh one.
    pub is_pending: bool,

    /// If true, this texture is so out-dated that it should have a loading spinner on top of it.
    pub show_spinner: bool,

    /// Format information about the original data from the video decoder.
    ///
    /// The texture is already converted to something the renderer can use directly.
    pub source_pixel_format: SourceImageDataFormat,

    /// Meta information about the decoded frame.
    pub frame_info: Option<re_video::decode::FrameInfo>,
}

/// Identifier for an independent video decoding stream.
///
/// A single video may use several decoders at a time to simultaneously decode frames at different timestamps.
/// The id does not need to be globally unique, just unique enough to distinguish streams of the same video.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]

pub struct VideoPlayerStreamId(pub u64);

struct PlayerEntry {
    player: player::VideoPlayer,
    frame_index: u64,
}

/// Video data + decoder(s).
///
/// Supports asynchronously decoding video into GPU textures via [`Video::frame_at`].
pub struct Video {
    debug_name: String,
    data: Arc<re_video::VideoData>,
    players: Mutex<HashMap<VideoPlayerStreamId, PlayerEntry>>,
    decode_settings: DecodeSettings,
}

impl Video {
    /// Loads a video from the given data.
    ///
    /// Currently supports the following media types:
    /// - `video/mp4`
    pub fn load(debug_name: String, data: Arc<VideoData>, decode_settings: DecodeSettings) -> Self {
        let players = Mutex::new(HashMap::default());

        Self {
            debug_name,
            data,
            players,
            decode_settings,
        }
    }

    /// The video data
    #[inline]
    pub fn data(&self) -> &Arc<re_video::VideoData> {
        &self.data
    }

    /// Natural width of the video.
    #[inline]
    pub fn width(&self) -> u32 {
        self.data.width()
    }

    /// Natural height of the video.
    #[inline]
    pub fn height(&self) -> u32 {
        self.data.height()
    }

    /// Returns a texture with the latest frame at the given time since video start.
    ///
    /// If the time is negative, a zeroed texture is returned.
    ///
    /// This API is _asynchronous_, meaning that the decoder may not yet have decoded the frame
    /// at the given timestamp. If the frame is not yet available, the returned texture will be
    /// empty.
    ///
    /// The time is specified in seconds since the start of the video.
    pub fn frame_at(
        &self,
        render_context: &RenderContext,
        player_stream_id: VideoPlayerStreamId,
        time_since_video_start_in_seconds: f64,
        video_data: &[u8],
    ) -> FrameDecodingResult {
        re_tracing::profile_function!();

        let global_frame_idx = render_context.active_frame_idx();

        // We could protect this hashmap by a RwLock and the individual decoders by a Mutex.
        // However, dealing with the RwLock efficiently is complicated:
        // Upgradable-reads exclude other upgradable-reads which means that if an element is not found,
        // we have to drop the unlock and relock with a write lock, during which new elements may be inserted.
        // This can be overcome by looping until successful, or instead we can just use a single Mutex lock and leave it there.
        let mut players = self.players.lock();
        let decoder_entry = match players.entry(player_stream_id) {
            Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),
            Entry::Vacant(vacant_entry) => {
                let new_player = player::VideoPlayer::new(
                    &self.debug_name,
                    render_context,
                    self.data.clone(),
                    &self.decode_settings,
                )?;
                vacant_entry.insert(PlayerEntry {
                    player: new_player,
                    frame_index: global_frame_idx,
                })
            }
        };

        decoder_entry.frame_index = render_context.active_frame_idx();
        decoder_entry.player.frame_at(
            render_context,
            time_since_video_start_in_seconds,
            video_data,
        )
    }

    /// Removes all decoders that have been unused in the last frame.
    ///
    /// Decoders are very memory intensive, so they should be cleaned up as soon they're no longer needed.
    pub fn purge_unused_decoders(&self, active_frame_idx: u64) {
        if active_frame_idx == 0 {
            return;
        }

        let mut players = self.players.lock();
        players.retain(|_, decoder| decoder.frame_index >= active_frame_idx - 1);
    }
}