tangent128/webmetro
1
0
Fork 0
Code Issues 1 Pull requests Releases Packages Wiki Activity

Merge branch 'nightly' - async stable now

Browse source
This commit is contained in:
Tangent Wantwight 2019-11-17 22:44:13 -05:00
commit 77401e9f51
15 changed files with 1281 additions and 620 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1018
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

17
Cargo.toml
View file

@ -7,13 +7,14 @@ edition = "2018"
[dependencies]
bytes = "0.4.12"
clap = "2.33.0"
custom_error = "1.6.0"
futures = "0.1.28"
http = "0.1.17"
hyper = "0.12.31"
custom_error = "1.7"
futures = "0.1.29"
futures3 = { package = "futures-preview", version="0.3.0-alpha", features = ["compat"] }
http = "0.1.18"
hyper = "0.12.35"
hyper13 = { package = "hyper", version="0.13.0-alpha.4", features = ["unstable-stream"] }
matches = "0.1.8"
odds = { version = "0.3.1", features = ["std-vec"] }
tokio = "0.1.22"
tokio-codec = "0.1.1"
tokio-io = "0.1.12"
warp = "0.1.16"
tokio2 = { package = "tokio", version="0.2.0-alpha.6" }
warp = "0.1.20"
weak-table = "0.2.3"

58
src/channel.rs
View file

@ -1,25 +1,27 @@
use std::pin::Pin;
use std::task::{
Context,
Poll
};
use std::sync::{
Arc,
Mutex
};
use futures::{
Async,
AsyncSink,
Sink,
Stream,
sync::mpsc::{
use futures3::{
channel::mpsc::{
channel as mpsc_channel,
Sender,
Receiver
}
},
Sink,
Stream,
Never
};
use odds::vec::VecExt;
use crate::chunk::Chunk;
pub enum Never {}
/// A collection of listeners to a stream of WebM chunks.
/// Sending a chunk may fail due to a client being disconnected,
/// or simply failing to keep up with the stream buffer. In either
@ -55,11 +57,14 @@ impl Transmitter {
}
}
impl Sink for Transmitter {
type SinkItem = Chunk;
type SinkError = Never; // never errors, slow clients are simply dropped
impl Sink<Chunk> for Transmitter {
type Error = Never; // never errors, slow clients are simply dropped
fn start_send(&mut self, chunk: Chunk) -> Result<AsyncSink<Chunk>, Never> {
fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<(), Never>> {
Poll::Ready(Ok(()))
}
fn start_send(self: Pin<&mut Self>, chunk: Chunk) -> Result<(), Never> {
let mut channel = self.channel.lock().expect("Locking channel");
if let Chunk::Headers { .. } = chunk {
@ -68,14 +73,27 @@ impl Sink for Transmitter {
channel.listeners.retain_mut(|listener| listener.start_send(chunk.clone()).is_ok());
Ok(AsyncSink::Ready)
Ok(())
}
fn poll_complete(&mut self) -> Result<Async<()>, Never> {
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), Never>> {
let mut channel = self.channel.lock().expect("Locking channel");
let mut result = Poll::Ready(Ok(()));
channel.listeners.retain_mut(|listener| listener.poll_complete().is_ok());
// just disconnect any erroring listeners
channel.listeners.retain_mut(|listener| match Pin::new(listener).poll_flush(cx) {
Poll::Pending => {result = Poll::Pending; true},
Poll::Ready(Ok(())) => true,
Poll::Ready(Err(_)) => false,
});
Ok(Async::Ready(()))
result
}
fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), Never>> {
// don't actually disconnect listeners, since other sources may want to transmit to this channel;
// just ensure we've sent everything we can out
self.poll_flush(cx)
}
}
@ -108,9 +126,9 @@ impl Listener {
impl Stream for Listener {
type Item = Chunk;
type Error = Never; // no transmitter errors are exposed to the listeners
fn poll(&mut self) -> Result<Async<Option<Chunk>>, Never> {
Ok(self.receiver.poll().expect("Channel receiving can't error"))
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Chunk>> {
let receiver = &mut self.get_mut().receiver;
Pin::new(receiver).poll_next(cx)
}
}

210
src/chunk.rs
View file

@ -1,10 +1,12 @@
use bytes::Bytes;
use futures::{Async, Stream};
use bytes::{Buf, Bytes};
use futures3::prelude::*;
use std::{
io::Cursor,
mem
mem,
pin::Pin,
task::{Context, Poll, Poll::*},
};
use crate::ebml::EbmlEventSource;
use crate::stream_parser::EbmlStreamingParser;
use crate::error::WebmetroError;
use crate::webm::*;
@ -103,7 +105,7 @@ enum ChunkerState {
}
pub struct WebmChunker<S> {
source: S,
source: EbmlStreamingParser<S>,
buffer_size_limit: Option<usize>,
state: ChunkerState
}
@ -128,146 +130,144 @@ fn encode(element: WebmElement, buffer: &mut Cursor<Vec<u8>>, limit: Option<usiz
encode_webm_element(element, buffer).map_err(|err| err.into())
}
impl<S: EbmlEventSource> Stream for WebmChunker<S>
where S::Error: Into<WebmetroError>
impl<I: Buf, S: Stream<Item = Result<I, WebmetroError>> + Unpin> Stream for WebmChunker<S>
{
type Item = Chunk;
type Error = WebmetroError;
type Item = Result<Chunk, WebmetroError>;
fn poll(&mut self) -> Result<Async<Option<Self::Item>>, WebmetroError> {
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Chunk, WebmetroError>>> {
let mut chunker = self.get_mut();
loop {
let mut return_value = None;
let mut new_state = None;
match self.state {
match chunker.state {
ChunkerState::BuildingHeader(ref mut buffer) => {
match self.source.poll_event() {
Err(passthru) => return Err(passthru.into()),
Ok(Async::NotReady) => return Ok(Async::NotReady),
Ok(Async::Ready(None)) => return Ok(Async::Ready(None)),
Ok(Async::Ready(Some(WebmElement::Cluster))) => {
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
let header_chunk = Chunk::Headers {bytes: Bytes::from(liberated_buffer.into_inner())};
match chunker.source.poll_event(cx) {
Ready(Some(Err(passthru))) => return Ready(Some(Err(passthru))),
Pending => return Pending,
Ready(None) => return Ready(None),
Ready(Some(Ok(element))) => match element {
WebmElement::Cluster => {
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
let header_chunk = Chunk::Headers {bytes: Bytes::from(liberated_buffer.into_inner())};
return_value = Some(Ok(Async::Ready(Some(header_chunk))));
new_state = Some(ChunkerState::BuildingCluster(
ClusterHead::new(0),
Cursor::new(Vec::new())
));
},
Ok(Async::Ready(Some(WebmElement::Info))) => {},
Ok(Async::Ready(Some(WebmElement::Void))) => {},
Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {},
Ok(Async::Ready(Some(element))) => {
encode(element, buffer, self.buffer_size_limit).unwrap_or_else(|err| {
return_value = Some(Err(err));
new_state = Some(ChunkerState::End);
});
chunker.state = ChunkerState::BuildingCluster(
ClusterHead::new(0),
Cursor::new(Vec::new())
);
return Ready(Some(Ok(header_chunk)));
},
WebmElement::Info => {},
WebmElement::Void => {},
WebmElement::Unknown(_) => {},
element => {
if let Err(err) = encode(element, buffer, chunker.buffer_size_limit) {
chunker.state = ChunkerState::End;
return Ready(Some(Err(err)));
}
}
}
}
},
ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => {
match self.source.poll_event() {
Err(passthru) => return Err(passthru.into()),
Ok(Async::NotReady) => return Ok(Async::NotReady),
Ok(Async::Ready(Some(element @ WebmElement::EbmlHead)))
| Ok(Async::Ready(Some(element @ WebmElement::Segment))) => {
let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0));
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
match chunker.source.poll_event(cx) {
Ready(Some(Err(passthru))) => return Ready(Some(Err(passthru))),
Pending => return Pending,
Ready(Some(Ok(element))) => match element {
WebmElement::EbmlHead | WebmElement::Segment => {
let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0));
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
let mut new_header_cursor = Cursor::new(Vec::new());
match encode(element, &mut new_header_cursor, self.buffer_size_limit) {
Ok(_) => {
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))));
new_state = Some(ChunkerState::EmittingClusterBodyBeforeNewHeader{
body: liberated_buffer.into_inner(),
new_header: new_header_cursor
});
},
Err(err) => {
return_value = Some(Err(err));
new_state = Some(ChunkerState::End);
let mut new_header_cursor = Cursor::new(Vec::new());
match encode(element, &mut new_header_cursor, chunker.buffer_size_limit) {
Ok(_) => {
chunker.state = ChunkerState::EmittingClusterBodyBeforeNewHeader{
body: liberated_buffer.into_inner(),
new_header: new_header_cursor
};
return Ready(Some(Ok(Chunk::ClusterHead(liberated_cluster_head))));
},
Err(err) => {
chunker.state = ChunkerState::End;
return Ready(Some(Err(err)));
}
}
}
}
Ok(Async::Ready(Some(WebmElement::Cluster))) => {
let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0));
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
},
WebmElement::Cluster => {
let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0));
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))));
new_state = Some(ChunkerState::EmittingClusterBody(liberated_buffer.into_inner()));
chunker.state = ChunkerState::EmittingClusterBody(liberated_buffer.into_inner());
return Ready(Some(Ok(Chunk::ClusterHead(liberated_cluster_head))));
},
WebmElement::Timecode(timecode) => {
cluster_head.update_timecode(timecode);
},
WebmElement::SimpleBlock(ref block) => {
if (block.flags & 0b10000000) != 0 {
// TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster
cluster_head.keyframe = true;
}
cluster_head.observe_simpleblock_timecode(block.timecode);
if let Err(err) = encode(WebmElement::SimpleBlock(*block), buffer, chunker.buffer_size_limit) {
chunker.state = ChunkerState::End;
return Ready(Some(Err(err)));
}
},
WebmElement::Info => {},
WebmElement::Void => {},
WebmElement::Unknown(_) => {},
element => {
if let Err(err) = encode(element, buffer, chunker.buffer_size_limit) {
chunker.state = ChunkerState::End;
return Ready(Some(Err(err)));
}
},
},
Ok(Async::Ready(Some(WebmElement::Timecode(timecode)))) => {
cluster_head.update_timecode(timecode);
},
Ok(Async::Ready(Some(WebmElement::SimpleBlock(ref block)))) => {
if (block.flags & 0b10000000) != 0 {
// TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster
cluster_head.keyframe = true;
}
cluster_head.observe_simpleblock_timecode(block.timecode);
encode(WebmElement::SimpleBlock(*block), buffer, self.buffer_size_limit).unwrap_or_else(|err| {
return_value = Some(Err(err));
new_state = Some(ChunkerState::End);
});
},
Ok(Async::Ready(Some(WebmElement::Info))) => {},
Ok(Async::Ready(Some(WebmElement::Void))) => {},
Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {},
Ok(Async::Ready(Some(element))) => {
encode(element, buffer, self.buffer_size_limit).unwrap_or_else(|err| {
return_value = Some(Err(err));
new_state = Some(ChunkerState::End);
});
},
Ok(Async::Ready(None)) => {
Ready(None) => {
// flush final Cluster on end of stream
let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0));
let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new()));
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))));
new_state = Some(ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner()));
chunker.state = ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner());
return Ready(Some(Ok(Chunk::ClusterHead(liberated_cluster_head))));
}
}
},
ChunkerState::EmittingClusterBody(ref mut buffer) => {
let liberated_buffer = mem::replace(buffer, Vec::new());
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Bytes::from(liberated_buffer)}))));
new_state = Some(ChunkerState::BuildingCluster(
chunker.state = ChunkerState::BuildingCluster(
ClusterHead::new(0),
Cursor::new(Vec::new())
));
);
return Ready(Some(Ok(Chunk::ClusterBody {bytes: Bytes::from(liberated_buffer)})));
},
ChunkerState::EmittingClusterBodyBeforeNewHeader { ref mut body, ref mut new_header } => {
let liberated_body = mem::replace(body, Vec::new());
let liberated_header_cursor = mem::replace(new_header, Cursor::new(Vec::new()));
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Bytes::from(liberated_body)}))));
new_state = Some(ChunkerState::BuildingHeader(liberated_header_cursor));
chunker.state = ChunkerState::BuildingHeader(liberated_header_cursor);
return Ready(Some(Ok(Chunk::ClusterBody {bytes: Bytes::from(liberated_body)})));
},
ChunkerState::EmittingFinalClusterBody(ref mut buffer) => {
// flush final Cluster on end of stream
let liberated_buffer = mem::replace(buffer, Vec::new());
return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Bytes::from(liberated_buffer)}))));
new_state = Some(ChunkerState::End);
chunker.state = ChunkerState::End;
return Ready(Some(Ok(Chunk::ClusterBody {bytes: Bytes::from(liberated_buffer)})));
},
ChunkerState::End => return Ok(Async::Ready(None))
ChunkerState::End => return Ready(None)
};
if let Some(new_state) = new_state {
self.state = new_state;
}
if let Some(return_value) = return_value {
return return_value;
}
}
}
}
pub trait WebmStream where Self: Sized + EbmlEventSource {
fn chunk_webm(self) -> WebmChunker<Self> {
pub trait WebmStream {
type Stream;
fn chunk_webm(self) -> WebmChunker<Self::Stream>;
}
impl<S: Stream> WebmStream for EbmlStreamingParser<S> {
type Stream = S;
fn chunk_webm(self) -> WebmChunker<S> {
WebmChunker {
source: self,
buffer_size_limit: None,
@ -276,8 +276,6 @@ pub trait WebmStream where Self: Sized + EbmlEventSource {
}
}
impl<T: EbmlEventSource> WebmStream for T {}
#[cfg(test)]
mod tests {

22
src/commands/dump.rs
View file

@ -1,5 +1,5 @@
use clap::{App, AppSettings, ArgMatches, SubCommand};
use futures::prelude::*;
use tokio2::runtime::Runtime;
use super::stdin_stream;
use webmetro::{
@ -21,15 +21,15 @@ pub fn run(_args: &ArgMatches) -> Result<(), WebmetroError> {
let mut events = stdin_stream().parse_ebml();
// stdin is sync so Async::NotReady will never happen
while let Ok(Async::Ready(Some(element))) = events.poll_event() {
match element {
// suppress printing byte arrays
Tracks(slice) => println!("Tracks[{}]", slice.len()),
SimpleBlock(SimpleBlock {timecode, ..}) => println!("SimpleBlock@{}", timecode),
other => println!("{:?}", other)
Runtime::new().unwrap().block_on(async {
while let Some(element) = events.next().await? {
match element {
// suppress printing byte arrays
Tracks(slice) => println!("Tracks[{}]", slice.len()),
SimpleBlock(SimpleBlock {timecode, ..}) => println!("SimpleBlock@{}", timecode),
other => println!("{:?}", other)
}
}
}
Ok(())
Ok(())
})
}

21
src/commands/filter.rs
View file

@ -4,8 +4,9 @@ use std::{
};
use clap::{App, Arg, ArgMatches, SubCommand};
use futures::prelude::*;
use tokio::runtime::Runtime;
use futures3::prelude::*;
use futures3::future::ready;
use tokio2::runtime::Runtime;
use super::stdin_stream;
use webmetro::{
@ -14,7 +15,10 @@ use webmetro::{
WebmStream
},
error::WebmetroError,
fixers::ChunkStream,
fixers::{
ChunkTimecodeFixer,
Throttle,
},
stream_parser::StreamEbml
};
@ -27,18 +31,19 @@ pub fn options() -> App<'static, 'static> {
}
pub fn run(args: &ArgMatches) -> Result<(), WebmetroError> {
let mut chunk_stream: Box<Stream<Item = Chunk, Error = WebmetroError> + Send> = Box::new(
let mut timecode_fixer = ChunkTimecodeFixer::new();
let mut chunk_stream: Box<dyn TryStream<Item = Result<Chunk, WebmetroError>, Ok = Chunk, Error = WebmetroError> + Send + Unpin> = Box::new(
stdin_stream()
.parse_ebml()
.chunk_webm()
.fix_timecodes()
.map_ok(move |chunk| timecode_fixer.process(chunk))
);
if args.is_present("throttle") {
chunk_stream = Box::new(chunk_stream.throttle());
chunk_stream = Box::new(Throttle::new(chunk_stream));
}
Runtime::new().unwrap().block_on(chunk_stream.for_each(|chunk| {
io::stdout().write_all(chunk.as_ref()).map_err(WebmetroError::from)
Runtime::new().unwrap().block_on(chunk_stream.try_for_each(|chunk| {
ready(io::stdout().write_all(chunk.as_ref()).map_err(WebmetroError::from))
}))
}

27
src/commands/mod.rs
View file

@ -1,15 +1,7 @@
use std::io::stdin;
use std::io::Cursor;
use bytes::{
Buf,
IntoBuf
};
use futures::prelude::*;
use tokio_io::io::AllowStdIo;
use tokio_codec::{
BytesCodec,
FramedRead
};
use bytes::Bytes;
use futures3::TryStreamExt;
use webmetro::error::WebmetroError;
pub mod dump;
@ -20,8 +12,13 @@ pub mod send;
/// An adapter that makes chunks of bytes from stdin available as a Stream;
/// is NOT actually async, and just uses blocking read. Don't use more than
/// one at once, who knows who gets which bytes.
pub fn stdin_stream() -> impl Stream<Item = impl Buf, Error = WebmetroError> {
FramedRead::new(AllowStdIo::new(stdin()), BytesCodec::new())
.map(|bytes| bytes.into_buf())
.map_err(WebmetroError::from)
pub fn stdin_stream() -> impl futures3::TryStream<
Item = Result<Cursor<Bytes>, WebmetroError>,
Ok = Cursor<Bytes>,
Error = WebmetroError,
> + Sized
+ Unpin {
tokio2::codec::FramedRead::new(tokio2::io::stdin(), tokio2::codec::BytesCodec::new())
.map_ok(|bytes| Cursor::new(bytes.freeze()))
.map_err(WebmetroError::from)
}

31
src/commands/relay.rs
View file

@ -13,6 +13,15 @@ use futures::{
Sink,
stream::empty
};
use futures3::{
compat::{
Compat,
CompatSink,
Compat01As03,
},
Never,
prelude::*,
};
use hyper::{
Body,
Response,
@ -38,28 +47,32 @@ use webmetro::{
},
chunk::WebmStream,
error::WebmetroError,
fixers::ChunkStream,
fixers::{
ChunkStream,
ChunkTimecodeFixer,
},
stream_parser::StreamEbml
};
const BUFFER_LIMIT: usize = 2 * 1024 * 1024;
fn get_stream(channel: Handle) -> impl Stream<Item = Bytes, Error = WebmetroError> {
Listener::new(channel)
.fix_timecodes()
let mut timecode_fixer = ChunkTimecodeFixer::new();
Compat::new(Listener::new(channel).map(|c| Ok(c))
.map_ok(move |chunk| timecode_fixer.process(chunk))
.find_starting_point()
.map(|webm_chunk| webm_chunk.into_bytes())
.map_err(|err| match err {})
.map_ok(|webm_chunk| webm_chunk.into_bytes())
.map_err(|err: Never| match err {}))
}
fn post_stream(channel: Handle, stream: impl Stream<Item = impl Buf, Error = warp::Error>) -> impl Stream<Item = Bytes, Error = WebmetroError> {
let source = stream
.map_err(WebmetroError::from)
let source = Compat01As03::new(stream
.map_err(WebmetroError::from))
.parse_ebml().with_soft_limit(BUFFER_LIMIT)
.chunk_webm().with_soft_limit(BUFFER_LIMIT);
let sink = Transmitter::new(channel);
let sink = CompatSink::new(Transmitter::new(channel));
source.forward(sink.sink_map_err(|err| -> WebmetroError {match err {}}))
Compat::new(source).forward(sink.sink_map_err(|err| -> WebmetroError {match err {}}))
.into_stream()
.map(|_| empty())
.map_err(|err| {

81
src/commands/send.rs
View file

@ -1,26 +1,15 @@
use clap::{App, Arg, ArgMatches, SubCommand};
use futures::{
prelude::*
};
use hyper::{
Body,
Client,
client::HttpConnector,
Request
};
use tokio::runtime::Runtime;
use futures3::prelude::*;
use hyper13::{client::HttpConnector, Body, Client, Request};
use std::io::{stdout, Write};
use tokio2::runtime::Runtime;
use super::{
stdin_stream
};
use super::stdin_stream;
use webmetro::{
chunk::{
Chunk,
WebmStream
},
chunk::{Chunk, WebmStream},
error::WebmetroError,
fixers::ChunkStream,
stream_parser::StreamEbml
fixers::{ChunkTimecodeFixer, Throttle},
stream_parser::StreamEbml,
};
pub fn options() -> App<'static, 'static> {
@ -34,45 +23,49 @@ pub fn options() -> App<'static, 'static> {
.help("Slow down upload to \"real time\" speed as determined by the timestamps (useful for streaming static files)"))
}
type BoxedChunkStream = Box<Stream<Item = Chunk, Error = WebmetroError> + Send>;
type BoxedChunkStream = Box<
dyn TryStream<Item = Result<Chunk, WebmetroError>, Ok = Chunk, Error = WebmetroError>
+ Send
+ Sync
+ Unpin,
>;
pub fn run(args: &ArgMatches) -> Result<(), WebmetroError> {
let mut timecode_fixer = ChunkTimecodeFixer::new();
let mut chunk_stream: BoxedChunkStream = Box::new(
stdin_stream()
.parse_ebml()
.chunk_webm()
.fix_timecodes()
.parse_ebml()
.chunk_webm()
.map_ok(move |chunk| timecode_fixer.process(chunk)),
);
let url_str = match args.value_of("url") {
Some(url) => String::from(url),
_ => return Err("Listen address wasn't provided".into())
_ => return Err("Listen address wasn't provided".into()),
};
if args.is_present("throttle") {
chunk_stream = Box::new(chunk_stream.throttle());
chunk_stream = Box::new(Throttle::new(chunk_stream));
}
let request_payload = Body::wrap_stream(chunk_stream.map(
|webm_chunk| webm_chunk.into_bytes()
).map_err(|err| {
eprintln!("{}", &err);
err
}));
let chunk_stream = chunk_stream
.map_ok(|webm_chunk| webm_chunk.into_bytes())
.map_err(|err| {
eprintln!("{}", &err);
err
});
let request = Request::put(url_str)
.body(request_payload)
.map_err(WebmetroError::from)?;
let request_payload = Body::wrap_stream(chunk_stream);
let client = Client::builder().build(HttpConnector::new(1));
let future = client.request(request)
.and_then(|response| {
response.into_body().for_each(|_chunk| {
Ok(())
})
let request = Request::put(url_str).body(request_payload)?;
let client = Client::builder().build(HttpConnector::new());
Runtime::new().unwrap().block_on(async {
let response = client.request(request).await?;
let mut response_stream = response.into_body();
while let Some(response_chunk) = response_stream.next().await.transpose()? {
stdout().write_all(&response_chunk)?;
}
Ok(())
})
.map_err(WebmetroError::from);
Runtime::new().unwrap().block_on(future)
}

56
src/ebml.rs
View file

@ -1,7 +1,6 @@
use bytes::{BigEndian, ByteOrder, BufMut};
use custom_error::custom_error;
use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom};
use futures::Async;
pub const EBML_HEAD_ID: u64 = 0x0A45DFA3;
pub const DOC_TYPE_ID: u64 = 0x0282;
@ -197,6 +196,12 @@ pub fn encode_integer<T: Write>(tag: u64, value: u64, output: &mut T) -> IoResul
output.write_all(&buffer.get_ref()[..])
}
pub struct EbmlLayout {
pub element_id: u64,
pub body_offset: usize,
pub element_len: usize,
}
pub trait FromEbml<'a>: Sized {
/// Indicates if this tag's contents should be treated as a blob,
/// or if the tag header should be reported as an event and with further
@ -210,13 +215,14 @@ pub trait FromEbml<'a>: Sized {
/// references into the given buffer.
fn decode(element_id: u64, bytes: &'a[u8]) -> Result<Self, EbmlError>;
/// Check if enough space exists in the given buffer for decode_element() to
/// be successful; parsing errors will be returned eagerly.
fn check_space(bytes: &[u8]) -> Result<Option<usize>, EbmlError> {
/// Check if enough space exists in the given buffer to decode an element;
/// it will not actually call `decode` or try to construct an instance,
/// but EBML errors with the next tag header will be returned eagerly.
fn check_space(bytes: &[u8]) -> Result<Option<EbmlLayout>, EbmlError> {
match decode_tag(bytes) {
Ok(None) => Ok(None),
Err(err) => Err(err),
Ok(Some((element_id, payload_size_tag, tag_size))) => {
Ok(Some((element_id, payload_size_tag, body_offset))) => {
let should_unwrap = Self::should_unwrap(element_id);
let payload_size = match (should_unwrap, payload_size_tag) {
@ -225,12 +231,16 @@ pub trait FromEbml<'a>: Sized {
(false, Varint::Value(size)) => size as usize
};
let element_size = tag_size + payload_size;
if element_size > bytes.len() {
let element_len = body_offset + payload_size;
if element_len > bytes.len() {
// need to read more still
Ok(None)
} else {
Ok(Some(element_size))
Ok(Some(EbmlLayout {
element_id,
body_offset,
element_len
}))
}
}
}
@ -238,26 +248,11 @@ pub trait FromEbml<'a>: Sized {
/// Attempt to construct an instance of this type from the given byte slice
fn decode_element(bytes: &'a[u8]) -> Result<Option<(Self, usize)>, EbmlError> {
match decode_tag(bytes) {
Ok(None) => Ok(None),
Err(err) => Err(err),
Ok(Some((element_id, payload_size_tag, tag_size))) => {
let should_unwrap = Self::should_unwrap(element_id);
let payload_size = match (should_unwrap, payload_size_tag) {
(true, _) => 0,
(false, Varint::Unknown) => return Err(EbmlError::UnknownElementLength),
(false, Varint::Value(size)) => size as usize
};
let element_size = tag_size + payload_size;
if element_size > bytes.len() {
// need to read more still
return Ok(None);
}
match Self::decode(element_id, &bytes[tag_size..element_size]) {
Ok(element) => Ok(Some((element, element_size))),
match Self::check_space(bytes)? {
None => Ok(None),
Some(info) => {
match Self::decode(info.element_id, &bytes[info.body_offset..info.element_len]) {
Ok(element) => Ok(Some((element, info.element_len))),
Err(error) => Err(error)
}
}
@ -265,11 +260,6 @@ pub trait FromEbml<'a>: Sized {
}
}
pub trait EbmlEventSource {
type Error;
fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result<Async<Option<T>>, Self::Error>;
}
#[cfg(test)]
mod tests {
use bytes::{BytesMut};

12
src/error.rs
View file

@ -3,12 +3,12 @@ use custom_error::custom_error;
custom_error!{pub WebmetroError
ResourcesExceeded = "resources exceeded",
EbmlError{source: crate::ebml::EbmlError} = "EBML error",
HttpError{source: http::Error} = "HTTP error",
HyperError{source: hyper::Error} = "Hyper error",
IoError{source: std::io::Error} = "IO error",
TimerError{source: tokio::timer::Error} = "Timer error",
WarpError{source: warp::Error} = "Warp error",
EbmlError{source: crate::ebml::EbmlError} = "EBML error: {source}",
HttpError{source: http::Error} = "HTTP error: {source}",
HyperError{source: hyper::Error} = "Hyper error: {source}",
Hyper13Error{source: hyper13::Error} = "Hyper error: {source}",
IoError{source: std::io::Error} = "IO error: {source}",
WarpError{source: warp::Error} = "Warp error: {source}",
ApplicationError{message: String} = "{message}"
}

118
src/fixers.rs
View file

@ -1,27 +1,36 @@
use std::pin::Pin;
use std::task::{
Context,
Poll
};
use std::time::{Duration, Instant};
use futures::prelude::*;
use tokio::timer::Delay;
use futures3::prelude::*;
use tokio2::timer::{
delay,
Delay
};
use crate::chunk::Chunk;
use crate::error::WebmetroError;
pub struct ChunkTimecodeFixer<S> {
stream: S,
pub struct ChunkTimecodeFixer {
current_offset: u64,
last_observed_timecode: u64,
assumed_duration: u64
}
impl<S: Stream<Item = Chunk>> Stream for ChunkTimecodeFixer<S>
{
type Item = S::Item;
type Error = S::Error;
fn poll(&mut self) -> Result<Async<Option<Self::Item>>, Self::Error> {
let mut poll_chunk = self.stream.poll();
match poll_chunk {
Ok(Async::Ready(Some(Chunk::ClusterHead(ref mut cluster_head)))) => {
impl ChunkTimecodeFixer {
pub fn new() -> ChunkTimecodeFixer {
ChunkTimecodeFixer {
current_offset: 0,
last_observed_timecode: 0,
assumed_duration: 33
}
}
pub fn process<'a>(&mut self, mut chunk: Chunk) -> Chunk {
match chunk {
Chunk::ClusterHead(ref mut cluster_head) => {
let start = cluster_head.start;
if start < self.last_observed_timecode {
let next_timecode = self.last_observed_timecode + self.assumed_duration;
@ -30,10 +39,10 @@ impl<S: Stream<Item = Chunk>> Stream for ChunkTimecodeFixer<S>
cluster_head.update_timecode(start + self.current_offset);
self.last_observed_timecode = cluster_head.end;
},
}
_ => {}
};
poll_chunk
}
chunk
}
}
@ -43,33 +52,32 @@ pub struct StartingPointFinder<S> {
seen_keyframe: bool
}
impl<S: Stream<Item = Chunk>> Stream for StartingPointFinder<S>
impl<S: TryStream<Ok = Chunk> + Unpin> Stream for StartingPointFinder<S>
{
type Item = S::Item;
type Error = S::Error;
type Item = Result<Chunk, S::Error>;
fn poll(&mut self) -> Result<Async<Option<Self::Item>>, Self::Error> {
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Chunk, S::Error>>> {
loop {
return match self.stream.poll() {
Ok(Async::Ready(Some(Chunk::ClusterHead(cluster_head)))) => {
return match self.stream.try_poll_next_unpin(cx) {
Poll::Ready(Some(Ok(Chunk::ClusterHead(cluster_head)))) => {
if cluster_head.keyframe {
self.seen_keyframe = true;
}
if self.seen_keyframe {
Ok(Async::Ready(Some(Chunk::ClusterHead(cluster_head))))
Poll::Ready(Some(Ok(Chunk::ClusterHead(cluster_head))))
} else {
continue;
}
},
chunk @ Ok(Async::Ready(Some(Chunk::ClusterBody {..}))) => {
chunk @ Poll::Ready(Some(Ok(Chunk::ClusterBody {..}))) => {
if self.seen_keyframe {
chunk
} else {
continue;
}
},
chunk @ Ok(Async::Ready(Some(Chunk::Headers {..}))) => {
chunk @ Poll::Ready(Some(Ok(Chunk::Headers {..}))) => {
if self.seen_header {
// new stream starting, we don't need a new header but should wait for a safe spot to resume
self.seen_keyframe = false;
@ -91,37 +99,46 @@ pub struct Throttle<S> {
sleep: Delay
}
impl<S: Stream<Item = Chunk, Error = WebmetroError>> Stream for Throttle<S>
{
type Item = S::Item;
type Error = WebmetroError;
impl<S> Throttle<S> {
pub fn new(wrap: S) -> Throttle<S> {
let now = Instant::now();
Throttle {
stream: wrap,
start_time: now,
sleep: delay(now)
}
}
}
fn poll(&mut self) -> Result<Async<Option<Self::Item>>, WebmetroError> {
match self.sleep.poll() {
Err(err) => return Err(err.into()),
Ok(Async::NotReady) => return Ok(Async::NotReady),
Ok(Async::Ready(())) => { /* can continue */ }
impl<S: TryStream<Ok = Chunk, Error = WebmetroError> + Unpin> Stream for Throttle<S>
{
type Item = Result<Chunk, WebmetroError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Result<Chunk, WebmetroError>>> {
match self.sleep.poll_unpin(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(()) => { /* can continue */ },
}
let next_chunk = self.stream.poll();
if let Ok(Async::Ready(Some(Chunk::ClusterHead(ref cluster_head)))) = next_chunk {
let next_chunk = self.stream.try_poll_next_unpin(cx);
if let Poll::Ready(Some(Ok(Chunk::ClusterHead(ref cluster_head)))) = next_chunk {
// snooze until real time has "caught up" to the stream
let offset = Duration::from_millis(cluster_head.end);
self.sleep.reset(self.start_time + offset);
let sleep_until = self.start_time + offset;
self.sleep.reset(sleep_until);
}
next_chunk
}
}
pub trait ChunkStream where Self : Sized + Stream<Item = Chunk> {
fn fix_timecodes(self) -> ChunkTimecodeFixer<Self> {
ChunkTimecodeFixer {
stream: self,
current_offset: 0,
last_observed_timecode: 0,
assumed_duration: 33
}
}
pub trait ChunkStream where Self : Sized + TryStream<Ok = Chunk> {
/*fn fix_timecodes(self) -> Map<_> {
let fixer = ;
self.map(move |chunk| {
fixer.process(chunk);
chunk
})
}*/
fn find_starting_point(self) -> StartingPointFinder<Self> {
StartingPointFinder {
@ -132,13 +149,8 @@ pub trait ChunkStream where Self : Sized + Stream<Item = Chunk> {
}
fn throttle(self) -> Throttle<Self> {
let now = Instant::now();
Throttle {
stream: self,
start_time: now,
sleep: Delay::new(now)
}
Throttle::new(self)
}
}
impl<T: Stream<Item = Chunk>> ChunkStream for T {}
impl<T: TryStream<Ok = Chunk>> ChunkStream for T {}

14
src/lib.rs
View file

@ -1,8 +1,8 @@
pub mod ebml;
pub mod error;
pub mod iterator;
pub mod slice;
pub mod stream_parser;
pub mod chunk;
@ -15,18 +15,6 @@ pub use crate::ebml::{EbmlError, FromEbml};
#[cfg(test)]
mod tests {
use futures::future::{ok, Future};
pub const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm");
pub const ENCODE_WEBM_TEST_FILE: &'static [u8] = include_bytes!("data/encode_webm_test.webm");
#[test]
fn hello_futures() {
let my_future = ok::<String, ()>("Hello".into())
.map(|hello| hello + ", Futures!");
let string_result = my_future.wait().unwrap();
assert_eq!(string_result, "Hello, Futures!");
}
}

18
src/slice.rs
View file

@ -1,18 +0,0 @@
use futures::Async;
use crate::ebml::EbmlError;
use crate::ebml::EbmlEventSource;
use crate::ebml::FromEbml;
pub struct EbmlSlice<'a>(pub &'a [u8]);
impl<'b> EbmlEventSource for EbmlSlice<'b> {
type Error = EbmlError;
fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result<Async<Option<T>>, EbmlError> {
T::decode_element(self.0).map(|option| option.map(|(element, element_size)| {
self.0 = &self.0[element_size..];
element
})).map(Async::Ready)
}
}

198
src/stream_parser.rs
View file

@ -1,24 +1,15 @@
use bytes::{
Buf,
BufMut,
BytesMut
};
use futures::{
Async,
stream::Stream
};
use bytes::{Buf, BufMut, Bytes, BytesMut};
use futures3::stream::{Stream, StreamExt, TryStream};
use std::task::{Context, Poll};
use crate::ebml::{
EbmlEventSource,
FromEbml
};
use crate::ebml::FromEbml;
use crate::error::WebmetroError;
pub struct EbmlStreamingParser<S> {
stream: S,
buffer: BytesMut,
buffer_size_limit: Option<usize>,
last_read: usize
borrowed: Bytes,
}
impl<S> EbmlStreamingParser<S> {
@ -31,70 +22,183 @@ impl<S> EbmlStreamingParser<S> {
}
}
pub trait StreamEbml where Self: Sized + Stream, Self::Item: Buf {
pub trait StreamEbml: Sized + TryStream + Unpin
where
Self: Sized + TryStream + Unpin,
Self::Ok: Buf,
{
fn parse_ebml(self) -> EbmlStreamingParser<Self> {
EbmlStreamingParser {
stream: self,
buffer: BytesMut::new(),
buffer_size_limit: None,
last_read: 0
borrowed: Bytes::new(),
}
}
}
impl<I: Buf, S: Stream<Item = I, Error = WebmetroError>> StreamEbml for S {}
impl<I: Buf, S: Stream<Item = I, Error = WebmetroError>> EbmlStreamingParser<S> {
pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result<Async<Option<T>>, WebmetroError> {
// release buffer from previous event
self.buffer.advance(self.last_read);
self.last_read = 0;
impl<I: Buf, S: Stream<Item = Result<I, WebmetroError>> + Unpin> StreamEbml for S {}
impl<I: Buf, S: Stream<Item = Result<I, WebmetroError>> + Unpin> EbmlStreamingParser<S> {
pub fn poll_event<'a, T: FromEbml<'a>>(
&'a mut self,
cx: &mut Context,
) -> Poll<Option<Result<T, WebmetroError>>> {
loop {
match T::check_space(&self.buffer) {
Ok(None) => {
match T::check_space(&self.buffer)? {
None => {
// need to refill buffer, below
},
other => return other.map_err(WebmetroError::from).and_then(move |_| {
match T::decode_element(&self.buffer) {
Err(err) => Err(err.into()),
Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."),
Ok(Some((element, element_size))) => {
self.last_read = element_size;
Ok(Async::Ready(Some(element)))
}
}
})
}
Some(info) => {
let mut bytes = self.buffer.split_to(info.element_len).freeze();
bytes.advance(info.body_offset);
self.borrowed = bytes;
return Poll::Ready(Some(T::decode(
info.element_id,
&self.borrowed,
).map_err(Into::into)));
}
}
if let Some(limit) = self.buffer_size_limit {
if limit <= self.buffer.len() {
return Err(WebmetroError::ResourcesExceeded);
return Poll::Ready(Some(Err(WebmetroError::ResourcesExceeded)));
}
}
match self.stream.poll() {
Ok(Async::Ready(Some(buf))) => {
match self.stream.poll_next_unpin(cx)? {
Poll::Ready(Some(buf)) => {
self.buffer.reserve(buf.remaining());
self.buffer.put(buf);
// ok can retry decoding now
},
other => return other.map(|async_status| async_status.map(|_| None))
}
Poll::Ready(None) => return Poll::Ready(None),
Poll::Pending => return Poll::Pending,
}
}
}
}
impl<I: Buf, S: Stream<Item = I, Error = WebmetroError>> EbmlEventSource for EbmlStreamingParser<S> {
type Error = WebmetroError;
impl<I: Buf, S: Stream<Item = Result<I, WebmetroError>> + Unpin> EbmlStreamingParser<S> {
pub async fn next<'a, T: FromEbml<'a>>(&'a mut self) -> Result<Option<T>, WebmetroError> {
loop {
if let Some(info) = T::check_space(&self.buffer)? {
let mut bytes = self.buffer.split_to(info.element_len).freeze();
bytes.advance(info.body_offset);
self.borrowed = bytes;
return Ok(Some(T::decode(info.element_id, &self.borrowed)?));
}
fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result<Async<Option<T>>, WebmetroError> {
return EbmlStreamingParser::poll_event(self);
if let Some(limit) = self.buffer_size_limit {
if limit <= self.buffer.len() {
// hit our buffer limit and still nothing parsed
return Err(WebmetroError::ResourcesExceeded);
}
}
match self.stream.next().await.transpose()? {
Some(refill) => {
self.buffer.reserve(refill.remaining());
self.buffer.put(refill);
}
None => {
// Nothing left, we're done
return Ok(None);
}
}
}
}
}
#[cfg(test)]
mod tests {
//#[test]
use bytes::IntoBuf;
use futures3::{future::poll_fn, stream::StreamExt, FutureExt};
use matches::assert_matches;
use std::task::Poll::*;
use crate::stream_parser::*;
use crate::tests::ENCODE_WEBM_TEST_FILE;
use crate::webm::*;
#[test]
fn stream_webm_test() {
poll_fn(|cx| {
let pieces = vec![
&ENCODE_WEBM_TEST_FILE[0..20],
&ENCODE_WEBM_TEST_FILE[20..40],
&ENCODE_WEBM_TEST_FILE[40..],
];
let mut stream_parser = futures3::stream::iter(pieces.iter())
.map(|bytes| Ok(bytes.into_buf()))
.parse_ebml();
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::EbmlHead)))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Segment)))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Tracks(_))))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Cluster)))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Timecode(0))))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::SimpleBlock(_))))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Cluster)))
);
assert_matches!(
stream_parser.poll_event(cx),
Ready(Some(Ok(WebmElement::Timecode(1000))))
);
std::task::Poll::Ready(())
})
.now_or_never()
.expect("Test tried to block on I/O");
}
#[test]
fn async_webm_test() {
let pieces = vec![
&ENCODE_WEBM_TEST_FILE[0..20],
&ENCODE_WEBM_TEST_FILE[20..40],
&ENCODE_WEBM_TEST_FILE[40..],
];
async {
let mut parser = futures3::stream::iter(pieces.iter())
.map(|bytes| Ok(bytes.into_buf()))
.parse_ebml();
assert_matches!(parser.next().await?, Some(WebmElement::EbmlHead));
assert_matches!(parser.next().await?, Some(WebmElement::Segment));
assert_matches!(parser.next().await?, Some(WebmElement::Tracks(_)));
assert_matches!(parser.next().await?, Some(WebmElement::Cluster));
assert_matches!(parser.next().await?, Some(WebmElement::Timecode(0)));
assert_matches!(parser.next().await?, Some(WebmElement::SimpleBlock(_)));
assert_matches!(parser.next().await?, Some(WebmElement::Cluster));
assert_matches!(parser.next().await?, Some(WebmElement::Timecode(1000)));
Result::<(), WebmetroError>::Ok(())
}
.now_or_never()
.expect("Test tried to block on I/O")
.expect("Parse failed");
}
}