From 0d9cb1dcfa74f9d63a49c15197b753d0c92e064a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 7 Jan 2017 01:45:42 -0500 Subject: [PATCH 001/164] Initial commit From 5de9bf9aee7d452086604986e483e691379cded8 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Jan 2017 22:40:21 -0500 Subject: [PATCH 002/164] Futures hello world. --- .gitignore | 2 ++ Cargo.toml | 7 +++++++ src/lib.rs | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/lib.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a9d37c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..861f132 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "lab_ebml" +version = "0.1.0" +authors = ["Tangent 128 "] + +[dependencies] +futures = "^0.1.7" diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..9194270 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,18 @@ + +extern crate futures; + +#[cfg(test)] +mod tests { + + use futures::future::{ok, Future}; + + #[test] + fn hello_futures() { + let my_future = ok::("Hello".into()) + .map(|hello| hello + ", Futures!"); + + let string_result = my_future.wait().unwrap(); + + assert_eq!(string_result, "Hello, Futures!"); + } +} From 112e7da3f300ea06b3c445daf1318ec7ef7bc901 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 00:41:35 -0500 Subject: [PATCH 003/164] Implement EBML varint parser --- src/lib.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 9194270..4e305b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,72 @@ extern crate futures; +#[derive(Debug, PartialEq)] +pub enum Error { + CorruptVarint +} + +#[derive(Debug, PartialEq)] +pub enum EbmlVarint { + Value(u64), + Unknown +} + +/// Try to parse an EBML varint starting at the start of the given slice. +/// Returns an Err() if the format is corrupt. +/// Returns Ok(None) if more bytes are needed to get a result. +/// Returns Ok(Some((varint, next))) to return a varint value and +/// the starting location fo the next +pub fn decode_varint(bytes: &[u8]) -> Result, Error> { + let mut value: u64 = 0; + let mut value_length = 1; + let mut mask: u8 = 0x80; + let mut unknown_marker: u64 = !0; + + if bytes.len() == 0 { + return Ok(None) + } + + // get length marker bit from first byte & parse first byte + while mask > 0 { + if (mask & bytes[0]) != 0 { + value = (bytes[0] & !mask) as u64; + unknown_marker = (mask - 1) as u64; + break + } + value_length += 1; + mask = mask >> 1; + } + + if mask == 0 { + return Err(Error::CorruptVarint) + } + + // check we have enough data to parse + if value_length > bytes.len() { + return Ok(None) + } + + // decode remaining bytes + for i in 1..value_length { + value = (value << 8) + (bytes[i] as u64); + unknown_marker = (unknown_marker << 8) + 0xFF; + } + + // determine result + if value == unknown_marker { + Ok(Some((EbmlVarint::Unknown, value_length))) + } else { + Ok(Some((EbmlVarint::Value(value), value_length))) + } +} + #[cfg(test)] mod tests { use futures::future::{ok, Future}; + use super::{decode_varint, Error}; + use super::EbmlVarint::{Unknown, Value}; #[test] fn hello_futures() { @@ -15,4 +77,29 @@ mod tests { assert_eq!(string_result, "Hello, Futures!"); } + + #[test] + fn fail_corrupted_varints() { + assert_eq!(decode_varint(&[0]), Err(Error::CorruptVarint)); + assert_eq!(decode_varint(&[0, 0, 0]), Err(Error::CorruptVarint)); + } + + #[test] + fn incomplete_varints() { + assert_eq!(decode_varint(&[]), Ok(None)); + assert_eq!(decode_varint(&[0x40]), Ok(None)); + assert_eq!(decode_varint(&[0x01, 0, 0]), Ok(None)); + } + + #[test] + fn parse_varints() { + assert_eq!(decode_varint(&[0xFF]), Ok(Some((Unknown, 1)))); + assert_eq!(decode_varint(&[0x7F, 0xFF]), Ok(Some((Unknown, 2)))); + assert_eq!(decode_varint(&[0x80]), Ok(Some((Value(0), 1)))); + assert_eq!(decode_varint(&[0x81]), Ok(Some((Value(1), 1)))); + assert_eq!(decode_varint(&[0x40, 52]), Ok(Some((Value(52), 2)))); + + // test extra data in buffer + assert_eq!(decode_varint(&[0x83, 0x11]), Ok(Some((Value(3), 1)))); + } } From 502da6b29d4f0296653dbe868dfc2b5a5574da04 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 07:37:29 -0500 Subject: [PATCH 004/164] simplify enum name --- src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4e305b4..fa25461 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,7 @@ pub enum Error { } #[derive(Debug, PartialEq)] -pub enum EbmlVarint { +pub enum Varint { Value(u64), Unknown } @@ -17,7 +17,7 @@ pub enum EbmlVarint { /// Returns Ok(None) if more bytes are needed to get a result. /// Returns Ok(Some((varint, next))) to return a varint value and /// the starting location fo the next -pub fn decode_varint(bytes: &[u8]) -> Result, Error> { +pub fn decode_varint(bytes: &[u8]) -> Result, Error> { let mut value: u64 = 0; let mut value_length = 1; let mut mask: u8 = 0x80; @@ -55,9 +55,9 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> // determine result if value == unknown_marker { - Ok(Some((EbmlVarint::Unknown, value_length))) + Ok(Some((Varint::Unknown, value_length))) } else { - Ok(Some((EbmlVarint::Value(value), value_length))) + Ok(Some((Varint::Value(value), value_length))) } } @@ -66,7 +66,7 @@ mod tests { use futures::future::{ok, Future}; use super::{decode_varint, Error}; - use super::EbmlVarint::{Unknown, Value}; + use super::Varint::{Unknown, Value}; #[test] fn hello_futures() { From a171b18e4fb8c4762757aced2d6043cbbac5183a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 07:55:49 -0500 Subject: [PATCH 005/164] Fix comments --- src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index fa25461..ce47d08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,9 @@ pub enum Error { #[derive(Debug, PartialEq)] pub enum Varint { + /// a numeric value Value(u64), + /// the reserved "unknown" value Unknown } @@ -16,7 +18,7 @@ pub enum Varint { /// Returns an Err() if the format is corrupt. /// Returns Ok(None) if more bytes are needed to get a result. /// Returns Ok(Some((varint, next))) to return a varint value and -/// the starting location fo the next +/// the size of the parsed varint. pub fn decode_varint(bytes: &[u8]) -> Result, Error> { let mut value: u64 = 0; let mut value_length = 1; From 3b6c946bb45b099cf986fd1d5259ab2a0b230641 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 07:59:09 -0500 Subject: [PATCH 006/164] fixup comment --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ce47d08..84912a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ pub enum Varint { Unknown } -/// Try to parse an EBML varint starting at the start of the given slice. +/// Try to parse an EBML varint at the start of the given slice. /// Returns an Err() if the format is corrupt. /// Returns Ok(None) if more bytes are needed to get a result. /// Returns Ok(Some((varint, next))) to return a varint value and From b571c8a29c52edadc50803eeceb6fc41be870e81 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 08:37:03 -0500 Subject: [PATCH 007/164] Import errors in tests --- src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 84912a5..75b6e41 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,8 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> { mod tests { use futures::future::{ok, Future}; - use super::{decode_varint, Error}; + use super::{decode_varint}; + use super::Error::{CorruptVarint}; use super::Varint::{Unknown, Value}; #[test] @@ -82,8 +83,8 @@ mod tests { #[test] fn fail_corrupted_varints() { - assert_eq!(decode_varint(&[0]), Err(Error::CorruptVarint)); - assert_eq!(decode_varint(&[0, 0, 0]), Err(Error::CorruptVarint)); + assert_eq!(decode_varint(&[0]), Err(CorruptVarint)); + assert_eq!(decode_varint(&[0, 0, 0]), Err(CorruptVarint)); } #[test] From a0bf40a25c00623a15ce2071d7f5a670e3560a0f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Jan 2017 08:44:48 -0500 Subject: [PATCH 008/164] Implement tag parsing --- src/lib.rs | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 75b6e41..a13d7c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,8 @@ extern crate futures; #[derive(Debug, PartialEq)] pub enum Error { - CorruptVarint + CorruptVarint, + UnknownElementId } #[derive(Debug, PartialEq)] @@ -63,12 +64,39 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> { } } +/// Try to parse an EBML element header at the start of the given slice. +/// Returns an Err() if the format is corrupt. +/// Returns Ok(None) if more bytes are needed to get a result. +/// Returns Ok(Some((id, varint, next))) to return the element id, +/// the size of the payload, and the size of the parsed varint. +pub fn decode_tag(bytes: &[u8]) -> Result, Error> { + // parse element ID + match decode_varint(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((Varint::Unknown, _))) => Err(Error::UnknownElementId), + Ok(Some((Varint::Value(element_id), id_size))) => { + // parse payload size + match decode_varint(&bytes[id_size..]) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_length, length_size))) => + Ok(Some(( + element_id, + element_length, + id_size + length_size + ))) + } + } + } +} + #[cfg(test)] mod tests { use futures::future::{ok, Future}; - use super::{decode_varint}; - use super::Error::{CorruptVarint}; + use super::*; + use super::Error::{CorruptVarint, UnknownElementId}; use super::Varint::{Unknown, Value}; #[test] @@ -105,4 +133,28 @@ mod tests { // test extra data in buffer assert_eq!(decode_varint(&[0x83, 0x11]), Ok(Some((Value(3), 1)))); } + + #[test] + fn fail_corrupted_tags() { + assert_eq!(decode_tag(&[0]), Err(CorruptVarint)); + assert_eq!(decode_tag(&[0x80, 0]), Err(CorruptVarint)); + assert_eq!(decode_tag(&[0xFF, 0x80]), Err(UnknownElementId)); + assert_eq!(decode_tag(&[0x7F, 0xFF, 0x40, 0]), Err(UnknownElementId)); + } + + #[test] + fn incomplete_tags() { + assert_eq!(decode_tag(&[]), Ok(None)); + assert_eq!(decode_tag(&[0x80]), Ok(None)); + assert_eq!(decode_tag(&[0x40, 0, 0x40]), Ok(None)); + } + + #[test] + fn parse_tags() { + assert_eq!(decode_tag(&[0x80, 0x80]), Ok(Some((0, Value(0), 2)))); + assert_eq!(decode_tag(&[0x81, 0x85]), Ok(Some((1, Value(5), 2)))); + assert_eq!(decode_tag(&[0x80, 0xFF]), Ok(Some((0, Unknown, 2)))); + assert_eq!(decode_tag(&[0x80, 0x7F, 0xFF]), Ok(Some((0, Unknown, 3)))); + assert_eq!(decode_tag(&[0x85, 0x40, 52]), Ok(Some((5, Value(52), 3)))); + } } From 0fc2f5e76c6df7158eafc1b780809ae82af24624 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 15 Jan 2017 20:30:52 -0500 Subject: [PATCH 009/164] Add some general error codes for future --- src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index a13d7c2..1bb9596 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,9 @@ extern crate futures; #[derive(Debug, PartialEq)] pub enum Error { CorruptVarint, - UnknownElementId + UnknownElementId, + UnknownElementLength, + CorruptPayload, } #[derive(Debug, PartialEq)] From 31b5eb893167de3039ca0e18ed41dde5f5c1c454 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Jan 2017 01:39:25 -0500 Subject: [PATCH 010/164] Fix comments --- src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1bb9596..2ed94e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ pub enum Varint { /// Try to parse an EBML varint at the start of the given slice. /// Returns an Err() if the format is corrupt. /// Returns Ok(None) if more bytes are needed to get a result. -/// Returns Ok(Some((varint, next))) to return a varint value and +/// Returns Ok(Some((varint, size))) to return a varint value and /// the size of the parsed varint. pub fn decode_varint(bytes: &[u8]) -> Result, Error> { let mut value: u64 = 0; @@ -69,8 +69,8 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> { /// Try to parse an EBML element header at the start of the given slice. /// Returns an Err() if the format is corrupt. /// Returns Ok(None) if more bytes are needed to get a result. -/// Returns Ok(Some((id, varint, next))) to return the element id, -/// the size of the payload, and the size of the parsed varint. +/// Returns Ok(Some((id, varint, size))) to return the element id, +/// the size of the payload, and the size of the parsed header. pub fn decode_tag(bytes: &[u8]) -> Result, Error> { // parse element ID match decode_varint(bytes) { From 6c82234c9e0b48ff1867fc36162b84791a2f8639 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Jan 2017 09:11:50 -0500 Subject: [PATCH 011/164] Add a test webm file --- src/data/test1.webm | Bin 0 -> 56179 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/data/test1.webm diff --git a/src/data/test1.webm b/src/data/test1.webm new file mode 100644 index 0000000000000000000000000000000000000000..63d7ba292da6b4d9e3ede7370f98e827cc975211 GIT binary patch literal 56179 zcmdSA1y@{46E-@yyIXJx?(XjH5Zv7f&fxAE9D)XShu{QvcMtB)e7xtJd++*w!kx8d z*7U5sr=EVQx~g|oHNM1F5g6c~7qMs<5KQ#(^L+z^6ukw)iUx&vnw!{*h6BMxLxJG9 zYJryjUb@DQrjX?(Os~)pt@bMfOQzgHtrB>k&=K=F!d9=gRjUjFBmZ|usMhhn;p*$z ze^M&`754v!&;L#AKh&(WOLf60qQVM+vGT?q793oR9Gr|S%*7U8{ zkBI;77OMX53+ECL%yke5mJI}uHDw38JD7!qY8q-P^8^FI$oGOnz~1sTw*R{pzBUL< zxE*-)0|-tm*6jZ8uWTV0r@It z=3R@NG}F1iny?=Oze6OXYFRX<4eLyDc$x^O4GYQ$IXy_Pm0J9X%$ud%ul>D0-0Z2I z)B3mjT(u7pqVVIlIBic?pYfo@sW*fgqvpCsY*;>gLrQ;1Tub!I_D6sJJa2<|8=kwa za>K6y?nL@3LF4W#4cQNy2=%n<9vYkZHl#o5t1#HRbwqlD;q0QNQXiyhLg(}QeLCcg zu)>yP9I~tx1Oe(y;BCFma&7HmplSZX=+HMZ zJLC{3NAtOd-*b7c$Hh!2W21f$_8z?xySIr~v(E4EK1YjKxJ+_Rt zhuxe1>7|D7yfr)M#FIk?C$15rcWvAc^*bqDV|Mk7eHqXd8<$uch%dlrF`2b8V_&Ju zXk}6CG5SOFl)#Y`6`(v7c3_a}<%b}Bbt}Vn3dymv3ECtBe`AvTW@^q+TFH~kUHYN7 zHa#nz%0#qXYbDe($?+ZKaps=68zt8C9Xdc&x}>{xD~OzSP~Of?U#)o)%yNn3%;9;s z=dU-SVI#eQ{H31Ut8=w*VZo?i2o?FaGzIv1ApGkDi7q>cN}1B9cSUN zs*342w$9EptXSdn8^QVtE$8Ko5B#p}fiG}js5MohnJRtE2xc911&zaxsGaL0NT!BR zwVcMk_d?+}>Rhimobuqov;(~12l@r6R$dNsLawlLzB|n)*tht(%~>ORT&Su9;FYFI z@4dQDr%j%!z4+3DX$5>>?O53V;WHUdFF*PHgKh?_>fPvQZAd3vLPT2-;M0#ii65{P zl&paboz6WYQD(;^2exh|W7hkS9oPxg0Q1v(xZM~%q``5DaTpPLZ5w%5kyx^MP3%xl zOE?e3S*;U)rdyg>^3aYexN>hGd>EE3HL%8CZ=GBB`$J+D0V#$=ojr+-v>_A10#%#X zIiK;|jQr)i0w;N~OAZlh$PPCWq4rb&yocNp*ll-Zw&>2xw8`1=mQK% zasXbv5-(e>W577^!)OU~Jg*tj89@D!T3oL$xI27?8~gpDBw*Bk7Z-uVDjWL=-pv6B zDmORi(0#V~+wID^n~lWXpeM~g$PwN>jSjC=4C})xER^~t`|#t0fgDLdKl_lL!(IEC zxKn`yP2ub7(}lU*)bEn8iJ&%Qk{XCGTj1mu$9jr*%;j~D(zwXdNMSz3q2Fk+P5+=TPRe!hGK|Cer1pZTP~v$uQ3-8(mM+a z`p`S7=23O&ans-Gj2HZMUD^{QE1xEy7K-mFUFl^huC6If$a4`Y%M zT4^#OI^4;IinvA)iCB|zZPU!?CfVe;qtHcrPh)H*;gfrlC8e zQl!=gs`8a1I}D+3lbme%EG^IY>oG0#7HC?j9j8lEx!EIn={NP5y1gl$C)f&-ZaQh3 zZ0bbGdoJT2pBUO^@>BA+BmR9*+LKGJXMxMBsX_Aj958m?%T3VLK&43ife7|vMLfgC zJX%Rr1?yoQA+LMyyY(YE24S81xHQ$w*_XAZuHQj|hvZRs3 z@401(v2WZC*5IG+0V%}yc?Wwxc64>-?((4--uRSq>)7kU+{30(7zjxSTdz<+=I`AE zR=?lCoye=TzcPl1{FVFQ6k*pm%l-uxapOY0)S87NAgMyxYw0vmC|g_X zJW%%?v3{b-RB_mDZ{Dit+DzPh9TCRN@v!MYG}#PJ!hY#ZH_I_M@K&y3#a*f0IqR>*+GPZdNMu2 zGPz7aElC%WK^vd($?&mvwNB7rQa{whfgtL1jkI|J>MuVy?VB(T;+A`Da<`1*EhK`Y z=fx7`(GxcIC;MJaB8hFp!e{+nc&EB*QM|ucPz8s1^Im z!;-|ZavoJ~FPDL5bi|ri?A>HOP8X4Qos>RW$GS?%#+>1OR!v`B**;n2N9A~2m8_U| zVN75_^0TlQ-=OC zDtps%ySA&DYq{jN|bdF)bW8R#@F#D!d((O#A}&dc=r*zVwi~ zG>ETYFG3n%BF*ICt+--%Xl#y$d1O$k>E)8`VicMs;(~9YN|oyA@#UA!SBG>sx*HIL zZXTSSwLm2HIi}o4e^o%m{2I$|yrJz~XN@-o`b7bbL*ii2srL{U!lgyejZxcMfIQoj z4+vCt!m*07rZ}coimOyrJ-l$={0DoUClaiOs$DZ&_qP}}>A~1vZhddUSGvcu~gRWX3rummloXtG*y%L8O zT_Hp=cQ!n+h^=q^!aea^PbV?L0KHKbYF@Bw$yrqVWb+Sl@<%zhvCiEbYr#vZiZ`al z#&a{aWK?zKtd0lb+_np;g8;cnSr1`@v{hvH;v)Nq!RMDCBhvO;cbWL~xC=L$c|^8j zK4WQfE2qrvFz25W2s}-WITXwqr+)pNfzQbf8Hyyo?X}1eV0;WFSX?O2+!9i@0}fMG zx{D!w2Y7jyRf*lmm^zql*OoWT(}-geB1d)DFL1okXb9m>*3HkxNnwkIMY(TF9pqB9iXP?KV@>=#s7$xB8K3Eb<96cBRgA_V{J+X>-3p0ikt;mp+cKdg?iSv6$2E zUx|%dkZrMx*RKs;+L{u}5C1CDHCpfzi7WLLgV143%<8Q0<80vHsRVBM{OV={dW*P3q{8t{syMs`b~rU19sWR*chbR!%=-fA1wO zO&)WAP|n_GUD+yye`gJONo~*(3=-0C!Gf2?3H@;1?9DfP?3633PxEr6e4ME>P&o2#3e%)F@el*%H zzuh2=<@FHRso`S4vG(cbp!wbyy*RVeV_Bdqb$T6D2XLQ}+p-}U`sy>Jjf5TFpx`Uyb8XLk_WDVQkXU~8Mq2~jRAeT~ek~!k^&lkeG4Fob zlqC%8-inJ6rRjhPgQ=@`KHb1t%?eZsR>M{7)wtyBtH#pNM-j6GgaL%pt_o-Y01BtkEN;gLp?rWQ z>#LNYc)pD)B)QEwcOh&aq50^a?n(QCOh}htPWW&&HyQ!ms^N%<-pSZ!d$lMVf7g)H zL*-6VZ^|{%0S4JT(X3LVnl(rudJNFaXNvt``8%D&%f32F;rj9Tl!!TI%p6*!FofrMp8VcCPJA z5m7t zzffK#hmRP<_DlIbXWWJ7NF_O-F2;C_Jbsu5;d03$J`S2&>DwoEk_s%k1p-3<1;H&( z0)RD|>7l3P2>c%gLX#~pQ;up4D}e|*iX_!*{w1M=kdZK3UH*?%AHs@;O$(Zy3-6Rg z9bi4GgymiP%(j1_5)2QYgN^W*^0xLuYicoQw9GZ~2sNxUy4$ZTH>lgi;W(_R2)F17 z2q^tWXpFEN2>{9ox?1>CB3oSHrp}iXB@X%hiDbWt0^>q1@F@=OZxgJ49**W^B{@TI z%O;ZLiRiEIemy6F%0|vb4lNWbd|&XgJK0iUqy#Dm(a-$_`YcukFTVK}7`vIWf*&0~ zS)!1-Dzgo_40VW;6bT0a0VDs2z80YY08mM%NdQ_f?#j3EdsY{AU{Q|_K^#?gR;NnU zuiu?8R7(i$D8xflysS)VW#m9x5&lO7%g1}d79XBC7Fey$u@k*Tdzztl1chIL0C(MY zc*98V*^?`_Aezx;T&d8Wa+_3(6zOmS`qGNpm=d?0=ucJXV!h7;q71DDpy!( zQv42q=Hn;F92jEU)%{wXJ0B&@i%x`yHFaTw!x7R8D7puNA^a!kOCSK#h+z32QMwk_ zrrmpEB5kXsL0a7O7bth3#IV!tu`y_7&{o}aPA+&M24w@ntaRGgj=Y#)lvT}hKcSvr z#oMurE|D(`4kzC+Hb>Bpzb8#mM+D5rb0iCL&t*BRI|w*$?8X7hrICCk zl8|iW?vQn*5A}=apy$H*c)aq@<>?g%i5icMIaH7QpGcKwuh^B>y2h&BV(HE~nG@@y zI`_8b{H!0Y1V^2wS)UU6`;CpPb+=#U5<~mE*F0bE^RNL z-(MgpfwI__Qy5(B8I4h1O^yWxdAb>=?#5r1uf8LNl$DOpSC5=97%GDGm{PoSFBk(VWDC{};8tE^>ljg%SmF|W@H@m_BhY#@d(Yw|q{^enxSN~?>bd9Ux9NMGqozYUP0$GLbN^n`$9%d`e0Fc8 zU7exXxGY$7-9mF*n;f*m?4%VKozyc8OE_~*SPFr-dMn0dvEDZR1^%%?>AXX_g%-0X zvVIsq1o=t?9{?b9f~vb15B%S{HIT+X*gPxz($YS+H_*+YPEch3YwxyH+dM&~{y0NQ z;Z4OKh@ILI96`&~WypDpaEL&FX)yjllAKoug3gHEb57i$L9q^0_3J*fxxH@IG9+GO z@VuV}ALPs2_4PlnmaIEI+JO)DE)`m(b^4C%^bt5p3K8}hV1>(pUd+Jj%IgMx+z=Qo;#xP?kq@+H!x2t}Yo|o=2L#Ycff3QxR9JyDGG|f@}7~Ek$q#776D8 z!Rr3QiuLK#KMN_J)O~wiVS!YM#5V5AJ$sRcy8Ch0a8t(5I}mqr!_r=zZ-lYKM(SGG zkE7~_%ARQ33t5c)^UlyJ$6|x|-6FR&&Ftz?j*)nuVB=|VfRHnv5+2BJX<0oJniLz%b8nf=NBH)mGk4ycuGcw z?t2Hu=pUW#Q(5GMa?<-XRPNmM2T~NxL-cUu40w-R(jwq)0y$|*j9jV9$;^1P<^_gt z=Wr=1U0D;2tb?34;llVf3lKJa(Nptl_~d7ED1wI|)Z59(@~qoe;{9jp?^jaC${{5XoX(G*lXLICCh4Gc(Y}1A`Zs!KZ2)c zQ^%K0tFFgBC{QTA9Flb$mmMk*#CX&qc1i9$*3%ywK;zC?Ba?MW zaOiNdxV*YxW)3H!azLA7^Olu?PTdWMCy4!rI zglaz%%Zm`%hf!)B_|$wQ5FG0tkTJ~SBmjs{&3|s45FvaRJ{XUpKMMri`0t32%m~Jt z&esh!qKg#k$gWsX-2$!{*l*exL8gCRkzG=u4fp;!J0-4E^K2J4jeTLa?|G20ab5=-eF_m@sw}+-yQ93() z*gQ+RYOCf?UNbz|<|O1tl$TzVm^ur5y_QTm--lcU{HCd|!r4mZ59N?=IskcE-+Ew* zXoJ4vg67r|&-fZf3x5#`qnsY~QNR>?6m9vl_*UL3rCOi26CgIHoIUmUoeM`^ogF2x zt)flEXvq*v0UK9gAob6RoCu4gS_Ys<1PTc5_1`=Y0RUVl_9;jJK=bH@aqM(a-Rqf% z%{{w))Semhe4d)V*6*1ATx^^G9R81c-W)T;pQ38?zktVvBtT?Wf~3j-9Q2Tb;n21m zULaBHD-`YU!?E1#Uc*v1b$4;wr2|vCCsx+)d`29x{l4krn-^A(K|542w2ep_2<1sV z=a9yvah=Vxx|0E6vK2k6Xc`u#f!WO&vWsj4o@)eb#ex;CF?lxxIi2Tx?5)mPY(A4*kVMy?QU z(emMNRY&qP!Vi`k_ZCXr@Orjr$@mZIgKW7Jb8=A{fA&N6^2-!^5W$Cg360J0M~>^&0rgYUFGWyZ+2GlJb6{|3_;=L5)L&&aBG!Ur@1N*;frORP%}^&6?0 z+QFt_iHdIp@U@WJa+Gi+eRR1?vLi)oubCJQo|r|>HJ~CsH+7`W7qXYSFwzIWNAj2K zn0X#*i|6)Jcyw0WEc!S($`tcYFzf{cUAd-D_$%<^I_nAiluI#DGTc#)8(tmOEac%f zd;R66ImAp`YmM*=vg@nED?FhRdRyuKgbDSFl^iHD;t%XhK`~MkbL#CS$D#nuzJ$4+ z48*L!7Nx(@7#&G?Z5SlIt&@aA*;*a__f_w22ef6(R*9dbMU+`Yt zs^BhLg-x4Ig+^6~mpg>-d#KmAnArIS!D?-h{3vT3cuK%$k&Ef>>=N`SCs+CbJOqiH zn-D*Qx4`?6y5ww3Cn`rt0jJBM2Oi(ikhLBnw_7QIqD)-UKm3%j*F;)>+8^GQ9$S#4 z#8=K*bUti{Vt^GqB2Vsq-)^GUs9Yk=#6euZA>SslWK zA1gvLn+D~mEcW*6W9z==zSlgdSw<$7A_Zp1!2aHfFbp>-bzO?*r{B&T8k-^MaUWo{ z%i??jN-2_*Q|;YR%=I-;w}t+G7BT*!!v}!VC7rj0MRU;6aRb=mQ2V7fwq_Tv{>@R0 zlFstAXFZ}cqceyZ{xKkA9p+&eRm2mYg+!1<)Fe<7(lcT{dGEJ)yvDG5XdZZjbF~V$ zrl1wJfnVq8!}NjxyhT#fS%bgsN1_*07Xo3`HB{#V>w3s=&QIC`8uVL|w3@`? zhw`HXL{s&O^x8sM6&2!Zd7~SjWpKlaPy%$`QqMhIK5-POSHT0H?>Xy`f_Zvg-$$2f z@9H*=F+-)rX#)zxanAZbmJA9!F1JjM`Uu9qt&3-2h%Ut7VnmtC#qPw5pihhk$HhDL zinj~b()&l^E;ESW?YgD2yKTIL|NTz-DI=;;X{9Vmg7N^bFMY<49i0}m}eT*(SthAJbAz`@nOOzyqezMro)+H zZLs!t`3xl%5q+6YMlB#di;e+*I5qYMSj+noT_uewg|~J`#oK+H zerGQSV#F?`>UCVQj! zYK8lf?~+px<*B3Guve4ottDvrNUHsJ_7+2B1U*0eQZ4V)30f$#7ey%jXvx))Bsl3N z4IAMupH8ON)Z{TWIn3o zblnBS@lzDdMK2%zr8~J-KFQhho(uHx438M?$Ng`oQI)8px=}A>q-*HYLuk8_DHV4y zbyfiIKs3UL73cF({(Gx!gOhNo|5Eu5@m*4N8AN_%B7Pn8?&xa-l<(0tO9l8c0) zV|Xh+T@IoL<1#f#Sf}1W`Er@7?De2p#UHBou@54pJEe&DeXQ9Y;1e`8|Z@=&in~vykGnEKq_{`p>l>17N)gL%SUu~b zVhfQAc7WEH=ooWoeKCtr;2Q6QbXj%D+{z!OE9mJ8oOKz)`GfABmJABI;c&YvOQ_r5 z@;*c>FT_MzUSBooT79G&?tIFK$bajB6QDPCN;mffRrXW4@wK~2wW)LEtFf9JW%To5 z2AdAWr2h<5-485$+k_)-so;CpKXdncwGLOI@v%KVjW%5AT}cw9RkYN}J2J6M%Ce5h zz0cQzp%25s@oW zB*nTZv6$E?X5bkN~HezZ<^)x1TP+JV+ z{5ObBi-);&6SbQ9%lnB$fO@FNqNYP?aP$2{0!-mr6!e#z-aa+aC!2lzb$5H_#+6#` z@|@PKIYGVUhBcupQIMzN4;r?+&h9i`4HlZ*;U0ZC1Tb)dpgaiGg~W^}D%Ul)2|q?$+Re4F zVjE%(*)tQJ0VTMngVVk!w8U{=v9bOI$hr3&8K=0$Qqc06(* zfBFNznc`BA)iSsHyQ2r2fBzcI+!$163x9C271KDy0p)~a?d>V5e0;fciV%%!;UmS1 zy1k!0rp8KDL+F=OYRaF!Iu&9^`WTy_#S!*4WJe=zm>=6FisubQ@Tf+v-5k`zDlv4F z-}@GSb|z0y_9}7xHu~7%u+%Xn@%1Mb+UJGPFJn^MvMahQkpA2)eWtmc5&oCvW{=ow z8Q&BJ*eYs4Mb`89TwlI7!k3zw5j<%~U{CIsm`k}^B%aKsDCth6oA@3SX`DUJ>Q$~! zSI4*Iy?A#NrmK=i#Tjq+MeKh~fabno$0;m!~#3Y5`Sn5=(^7 zV7~HZm|K)CII@`sh*LjQcffnt1=rSQuG-B+Inz`~&JJx~v0_HqxOZKVA8#)fv4Ad6 zHX{R2ya3=E$W}9?JvjC`IYJ4Vtu)yRur&ShZ&a9&Y?h6AgjKad_!2mVjHDC(+p2}Q za&EM~=rsx}xM|F`y#@)6#QN89Yl;Ulnu?3L|IBnOZ=hseakIrndkM9J0~ z*9gYYGNs>1#L|N($3M2#{jQp1+%P=l(DlK*90Ew|sV_Vg1|ThrBd;ioL8!>Q7d&uu z<8m#s@l;Nx10$`%@iefLgI$WSn8I;M4~AIg^YVIZ9j+E7HWtJZK1*uM;a5iB2zt!2 z;8f2glwp=6JGzbUf_ASs9 zJr#AOVw1)xk+Haxi$q+2pQ+-1)><7b0Kn}kqEq-;wy-1wFq8emCBfcEzhRc`;AXRd zrmW(&_T|quBHvE*eXuO{gQHFJveeu4&L5xE9LFrSHT5(&wl;M!pFA56FRmstun#!<7b6&M9)xO}QlzG+5}au#S}7U7 z8(UKk`1dq;z>^9(T>Movu`!nz2wV-t75eZeg4IbOU@(} zcSPS}!d0;9A_p$#lKf%$M)6pU8w~Y5&{u4Y zDvBSw3$5o~%rpK|H6SLLOr%*MpBib)IZMOGb5_#RRD)d~d00bsD8f^st$GyG+996S z!prXqYj^L8Zv*>o?3qHJb{hzU5c$u+!36#<;TV7Pncw@g++@^MCE8+oO)*?%)EE@0 zvKxYCjZgzpyiG@{h2xD>`ZCVK50b+4u;*Iu!PQl=Ea;Z2Bx{l|%jFK`?2T z91a2PA}i-MR4Y)(N+dCs#4MA^-!7|4M>_&5n(B@!A>&1RH`tm0beqU_aHavTRqDf> zOJyim{fMu=$^(qmwNagJ)jrwYf8aFrW?9f!x#R>c!VkMsiO}^2GJYR_KV-(-_UOp$ z9q`=f(y4sL6BP6uO#W5g?}kS&M07$R))&3ySijHzS$`DK0YaGmKPmtK*j+u3Ju}DS z%rwM3J{8eG)D1sd^vaAbmDCeW%#Hi)jNn}FW(3yp@6=BCCBehb;AE1n6lVju>$9GH z{enLi`Jps}mZXC-HK)A_fcWC{CKBkBYv*M`D|JB|K4$C(J1Ped#+kT7c*v>h91d=p z35CV^cYeL^3E>2JOOK902PX%9h3uGMa4Cc!Nekr|K%Ql18=O9~0lBm{>h-dkW4NGO zp=5QTjI*rIK=dpb^dSu4MDzi#epy{yNbK(v0?`OtBtw>s_3-9|@D+=ogmZhwb7bxz zE$_X-i6T1&L*>1h6-?6BP;dEzRIa}((1Pqrpk7X)W6F^M$02gA`RT#bV_HedJ@7<( zJUwtc$RG}ZB@gg-zWA=;)sPD;ua!FXh%xKA#s>C3OP#*MKEys$PsI&PC|7IVGDCpXg=rJN?Rqi$pSj5F!6@3;pSq{R{tR+@G8x#C>aSS7ovan4U03g)%cv7URvu zV&A2LTMr5`MUOIPxA#Hf|Lz^`bv8~TcN7Rzpq7@|Nov|zu_hlGzYIdVxdESyR-ogA z=30mOvXWaM2F26wP+;gPtx2J37PqU|ggyRkK9pXMGAKn~@5g(?R(7VWp_2ZId{(K3 zqNOW3dGbbYucdLT^oJg%k*_EY0i() zo%M#rlGngDF_R0+0wy%=mr)L=5d)hu30G<rw@8x_ZcqQ6jh-a2#r%cE}ZM~{X>R+X!8^r z>qt!)>Dz#>!r}hLY@J^wo*1vN!P(B#@yx!GV;s0oh2y#T5y zut=l_2vPj6YFMBG0rZ|&Kg~xyLW;bu0Jcc;*S=vO$eWIf3dxznr`z|~4+X92k4{z- zk+rPF+EY%NQ+SOo-6q`BgC}n3L(j9YPD^4>F;i_0&ji?}U+qBAJq#W)w&Cog0wt+)ShoDv|@jl)_>E_%1Xu&%&rabB^51Gx8bcccm zV4{7d*q9LwV|G-dxU^E&mN!AY^Q2kF`d&E9E`D;#^uov3(K9(Zkd>FtiH0!}UtX6`C>mL5}9ow*ji@%j!@r=sOJ-q_EFUZkFze?{Vr+kjq@a4Pdz0VsA<-sUbJ(E zBJjgGZDHi42Iejxl6^+drx)7PD|i??s>&@lbajp(#bD84K4QRFfaCYhZ@V@+HK z$-^{8aJQfsm-K!qm;=LF#7hww17+5?q;=wBXV9kxtAP;1|7s8o3IGuH6nkf;>Zo|! zHow2x{`tAiHta?=V0NnS;B(5hEES^oMyle$-f~(w6>SR>xgqtA_F~>6i56poo1uP# zpCLc5@P`oO17jtgOTubmeT=J9F%w(jG~n$sNnO5+6OY*nbDF*b23Ly_C+?P ziFL=yUbH;hC#I|kRXU(EP>2pWO_wn7@V2fBqNKwv)Q+TG6m z&RV!|Yau_(ULl9!eS_QN?mOrN?F4L&xmy|2qYwDs>bD zaa^Rl_^TpVk`jzKnHFUThZ?jw3GYSr*D@2}zdl}WSj+B3I zUkg22!gGfvX=12Qn5XPC&)b6B(ehV2{gsd?eRkIPzxMzi)LhXnL5s3_GQ0NL?srnb#X$Sa)SQ=Ghq8!!a|yoppY89A3F?Za9*j zfH$|e*D~s#vV()sE+XZ2HVs`mHF%?k#oiZlpZ+;DAlBQb5duvUjmqve3|d0x%5va) zU%a#gPy(lem>xmzVCtp0aBcCuewH~>l$<`kPy%UvO7%yMp&dI4y7x}q2)BG7978^I zkSgCPXTZGvV`kNsGrR{u8fi19MW!7>Nd{UGk?;3u&BLl*Kzc#J;rp||a zsf#LZyul7H@eAxx{oybBm;$opon%ai1$u9zJMAlV`+X6zB553SOnnqvnrq{iE8TD* zrOZ@iF$FBAjW9uEM#Q81=Vp0dcz59KEDQxd`-LU0q4_mSTHCyMIMS=Qy}V4}Dt@WD zAfkYdVfwzP?d&ky!=ITUbHLeNN3wslG#Wm*IeUK7O1(=3E@A<>SLpebAZ&Yj1Mcg( zG5g@#e$^Il@9Doyc;!=yOzD#<#7vDy^BVDfPYY%t&=e$|cCw|uP{nmeE-}z+byo;I zX;+UUi6#>n%14WXVLX8A|EWBMk1yca1w(ONCEi3{veNO*KtAXQr0BuOTzj?;T0T%b z3aZdR9_!I)9)G}$_AWc6alR1iy6crL&Z(u13lDJy)aFB z=+~n+%aJ4Uo9O3D3YUdeOQ-Y>h8Aw zW{|>7tnanK5pvt_*MYj4GTI`J3X|X0Hx+EMU=^I=(7w(vGvfr)2iOUCI9Du%mkdu6 z;;H@`?TwO$BcrGsM!!LF-~ip>ym=l{UdLoa`|I69Xq+uf$v@Vd8{95;r^b5fVa#Si ze9Yl**0Int+n$G@zZ{hOjJ_Q`{Vj89kb~o^2**LR-*U-xQAhOR5aOe$>!`?wEEFfC z(Bn20Z_;|N@rLt@=$Sp&H}08UKi8LNcZyHS4IuOKEiph zfsjdVib&0SP>jtogoP+Z!1uj0Wm_vEHq#&XYihL1t4trPxYP^C{uU)3Goc=D>u7X> z-N7}YJz$_F_FGCJ<`|wrrYYQEX$A=#%R>N2j8Rr)xw{9Hj5LErlY7c5_N4d~kRFkt zDC7TGe9FNH2fd@e{QcsO4MP1PveAFpS?QMS<2|Sn8PJd8a#ipR?=ifKwjpyn53zyx z)uvKB13 zIC-!Mj7<$|ONHT0Mt8SV${`7>!J(t=GjPg`7aMY-mP(t|A0f1zEm^f`O2eJB3~}R>JqtU8RZHwWYqu8)PlE zA6+cnQ=l70A8fadLkGv!VvcLcYOYFbpx)ThmM`r)>2#nWs}9Dng2G)jW`jcAPO85%?tjl#Gto{@U5g&n`?ics!}kHc&I3Wd8^X`Wk!!bf^0qt zzGT`y52XdVM3|4Tg!7O3kb^~6PuE;ImyU6z)F1#Mgk7YNxG;uB!>j;QHvCoz{X2KV z{QSprWS%cXjk3@0+l*XK`<&?`KU%OaEOs>}43Aq465IX+g9bS6iw>2GYCIX^6?942 z$B384rmt=cbb^(twot~}`Y2G+;2~JrXce)J&A7zf0z-lOfl_j|BjRU0=*P)06L)N? ziEmi%{>jVs29VhFuN_&Jf~NTGFy`9tr2XieF7~8vR_X33NPAy89!c38GBY?qAkb~0 z69`mjzWD1D1ft*jh;^}FxeK*>`8nRWUJ~FOmnLOa6q&z!luPm06F-A1^vez~70+GO z4!5ta1Uj~Mc#(Y#THKR=xl!Aw#zpd|;}ItW>=t>RvJy>}VQDq{CbN8_2g#*eTeRVm zWUPE3m!qUIpBXl>R*Z-u)Tu__gq<)_;_4W@P1?mSJFmfIwoK#3oXIOugsDjGd%IL? zWxO->JcVvI$&*%uw28!>Midman}T*teBM=l%K9GQ+;uMx2Ri85z~zz6aE_T4AW)uP zZ^`0bE0!aPB^M#iKb-6xxry{5({lMYkgop)YWP?V41!t{?aLmmI*pF$c}%rK-^A$~-a0 z6oeK~zlz2}zvQ7VhcuRUz652__@Cp!wmrfjQWz{83@MvtK)z`kDgV8mE_#BxxpR?T z!G;2a024IJ{(2_oS(FhfofM#2TpLsNmOZhB>Xzc2J@vMEqfpa-NoMzqB(% zG%IM(=?B54Cf0I;5t&p2nuccYjQZH5~;Boga-_kly7AInAu#b{MpmYQNCmrB-V1JsL_3sp=z% zvgc^0PBmo++NedtrVlsKVLyfdPv4uECb||SCEUnqMr_Q!H?1cJi12?~Ks~?+Nic-4 z;+(kI!%B9w#vi8+g<}<`AZEk*twxMyamP1>6}`R~Yz^8(_z-H(31OcR(FcRc&zX&) z_Tn|&Dx>Oo!x0p@9VV`RXrv!2hRrZK4e3g3>0w0OcKULg0Qp#u$djnrg!_MpdZ)n3 zm*#DI#kMuEZF8cDZQHgru`{u4+qP|EVw+#~e*gRV&eq8~`BisS_g!^g4Tk{4X5`US z3B5by?r7C~L!(%^&{k3W!iOVfGPcRTG;ME&`(fy+H&GVOEP{-iIt48(abS3^G(2J7 zO^a~n5RbMOPH6TkoumWR5IE<_Uka=iip6d*_r`*uAYn#1KccqqZw@T#KpB}2seazZ zCTt7B62lpCAlYUIeg>~P*dQJ|Y1v@O&!r3D$FwS!669F(Aik~ZSnl?Y9dLnyloSi9 z!AncQhOaBIFZ@lJK5Vm`DnQ_jJX9EA_JeXN(T~6RaShqe*#txMeHhgb~?ax#MlS+Z=-xpi0?(+mZ)@D8Z^S~e;l=)X+ zSUljqZOq+b`+rm&1ax@K@Y6b>HNKFT5BVw~fzP`Tf$49cx7%Jv_~H42Nn>p9^`6<> z1xpSI6YKvxnM6|5>WQ%YY7!K}#vuhTmF^NFX9^eG z0)O1t&54pC(Zy?vNXAcc?iM6iHj$-+7Q{>@JNI2J%DY~gF}_mD)FMAP0EVV}zeqkj z@DVEwetz}LfV6*rL)C%gKLGhj2CM6UG73nPzdMUlbpIKhGK)#clv=BUJx^7NNKf7| zMn!CUt+6461&tF!J-emk(6gT&yKya7^Vnb{Lq$9;Sa7IJm}`F`9&c~4d}Hzy(?I79 zy4v2m?G9erzMke;7lYXx)&R4;;vUlJIvyc8qdW}75)ux|z#$!sq-Uof{T8~YhPD4N z^{5(lv5eGC)hxtUTBgT0w@zx5XNa%5&O8ACR~j9a=br_F z785;~h9%3jhV=JP^d=^D*v0+4-sHiLc;c|fLjaFz`n+y#S-)0x1rdlw?-a}E+7@9% zmz6IFd{RPS=O(4NugmcCDJ=a&_N78VIA3=v7n*AA`6GwAbc(jaNqmb5dy=hji$!(8 zYJ2c!y~FuKEHRi2wW*ZxY?F27X(@jIBKU+9_gd!!4e2mv&I@~ey$Z+6)r^j+Gc9@N zlCmuUq0p``*3VkKLX7^hTHx)HUwkgp=+L88=^BpdF`Wi|0ak2haw*K@#(uKI|=9NYf$0~hV62d zk+rHkrN2>ioC;b}UC5*I5eDdUZ?Cj7L?%14b0%q179SB{Pm5yC>VSNa>Hir9A%pw? zSzeXT+tr5mIyonw*)0q4u^S*S+tv3z=ZP<$tj6AoHS{|5%Rbx>ykBE-UQgA&w)?if z!CSn~jp2#CT``8bDu2Iw55MC!4@=n|Rrdh?&p^mBPrV=WNrU$UZ+-HMa z#7c~~bvkik9y41LrkE2LiFI(L>W}Z5W3?-2!8=6)WuAoSVTcXGCYbT6F@P=TK;>AV zP9-G4i0nblBxM^FQGBN@VxAGaj~m4d=%jA1i))b7~|wN!Kyc50A(xJP=E{{(j!&V)GMf>Bp~d9j^zEq?Y$A9 zC)u7e;Xfl#iYXgf#PS>8uYN{Qg2X_Z@1O;Sdmet9xslwChk;W}2!=&Kx(ZN0+0?Zw zSrN8@0?L97zU_X-c-8bK3f$`qg=rg9aT6|Y@8KqAxZuz{8b8^$-R0UdVwkG_0iN>U z0#7ef(F-0M020*%=EeAGZuzSLC|OwKBJ0~CTfA`7sIMmu`3N6>Zal>PoB1nT0ZF(n z*D~kXV+gnKbnPhCa)bLA_H#oW&C)mx?rN1@@;Rf1eVy%Yvit|g)oJT&fkgAt7Pea^fprJQQH!;5`ni^%8v-zQ$M>C6u74Td4ZBiw|ILupR0~g*a z_z`4_g+Rj?uC_UP?52j&mpSP|b+_9qcJKCJjB1%l`ffG-9`6SW{@)3fa(|#xGE^<_$2>(NUM-G&3lpd&a4{KPK5< zbTxAkm1V|nazPO>Sq4%Zhs=liuMhhwFqFXZ0t3j&m#Px%b{JFeOOCFoO;$Y*K}F}@ zF7r4&$J3gGeUgEC)+XKBGSXBa`wRZu&e4RHOBxO9QD@t(xy_uIbV4xd#S$kr)VnDD z^()lgfre{q4$)ttMwi*=thd+j=vQEEDv{*Z_~u>GP-BoDlE_EYcIv`kMx zjin=fwiTCUtdt|8HLAsge44e>_&n>7_(Z1!ut>AksY#}j;LgH!$xR)%1k92l>SA+} z(lGck+i^@#(_epYXvLFg5WDAULmMQ^Q(S^E=aGm%L@P@OU63|0)o{Ls-Y*9vCkhZZ zHd1gBG(hvIj)5=NZwsUC>;3X;{%r}%e=E!-uCDIg=!TfCF)^<`v`k8{gM)WUv_bML z2a(E<*3ld=;a+%ww;b--uK7D`Q(m8nX2n$`NS4K-I& zOun&4NN5!aZ!6$dlE+ChuZ4XpPHUKiJM971w;B$(#=10tw*{uEn08gX%tO3auaOsH zS9{90F}h%p3ty-~!HDhSl3N_SN(egwrA>Hjt1ECsWLI5@<f}>77eY<a17gj<_VJ=jt5I`HlH)GNgJH#H;q+0FXwK{uj{4?Pj)Q=JBi zNBk5%9r4gB1h+WlH6sPYfa?vUjqF*3kvY%w71nlcOjF-;R5p4PU-%5x=gV*ge&Ca( z3z!DsxycM%MNy%Ck~jr*}TXy+BFnA#8M$2t@iWe%Tm8kxnRPUD|y=>!dAAznn^ zqG8!XkF;`qA#=>750|0^vM$$e7)}Y?LB_9KG)w)|sjD5r=0XMpvH}HOFC`>bu`iF8 zVnuPy^iLCElVdP}>4-l-yN+LiHRjztXfN^#sf0`igJnOe{}_3l?*o_EvwaXvtPK_J#R6wX?~Ih zk7b3ITr*OMugV1zV}k-oz{sk~W#lA(H^O=TGvZz&eaD!=(G3<@WCm2-w?XeazOZ#; z2)rE9WEYbufvXLUq-om+t3yV8WFylx*>lvR{+-!;#s*sucv?Ub4>ks~LFU`bN9N4RyAUm4yj2#hiz^ zGo^T_bDz0bVnG0RFmfQ$(U&WGT`y3uxLO||PwKzX8k;acepASf7PtkoA4)TRy|FaNB0t5*F2)?0?>~Q6(d4;SsfO!Bw zOWHG>tatP`uHs0TJIdFjiLlI-R@eo3m-+By!m4c<#tXY~;)lSS*ZPTlUrL?CjSru! zEK9i?5;LQh&|*k&!3>4rn_OxIL5(UXSRAZaUhx>le4cnJx7_}sh+T4kubkcC% z*oLQ9FpW?=oWFoWVODuwpS63_1tKV-bcR8o5r~_oJYpaWA3mZSJoQ~9TWH;w1Pan zjw9>SPgn(PvG|lqm;G(H>mG)KZtHsr-vk3ryB+^$DNd^_?#Yg^Y;8wag1u4MxL{02 zRO>1n1Z)X5dE(aG39To4IZ(_x%Wto(pwEBi?(dBG&qNwc(?4kvww_N%aPEc^2&`5=tO65{_E$UifksDF%At9 zlcPtth(hwQ1fA0^F}K;al$qm+kAWX9%j#t z8_SFM@%)R$9Wkj)f%GgoVZ@V49#bW@OFxeqX^KlzbH=WdQI!e-F45`pV{(h6p$CZl(OP+o<6Zpb1%KKPBW{?orZ~F&?~mvLCY* zVsDV8?+0IUiaYRM|7_#SNYQn5SUzlh!##XS>02C2r;mtdhxiZ#>^+_xGTh*v17ugE zy+|kFBane0DUc}fQS7@j{%fu=Fe)^j#RXz9%JP?g1}tcz*)H~<)*`)hYxtbn$kt>!TCZ+|5@Cxfk6K; zPQ`KC%}M(Go62-;joJhnB+G(PD^?nQR9>Z{H&I1&zn(14dWZ(I`ZoJ~0nD=(+<%-VsN5OP069gLJvfM}t zHlA(fbzfCTLqe2&Wl;Z;Ske|fxne+uT85}Ze21}r>cfi{ZGyL~;B`h##IxMBRZpl` zfg|cQ%a{qNZhj=X4fDq8AXCgTNBu@JR*V%9boOq`e#EjdAj%F2mIo}2L3nPB!MQ|~g7hDiNh4P6$1n<>HC`YISX z1}(b$MqevYWSV`AeSUEyp>}dT@&|-lqZ;QnlkgzT*Ca zW2U>(B}1gyu4`^S&)`&c!gkvWx0&K)UpV~Ny}wsyJTRTX9s>h)m4{Y5<&Q23BNzfO zF}yXZL2tZS1CxFIOIYN3gm^W$ryDJVEd8IYsoV{_kxS?_nS0Uvj?mQW9hKL_wwYD- zX2ZQdx40)(y_is99eg@4%j#8Cj_k5dI7_$U^n>)CNl1krVOY1C_#VXN~s->s&8K@DC;b8dGdVyl1Md@NA6!I#SCQwD=!& z;QwvP?_KkMwM~bV5J3fO^GCeib4e_X3;7G{Sz*ZK>u`sb_Td?rq}?<2as=uO$}^Q#hg@rv{-(Fa@_7V5z@y>}kWrhIM(iXts(69jgOi?DMb=n0v}&6Xa-UbJAjwyYR)G< zsQjou`wefG4(R?COBH4AO}md!)0>C<0OgLFnV$|X0K=a5aOqDZ@qd&8| zSxQ{}UMXe>ZN<}5hH~FGUt4B3Z!TtuNvU4?^0z}SzS?xVp||Uaxf6*xtvPU!qzjxg z1TO0mBMqRsL^PdLG@EKOP=bL}?i7!Q0KXLD7}QNT5AZWdYkAGzHu&>h$XU=uJG8pF zWr%#-cUWDc)W!L+y-G!L4i4dA&<=uueO3xwm}8#^{=wfFaQ#2>!oLOaA5W;3sI*OE zYX&9JF{ljOFO1L*Nh%%Ssd{|~-BS!Z-6M<_){7T~?QfHR!_T^q*q8X}!TJi()(7~L zJ>xhPDne9be9Ml^K#U!ZwaF6x-uk#AlnsO$BycNZ2h+F0tPy!XmuQ((mA-uhNQWr= z+CHE2-M4L4ql{MIXc|V&B(>gT??>d@xm-${7qxR-L@9bpH=f~o5k3tMg!Za~Twv_} zzhg0QG*T-I6!|zwM?L3$X=U!G>pad3pRT|g*BFoD-PQ3WvWiSrEjgz@n&pU^R;3CU zjc+DtPCB?5#=m>MMYd)EAcy3iGhQ?`Y?57Lcd`EARr^m_4_a$nl1lkVkH7>mn;>Hv zOHCi)%9eE5RqVDPsPX|EU@#_c9V-5Ak{vBU#Zw$^xsDm}{`O08G@jgNco(QDa!H|h z{W0@=Q&Z_^$k)L>a}JO7u4rZ1+->AL#-3&{9RNeUJ#ep{DpExD#{u$(UVM}=KP&6$ z8M(&?$~mXBzx-Vf%fxtu8D>G`O8sX>rccOKUa}uo+7hpgn$29L!OxfLR})Rzci`H) zEc;yA-3jnguGU_=`=2KcwqDN0;pVPjr1N@%w}E?(!p{=Z-_iZ{rT-t`g77=NviL6C z`oA*kRs%q?|6iT!Ijmr>JYj@Ag$d}5Tyq)FX#IuT;I64yu-x>B8sFj> zq-e|>mbtfleIdkCMMx;u%n!&AmUxvYh8-SXlpG7^H6yEFM%dD?9*9nYaqwN=USN4= zyo0J}UI3{=dMUz8eOZ-+URi7Tu>?fbm57?s8h#`Hkx##{+86JV=i$}gJh#H`$={{H zshd-?zhG8uJ%}X3E3eu4XwPi0A13}fDq90C#9A-MCjfVo%=AGU?q)l3mF%vRKOk|S zDOG7+vzVZZk8C=NYy11aE`qD2@GN@2^cNnO_|vOv-CQ1*6@MNxo7|o|&qjpb*c0bZI8~32e#0^tGz@U(J<0-)l_PcT((71#{qF3@Q)eeKGNT~tCETviu8kx zNTw&sqIN|gs$z%;=SC@Gddmt6#@U3TAEB2YR7M3?m|m>57hE!LqM`Y1|345!2mk=L zD^D2{09f}LfXbR-oec^u&_$&z`LBBAi+BIW4DkT? z^#CHhj_$uuo}XMLZho}0-r-kk6{0xXMjAwfY-ua5E)o=A5Sr2v&f}TTTQGzA)!{*Q;`)qiDzoi(!H`awaNM z<_@gJHw)bv+NMSdck&vK>xtOI($MoXkCYXq3J!>COzAkr<&6j_C8RUU{tE`g&%Ulj2yBx z9ZiqE@m%Qk^O1vJ4=JpXZ?9e}iAGC3En~bx^kkx4c&7n}YCvtwS0PSYQ+OlV-dG@2 z=|2~PN7>T68gxTqw7?H`0*U5A7VcmO>12U>Zr%P$mJh`@{aQh?9S#*0bIVzWzvPdq zx{@`1e(-&XE*q-^K}rPdnM;M_=4{Ap&?`Mj30-?0%=}|6vinFJ0(C`OcJ=$|$Ma^P6FWJGip`wN1z$9G z4Cqo;WA@Rsut8@Ql*mdKbl*Ac6dU;i&F7JLDK(nX`8U#~actk*uW-ye)l#T2xdao_ z{MTQR ze#OaMq@0j#(j(X-yOo!vckK6+M_rsmzC`J;KzE;d=4Vi`S0Nr?TYRq+9#6Jv2QXG+ zucal3RbKJZDKlz0cusmkZIz^7@q>4 z*$@Y^lxIG3os@q@_Pe@AxS7&Tvky|@aZ9xomjx3kIog6k6Fv=hX0T~MLKM`2hXa|w zqW1SEADTmxWd~Ku?!TBk_t0^KT7*Y*hclTVNvB|$u(O65N^IsZI3$m;_O(qhB z@NPe7>zCN)R4A6;%eZ|aIXSRY>FOky%oQ2HXgA#);5_M+yF15mBo91&GEMg|zPL%j zvaci<_%|)ouhYLONb%J_2pNbR0EH!ZX0JKF$Oc{ii~+ccI`f*pj_R)(w$EWoVY}}@ z1-{}ofW1C0z=Rp{Hk~mSp>bH>q>fD&RNFbrBb;BSfJ8PWw?`no3m`7`mHl^yzLGKq zxV;>RN2ad0rwsM@G!&kinYsIbtJKkCeVzj%LxN)6zx*9`3ZO~OMZO5V%kMA`J)RZ& zhwezOm2-{6+wb%bOi>S;I z;$hOPXN!p@ZB!CoD$l;8;sPCfU zD$3uh(>n~UjWh-D(C^EB_oM!psCwVHL`TzwGrE?)sdDH4GsRplgX!l?N0NI%9QAl{ z&bMMc3B_9Mbw`A{nyStJi6JwZ80`n6Nd9H{*Zv+Mh zQWwEYVIoQvejLLG#*tj2p|~l92R5d7R-_cvQ-YW9%wV$J5vi zO|pi%%%ucz-s0X{S&$2N&G`!M+O%HSR3Z!3KmdB<3hX(^48HhG0d(k)X%jJN+v2FQ zQ=7x8;Ft0FL&b0?$YIQbg8pMG^gRb>YODH0`(DNcX6WakkoPhjz>pI5z@K`v(^2rH zNA0kR!X^SQPl`q3g*;l@M&x3souR&CqX)4_a#M=qXSyL!|B=h2Z`07<(apGK09Lv^ zNtQ!=#kMKpPsuw-;`v_BP3Sn$O2J6H8E*MmGC_Ef&t=t%SBFVYqIwc-Joukf-xEdL z*;_I3c+}1CROGxdol113cZTA6_vROsYRA9(l^Y~|OY7za_{HX1=n*D`-YFJSuaY?1 zxci-&BfPHo^%VsfvBw5Yzuf&MdWSUhT^|_yB?@d&Y>p!lc}qw@QHiiCTkmI@brd_* z#(fcva?mxV#o<_czUQk2;r~6q!r7H`G{|-PFT=Iq*a57m1}{YWd)rO}*gmI+ozM7s z3AHBcZ$5*Ul*hdCp-H}2;-Xj9po(W9rLqs2Yxzh7KNB}rdrhm)$0KSTv-VZWE&E)} zYZx}Wm7OWoA-?L&wIt4wOk7kX3h8KRbCv&9u+Q z=i z0Hz&t)9m*V^{bHm3aeeZ-+GJDJJ&6Ke2tYhE9cmDXb?lAVPKpR$e0>EyiV;VRWN^*ke2v^@4?Ij32fc4I4SslT5izf`fBH|b*ybULmDz6Tqm#%*q=P= z(glB&QFe;0Ctn&Q)2bO$?Y_&`rs!Sxh?aD~(?IkEI4acJ4n8`_oX1`<(1p}Ka9K*O zg`xXPLA?3f|1R71duY`O-x2La=O7uc5Hk^7%p`Q$7PzKwDLorF%n*U<9cwk=f%5<* ztv$;o%q#DsN)g+HIy7MAeTBI~jCQn27!+~_R}Rsci-CqJs-sS{4F<~)@v}wj8S2t5 zQU{xIs>IY<1^LVV-hClyv8{)hU)txo!s;WhU6dk%Vu*SS5~|%y~D`#-c+lcPCS}_qemCJXHUrn1J?eGa2GTO)8(n$Crt-=Zle_ zmxgrItf@9|MA=Dghq#e>t@KC0*gEA6pZCDL{VA1SgN6OMnHbX1UA5+PCp)QqE)4}u z{WYu?ztmle+}XPuL3XvX2>pZC8ekRj%M8JnFEt`%sUj4>y9pm=Bh=L%mWs~%!NIT& zUq_f4#jbgMlL#gwjhK>Ky&#sq65r92dOOuv(`-T&O!hupv13kgtpWoFezHDF$D&Qk zc26W%0O`XpFY{jo*l9EF)6Iq*sa*8L@>hE*$CZZ1oHj;5LFuDuGn6C>G43alb=#)| zo7~y_^*1X%0sDy1A|wDicrSIT&m#yB+&LB2Kmm!+eLDA2RQh0cYA{j=XXOQkkvYZ- zX>40>wBW^DP-)e^&BIO-1X=I zeTV4cwlRb_{QiL$ZK=Yqx)+rh6-F~IqaA?5^gdA?jYy8+p0o9s+C8+$+;Yqn#!H zsV}lCw+LeCZ|W1OfIbG{pn~kYuw(8nrL2Np$j-PQ)I6++rK(mW_$IEK%&<)mz!$xa z!m--3D;I{kIa&pL!t}l(WUwp!yR!MA zTm5_m2>Eirf;xyY&cuk>ihy*NC610cjCQMnll0TT+TGTXN>JI>7?It#10T|rL;q%t z0>}PdrCIeU(3W^_*20<1NDTDg_GHNr&=@Wer?$D8kV>B!Hi~{ctm{KG4mt=M?9j)b z;F?3u^49ss?{rabGZjd8b3@FGeRrR~Q|w*_q)NgYf?Xl-;q25bKVX^+G2 zAcd&H%e_{|RoWw5du88PVp^4* zK-FJMerdMrO!gvA^Yz<29v35w%;Lp+V7k&w@|2oDg-s(I^C<&0jxeSQiY5~52AsJ06+ z{CW;mBqQ;j1$w2V`4KZ7HDBHzFrRp+I6@{MKjw7Mb}2Pb3Iy~*cJc*_|3jvIe_6o4 zu)Ke2(hCTDi#F1O0CcS#nren3r4E)tM@r1kSHsAAmkl70F0 TaFuH*X7xR8Cnuq z<|L{Jj@*RRo^VtHq;BiPbI8;4_f@!Vx|<&*=X&hkeSc1`;+3T3+AKukJml_AD#KbC57ea9?pb1cdnWKNZ*21L-{wO z@at{$fl+KD##WcQ=LpUVhE_s4u>F$BP`_&F-jvRv*sl%3H=b z+iRXl)hUs2!y`S?oaK8$z|NZ+9Cf;HM$mzDeAl~U{loTVO3K+Qyx{=itKU@?8eE<- zhHVjI)^REBk8`xDq(vd2i`5LWDm}S&1a@Qce(*Yte+@j2+Ul^&74e}!zQE`IXmOwd zJKvn%QGst(+-4yrCiK)skq`-Hcg6hzBXBHuVo=!Rt&*e_ZYU3hNRxm-ezaWa8E)>X zT$S}yNSNchV2RK=Us^*DMiK}1Ge2Y$|4BfbPn0i+*^P|v|xkar14T?Fk!HDBWMF={4>mreKxGi9uUNd1>ra5C%$K_-FE2r8<`-5B}hf&Z?VyhB$suZ9uP#rH03dyj~+#oE{N>7;C zg)hW}91nKD*!9d(RqjPu2oG|0;JjhDf?5NSNFAJP=to+zHyln4<{|^|)^KZPC-bQ2v+kR?@IGy!OKOq!@r+067+{+XQk)rOgSEnY(}2&L?{GpW^s zuY5FP%7{sA>TGc>7|)72^dK_|nWb*AzwyQ<1hL}n9Y`>cSM0^oB2P5&yMUj?AJ;b; z-mmKz_~obn{~*EdLmCLdH|4ojPge7%UFUcA`B%#SP=WwI0f3ZO*%Nx_YfQb@>q8|9 zA7~2O^aADMPi@GZe;uz@h<)i`TKU2q zd$fTWz*6stT~T(An9Xk6OKJc(HsjL=X{k+j@O8WU`|nj?1YsZWnAGAkB-CRT+_fm}OF=$8}TN=(GIDnB{n%a2cc z)F4GQG${nyb-qt-%a6}^8HcS>JRdzNHF^28?ueUV%u%` z0H${hy=p&q(y&FQaSUW7t1QkLk1301O;Ue*&;F%YGBoE)`0e5ks);WRnVfs-h17D? z?3cRu6ZGZ^E&!im5hDIv!lQkR*4_E;)>#>eS0deDU8*m+Q6gD3bOX^W(xj-$3e2=C z6$U!(sU9C>qat;3eHs~>^AI<`41b70h55I}*G+WFaWKrhXb^73W_kdxr}0L=XYf22 zOKf1>d6|I)-+RBE&5$P4nt2TJEntz)xY$2#7f96^*9Mq46 z1wCyF{1E5>wtn%#gH%m+uWdl7tMm80Op>s#vPYEOXq_T3kmOf7e1DHO=_=MpQ`Q zP*&K+!^f15VL$_J_M~>wF4>UW5Ck&!qYrk$OrIR-EGt+HGc{6>AQqp@dR$jI|kvJPFKPC3Y zOJUG$<@TWQ#WsM!enQ6m+8swDF9tfc$iR4J_ybXE&G6ig&JA?6&DzvFXYJxTzYX+- zJtR73iuv0jr}1MFDx&GsHl3m%#YE@xm`}=dYn-VqTn?QPAZ?=Ocj1czoxaNU z52u+}O|$RhKhze&w}+tCsHlhb2xa{Spa6zbxl~wAfIqs*`d;(S zqT?x4upsE|XrdkwRTv@qkescZERaFz2oy=FiTHIb86rlJL*M+Sph7N@Q>`0h=o&Zg z3XYClBS!BmU_~TA`SNjp*o3WJA6Pe`3G)v%gy8H5NhiftHp+YjvS9~Z6m4JOsXUtX zbXBarq5b{~1M^xu-uzwtn7fdDCLWgKZ(iVUEoQUa3k@0HeAl?K#any;2288WQDQy^ zT1{ygnOd&oVfgdeJ(U_bGrI#_qYYwNMY{N!@Dt56L{>QLJ}3o#zXS~St`@gOYuCo2 zTvgiz=iOwX&rkFe7~~!I(i`Sxosb2J6F!>#d)c$jahMfYUX;5`<0fSz(#4{Zc_&Y_ zNh8$!5&|L$J~&Tvndkzxb)BjY5%J=?XcSfkPY{4nJt*dGZ)Q5n`(fC8EU5H71oOV`JWisca9H9+^b; z^<6t?=|{bcSm}4Sn>NYdrJ}3OoNckiS0pv2TK*lL1_J&br*tJ-x#}+C%)xvG04%k( z_oax+?JBoXQ+V-)J?MTuqsM=prAawIZiv1k))#_n&_8exw1V{7fek8IgVQBAx!R)YZz6iJdwywdeLV0QW&i>3S>_oMWM1moPi$}(`_4|HnpifBV*Bqy&v2DUDU zwr3seYb$(}5YTG3LKN|8Qyzg}n=?@ka@HKDf#HO}dPGZl+}z8`N2)@2Jz zgVgi6?ovq^5Ug5}G=)eMWQ>gX>hJh9Bg80_AS0xSuH6{3ne@!e0&KRJ@w2{}Lybq^ z6FowZN2Sd_?#ylu`qaO3fOCHOQtwiFtt*To!<@W@yLPE zFu-Vw3(07D~z}d6)$;uS7F5{j=p;ZwwDBgAH^wx;r~$| z`RX1jEWzo6i+@^h$7{cw8-_n&0(V#`v;UKBd7LXS8VCUc{4GZZ{g2}J&$__$%>Z;R z0aW#(`#uEJ(%T6ABD9Ol7@$d?GNgywkOcb_!wBBj)FN59k^Pq0iEJ^l~=T736pM@u4D!Ty&)hZXcU_{=7V$> zL*wjgz&ad-!9LBA84;BCmRaFpCIcmQzqI_AzN-a}<~$L5t?PDVcAGqNv)szaH_Ofq zcud6eg6_e&r;dKb(hQ>mreE&FO}v_`3Fsz*=3*8gN*aJ~Yl}?}sWug;R8rANPGo3| z)1yAL`SJ()V85+Qeex;r(B$cQ8+jY#PQ*^HT_(H7^%;}y4Q5C?(pRLhrcB?O%2vR$ z%-L$GVY)Jn051%?DOl4Jx@~)+lhQ>JEL2>Vq)_)>W3;3>cafaRYgjFC>d#QIIP+1? zp(KCB?=h_V?ZX-oH**V`rzj7_qgO7MO2&S5g<4Ek76|P05WK>%*8gaqBAA|_37>NW z#q*X`!jUh;`#)mfKVd+%Lu>H+Ckn%gWZx)sb!CZacC2NsGYT`re4SSaLmTnNQU4l3xfu1J4P=wCWZtMNSxMd5j8L@3-`XKpC;U|X(_9NsD)Ie~7*Sc*Z7r)#p^ zB@LYR6UNjwD0Giz&oSMCTL%|M1&7Oj=x&nEWab(N`IoU;_LD7 z#2v=5CAaqTM?#B(xw(@uJJa;QN%YYk4YX!~jX(cMnu@l@coQkOG##k-b& zDbatTdE0hcUB`HoHP6rh<2C`OeqjS zwn3Wt(PmX^!M~xEVnTK4IPtl(4>>7Y+(9|ddrFMPe7!O9Olu~N5gqCMn`gZCGqSG1 zg}J}n+yratE^nd!Jn64;Ab^I_>zclZ83fHkhAaKb*|^dZpPURK>=16NilE17l6Clg zFCm2;(SIP|^28eGvUN&)dbF}XRG{lg2DJ)5;6r>NRRNfq=^HrQ@zkfJUa9fJ!CGPc zfE)^Y#)|VBCNrII;K|1ehm=#fW0`kO3ch@fSLAEj8ii91XpC&lU3*sfi8pZwf zS4|o}-3+3kxLL&O4(UfLEh6>6g&pkT4+fvBdHz1ueAZV%|B=xrOnyVb*lz9L-Ro~_ zo(}j}cBp(L_{zMz?klzgjg3u`;7F=-dLrJ^j2*cZ zz+(-Xru1&+EMeLB(Rbjz7>Q%wEU{7RH0WR^DMTN+h(8`Uo>=kbW+PF6Jnb53l9t}4 zc&+(C3)k;=~nS1~$f;B(uo>WR`9r%{%E7;Nq9g)F`LmgYd;$dyi=d0lh2q-|N zRy#l@y?NRSEkiGfWf~H-g5##zlEwW9^F-}%k5j_$U%uV{VIBY3pCm3t)X$WS-zwSn zs_1QbVp;zFCitqK!=1io^<#l+*kwL+Fi7c32P|{1r-|W2rxs>$zWNEo0{zy499jhc zU#m2i)S}pmEsb9aW}C&2q;;Ju^>$;29mjt$P?y>Fcc+whA&2$TXF|+hnBarL0eslD zSuVVwU47mDeolLRn$yA39DObmt?ZR0ny4O??^l=%TO!T{Q=VN2e(EH5K5hJti+K2r z3sqCA(VONfgXN3q$FU27XqL-j7CytjwNBNW<2@w8Xb5k5d`OnTG?JVmOCL>z50LH? zREAHr<63f1!JP?^sXS1Ecdm2iuw>fD5Bsy4Col6Z%+s3@K`A(&2~nnfT2j zmt?BK!Ha#aP{$d;OZDOc8efh59IH;P%7Z~&$Q)-rh59cmjO)7z5Tnj~y{LG+t3)0z zsXLsg1-6rWeH|v|eFxyje8IM~PwPi-wu58Q=xa|4S~;fK!3KMVV`@hv7P4C_lx?AH z!U=mm2SQQXfk?^2FInNfqVO*vw|ACBu7G8QnJT-Rx}^4+F4{9+%+cGdcAhYdm9D$) zB>n9*tiLnt?SJsdZ`HX60Pl7D$Ot>>iZcDL8L;>7%Nk(&_z*MLNN z-zu4%Cx|Pgn#zfsv$rajChKZLWh*{E$22&fXTZEmA}o0VfU|^e8^TlC7VnnjfCc#4o^v1Z~}2GgevQbBzORO zTvZL0H05>wA64fVp4qmv>)5vKj&0kvZ6_Vuww;dIv7L_9v2ELVPu4nT@9#VR=D#`T zHO8nKS5-atgRGb2NMkXd=GEHD4I)QFcTmc~)%ZJA%_Y(L&E;?@&I8=m!Ep(&Sx_01 zV4uUgW^JL+^EDB?YH$qOW4*7b>SFI4KjJq6erz?#V(xVat3}F(wq0KOHo`CpyK1)x z+GeoA(s}oT7@r|L!VxXIGCqTN%yJ z(Wpvx!qz|t*ayy3T!Q7D?2@y(@X@L+Kzpnw0>X`5;zWh^RZ8IQ1d zqz?Bk@U$xg2t2BA82`XS1Ob2IwOIeQG7b9W_iV7v!=Pp$n8 zlHHq`pi>fp(6a+iE5usouo!hJ{|`O!HC>)a*S@{0qsc@=hv{Yct1_F43ULHx{ZVLq z7+cuD4^tGPr_q+p>~_qq&Wy%QE_+&bj#a;o#ea~w;r}I1{ zudRlqmQJSb*SOLy;*ZPRw$yB&1G|X=NSICxE1RG6!U9yWrTe4@W)2T}`5|Ot6u%=8 z{!$=UsS5FmxSjsCLnUGr`Duu&2?so}*3FR>;!-Wx>qru*S`SUO+I<95$=x36GlKw# z$9u-F!z$44CJ+39M#!>eGK8|lwBAS92z_6VL>n9+k&8(Z%8|RCdjP>3sUZ%e27X6z z=Zz4%y$h#%m7RmDN-5_HCftP4O-;>wd54NV)=dfi>^=p7CrRcnK3R&mS8U2xtd~>n zEIra7Rp4EtRE!xe5fzv?3u;Vb%?AlpO)|Qr1V?tJ2}#}s)Vv1CloQH>9j8@`JKeBU zncP&BA#-#>$zSgjQt1ZGZtb}!Ny&W(-)~1&Ml3XKi6q(CQpNOYFFvZSy`cRtS0@m0 z(1B`>^Ts%sKzLPY<3N$6tz{;0KFp9%m;WB=kr)IEB~4Kfgda4p`Dvly=d)(pK)rku z5%51qNfoN&AjsKbm8L)-pnxE$U|}HC8-%BSk&+-lPznDaCF75s0RFCogKvIU{J|)I z?gzrggSUM6v+mtqjgwD-s)H|wx!WFzjpw81pQG&(S1%znm>@J|8>ri@z9( zKc8I#81ps0(4>2Bz9EwS059JJT*|#TH z&i7mn6Z~%KoC3`N4DB7-iJucw*fiq6V8SsB z39#5edvepX4s+9m&M>YFyqo;3)~P9fe)1bN^>8;RWt>_#|CSSo8_C z!q}BON_6&lY|Mm>%oH|mHuzeM^Jrikpy}lzaMW!YwDz%3WzD4Smi4-H!h4GAtTdp? zgd=Bxr_S>dk_BtBa@~Ko^c~pRwPu_pQad14YDGcKQkzURt^;Ks!2%=#L<=QqBwesC`;5&H?@0u)5c>u`-_ng3Za;dT%( zL{OSqZ`#z7{TnAK;WQsH=PMowuD}$BM0RCEJ9$r6Bns@Qs9gg_DlT>rP zuXu}3#SrDC*8#gldt3(rUP>qVD-GXn-Vl!5+i}MZd=_uOl|me%s9(*=-0}>LYVS*) zd`x|j6NXaL9et|hj1p9;r+I!RQkkM6%lk&xO$eAN9QP#@yL6gBU-sLIPdpLo3^i`e z{U$uLhJjG~Zml{H;&3$^=ZmFM_X?s7Od+;QTi6iYu$P!ia@sUK*cL)v)J@6C62Aj4 zRi|!;SL~9o?DDrHlOD?;!q0pO<<8~yqK*~C$L-Xk^`{L+5)mQWHL^i(y27b;v0DeF{n2)bSnq9alY9w;q z2q0SV#OZPFjYS49s~@@g*CdnUAQ`(z^q|pXCGXm7(zBN;aY)&fX8J~>+CoBLGjfYW zE&R99*H4~flx;ON6QiZIT?0VkJHDf%%-vfW#M=|rPo{cHqmurXr=PI~S;nqv@C(8v z@j(R6^ZpXOI{kq@BO0-fIcesOo*}j#0DAvn4zj5A@`Sbfyb71jmZcIa{|tKt7h@f- zub}RcKW{{K0UGrA2&D#9jRdv>AUN|1LUO??6Z^>ZDr%!2PL=$Xvdt%*VroW!P-=h1 z5*+pP*RX?UcJkJ`6I#DpHmpM~z@}>$6Uz;v&7eq*Az%G z*UHpvsK9P>>YTbS85e_8iKWd_WP0_V6exWWact19y18@O*XeC}I&S7mJf+K505zxD zT{H1ACmGxwzzf;u)|DQj5MDW5U*fXA9wBL*EJO=i9x! zSW+jv?nFU%unN2!L1^hX=gbd(1Sy9pwYi62bG2XfQACS-X=1AbjRtQMMWRs=M?npF zjdjo>w?QzAMx2qbSH0dBCgzs%epMss*^{cE@v^(Nv1L3akM0{wb+9Lw=m+e^;ghuR z>FYq2XBxYVs4XCN$2;u>v+u8kOxu)3(PF@P35Pt7d8ZBVd6^ToR-9ICn`mCbb0b4e zgSj2+rA)-%bcwFugkdx*21_HzucY#x-}WlytsDL(h#lQ@HEDZgL%^hoyroh0grqBQ zZMLx0ZQE}j=6|Feb54hFs)NTw&pqq=UTf}0M`r}BE~?%;vb z%9I}Dm!%77H{dZy^b6U%16Q_Y{CORq(+{`Xan zIm#UO0GFW8kI}8O7``w|0PuXy!UQY~06ZazTm#T5@CbJT@67uYxuzaQh~P zi#z&%8V>b_*!>EdoWYWCXM&sV8GYz2<#WR3`S15i*Zj^2e)(=Ye&MZ&+5JdA)e$MZ zd!O5YdULp7jVsHtiLqbrLujY+lg)j4XBRRCDA;`1?@6b;7?Mx(>|RM?DFm3SGqI#q0jBcTQ;#q|!Sm*kIWRMf-+-Lvl zeSQ9Td9Mq#xz(0uRgJG`dPc)={0U@Fn!K_=-CLH=qkSeIe`xOd%7#EzCiU zokmfsuEk=nsDFv{StNZjI~5SOMTx6jqxYSf?fWn;X(S%=!%DGA>m4iCbKnDoH*j>@ z3#+XDl}W(95#K5*T}s$@C_|9s?i0 z0GMwy^2TnpWaOEq%p5>da#2ypKH-@VE^en@T*R$$0S71CP@N}jFi_aktxeW|q956r zi;1W1K&luT?`#hL;9uxWsbj|{st8J#i zoemEnq^NoBRtMF;aHAKJQiR!-X06YmUF24W(Mf2d{Gg2fAi~VS^32mc^~_%^!VCU* zvNy_4zJ2!{2Ia}0&W3DH=m@Ak>R!0iebF?k61gbvY^Ybs;>sJahz`(SL>QPW@e3kI zNq`Xale)Ic!UFuTfH9>^_sD&j_u+fsxd{Mv)mb?3HY(B*_74CL`4Vf4{E_dh+Am)Ue zDv6|M>Ue}cwmbR9l*FD-ke$NntA{55zSFE=;pajl)Wfw~pZq1Ozti{%3f}|z!-8Jx z<>KA?g7&JWdJt7{{5BNA?%$H%7tki&EN`iHwm;d${wM&RZUOvWAjh*EEM2fa;42q8 zLl?%&2;Xr@#l1-wD50K<4;{8zne>-N3PMT)le1E}M0*86`aQa9bdoYq-ZbnLh*Yjq zyn~jsDHU@ecDb=Xm1|V~k_$d2Cizmdnshj3kNL}Vtx%HtYu{eDpM2@$qQ|CprT}50 z_G}k&wRHxs^t|})LI`xaYG%qcQgV~Jv2$C7S=>0b0={@FF##IpgO%jG@f7K#Gr=k6 zR4OOq40Lt`jbLJw%wz#stfM(<&TPrEfaQI5W|fd*d31U;?+C*L1U>`6^FCfHIyIq^LYmG4yB&Vck$HDwiTV92pTW)kX6O8 zYn9UIs_tCI1WIa{6n-R(>V!POx2dtzfLqAmf{d9n^qg zNaZydID)~hZ%AQx0CVj?NAs8}Sk`HHG677~$D5$*qLi!y29wZ4LH-Pwr?fvFx+&aV z`b%|L@4kS&{Xh)cTQX_UoJGp!mhQ6sc3qTd&7up4hzGd(m+kGYGTKZe@$+@|mOo7> zY$7e5T#89YUBMjzgK2tfZ!ylXouu@@4D+anea}ejEabNAERCw|lxOJrt>JqhHu;1v z^dB(robeqgBfO)f1v=hY$flHaDk1LJB)t^=Nmy z(+$Ty2NvVnaV-0h%nLKt}NYJW{$JW9lu&HOQ{E*G6q>~gMFT7X`r z9%@NH$3x2G+#Ac}3{o_eRNICgDfVmuCI`wDmk@HZHBB)McC&Z?>sD7L-G=ALlOE8g zjya6P$+V^QarBmowsYOcd?7^d^K)JXqOC<&wV@R7F?6c_n{rjw6SocJKso^|%jc9p z#rHe|quH><1PIwD!}m2+4DIeEhQ@Afy_7-JQ@LuL^q$~CR zp2(%do}^NWb#!1*-o)}>^}ioKuGcl~UK5+~qphr5q@bN?V`u3H45F$JLGn}3Jzlq7 z=-R7xF>pEC1#pXc)U=o4 zjMdz)d|}aCBX_Om2zwJ94M|hwjaRa(Vtk{MIj_Oz81MY+(#am2AoOp<9{kQWKXEi6 z9ZF+aHS1m}ne~}4*yBxeCS|9t|ZE_uk%x;e}ai_BH@jLxqlK;AjpyRKNnhB zf?Un#=uOT1oMB1C&3Csm5Xe{PZ}oRP7MaBCALl!)dfS8bO zN*(oIf}oKhohR-QN5hADQ>|JsgGUz}MLQ?0rq-8&SPmAl5Th1SL(pg{hnFn|-O72G zv+ED7(6FkDV46r$5g|=nS$@yQFT-pGpWEfi{(m>Vw9g&P?R3{6Wi?uoY)De3_(#%5 zwmn|%CD?{^W1Moqv#EB(Z^)-m>$}P_>kr17Bf1-8r5V^dklzTTN59CW=KsMQf?N3MMlp&3e@6uuu+CG zykK;obl1TkhV8A6kOC6nTc!)gzKu#1MEqHJ*iSV{su@oNXaAU$A+FsH5xR)Jri#MI z+s1?Ur>H`gE17f(dXeE8-F1@pK3w#w6a$mGx{o_}&+0&eM^1}KNJMu(u8{rcqLeeD z#TjkJnkJR|t?k8-#R@=Gh#>$lLVM9YO=izmEtoUm`Isg@%ef4%8}=adh&&s&Ure|E zbnkK&LqJ)GL)#_LWo~ugA7fDcpJ#f8FAd2T9hP#-N+TI5uv%Ndhqmn;i~u#;v(VlP zA8##@vA^KMnhBZDSJ7FVmG&B3deOpg7$TOSZNsdR>^=INtDZn={o>$OxiJ4jw>|mU z6e*qfKz_|PXr9Zjeo(_on`FD_awsoJ4|X}KKQ1_!_4w~;GldnreAXI|ncE~?@1DzQ z0u@SXJ#-Q(EmWT=5>U9)LL*ZQ#6S|0%tMAv-CuAQ6jOq z;oH++9%c_#N@-#21<8V=UJzD6Ho%{UNRX2E@h>QzAGn0YqXe&B6Wu7EyGi1#=%6>Q z52c0f&egsPTPtc@Wj|$w%v#B`UAKCHLZ-!?BPf36o0cmqo1DYoYaTq`ix6W=c=6Mc z#d0!}HX>d7%Sa>oVf_JLavdp_g*hX6s)QGwHZ5ZhqmN z{p8D-o)9aS;;F;0%zMOJZ7}x6Wq_w6CK4>rBd}#mV{#%3<}%aybl$~Jn|iwNd_L~s z2RUuEEt6mVah->B+TUA0E-;N`)*HQEeAD|C{0408j`e9u)i2GQY>iv6hp2OI=L*Lt zV1nI}M~j#;Sp9zJFcZRKmaYz+T>LiW*lci{08yL!Cl|!C6iY}*Et(VpC(l9Z76 zvJWAqmOq95%gqhl6hNwW6Reym+;naR9>~#Uz(irMOpw?>0!DG6Z)tiD3fuM*88@lS@ft3ydW*oxKIo>}9EX3hqPra3;v7lzdJt*G1>B1C%k&K8K*8$}9 zOTkf!NKCSID>fL>I|jp=&%X@T9Kuc#$Z2=!lAX^OrYpLZA3AVjXam&{1&?&-(dLQW z$@rZuP~2p9AL^0@eF=nd4bn0$s+fUr!g((Y0BAt$$p14n95hWP#64pX%+;&1Kd>wG&g(`24q1r&?%F+R8uo2v4$me7~iM1%JSA9!91tBtG& zx{Q8d0xEHg9-YsPEpXclyrf#zN(US{1}RUbyo`?D0XCGFzT^KnfqjobZ1|@Nom50me9Ts!%ILv$MDeGD(8rhS`ws;ICmjN+PDI9~57*_;&-{_gNcfcTd5sdoY| zMll`sJ}U}HW80T_Hz3D*0ns`ZrR@R(?wVA}4E-W*#(TE_!XErM7v=+mE5sq*xUb z{E{&#zDI*YPy>Hrw^A$SBdJz$bFt{hmXU?)HN0kyJ9qagU+F>pzOQF5m|e6Az=BC- zmmvhxgXy7OoM=WDrLE-^QuW^##yvBf72&_Yd6FJ=imKMR^hDW;QKE$}n5VcF;vc>DK^uI|>%DnfPmr&A#Fi!X~cAj0Y`Cj7vwVEfNjzen!*ZU zb**wfXZ$%LWBIkVVGFW zB;)Tc4it>-r^=DeMRfI=6NYn7Bu$xFfev8Tr{Coy?$DnpVOEba3J!uH@WbJ(Gr6!m zZ|wx+k!b^hy+AiqbAx0AT-M>l0ZP`vb8DgnhDd5`9HtxmGZPmy*uNXkyo3W+T8N5yXPS@bY{y7^v>7`+3?i>xY8p1y?LVp+%3ymY-oWyf~HHd?SuVnVm zGeg0*DU`XBu{_A)ys`PKXDaLIiqV7*q@}2y9FcJy%E1Dk##Js(T)|d~W0?4L zL*0WV8ww4=NqxD9-$GIwt70IC;-HmV*0;2!Id|%k1@z&FSRn&C3h#?QR4N*ae40U| z&qD6`q-qoW*Y613)}A}Y$l*dQXXmW#LYyC>qD~{kdkNe>)>zc2yU@Fz??zh!qfN%x ziNgcLdQh_sE%^(lLD?qrPT~GUbTfrT!%JZl&Mf=7`ZsOm3qjkUY#vjs`6ahnRSK+_ zw=n}`gURgN2}}mfvk1Tqd@szRl+12E-&vAXo1s#dSkbS%EJqpG@m=XHxaX2?YOq;W z;Ccb`BZZ%{Dd*gY>T`Q(U%;eSe&MhovbKVUW0V#9)643*Ta0O`gg9&GkhBf@xs6D& zfR@w*V3eSk!Qd)lAy2bU$hIu zOUgn+SX}R%2_`ZnP{e>M7DF?NPdFk=)cD4{mMWe}30yjxn7XtD^<@t%RWFvT8bta{ z8jIErROeZ!$)aahXq|Z~jTcQikpj~C@HxbicpFtl9)S(diFb4`V*E1mO?mSHVd7*{ zEF|3@YM_%m)|nu(aDWd*f44|qUkhb0G8=|WS)I3MiT8i z9S_7ief(FMe39fMMT^qhGR>47t49_yt$MJQi{b8%ncI09B9E_6w%?Ar^woof3QmxG z#BpYba>^?6YanuzBuqHX`HAMHIpB<07Em8)x%lJpp`}vvNHTG-IQzn9u zppT$d`)$>tXTll#i{WiaSJB3?&r2!s6Lw|YL#ps^4ofX`e55RAa6*xMTXtGJpZy8f z?Zlviw)E6%FZ`acws~Ie^cu~{<_0Mn@*kV|gSFTPRd*d~oIQxkBNJ$c(?7BPr~f!0 z19MFdLwRjPM-5pig!+nX@f7`T*COP)@@0D31WnyNz-ES>&xf`>x)Ah&VV(j$ZS;Zt z4a`)eIcJ0SEZfF;dg%cMKvh#(q9YT%v?GJb++ANRR2&FY_J1?;Ka&zzjzx7{-`aU} zujCXz&=+-_8^bV0A_Dzssf!mGKrY$(Kr8JLX=@FSg$9KmqkGU|F@ zZnam8er%{*!b>)#uH3IsGGsSubr#$^wNVG#Yvj%vgCdw``-p05p4&4On-Nju*yTw` zQ!`JmtnAmH(hw}}u>z^&43FN_p{rZFK?IEWymzEGco?7#pSBAUkmj2hcBBLCn8JiKvAmEw}F*DscQr zjaK5SBWl3*jtdn4+8}nQU%g#|2aSa@cXw0BvyOPa4#g16{T>+? z)Z9GN6VF4P;S1Z(B6PRx^x(Uwm)EWB64Ss@j3Vj4mR$`Vfy{HOS%!=K%tU0XcY^^NRUxhA1=k0z z=(6yiDGu&L-+M#!u;@KxSfPO~4`-x-2z{>dtq~zDKe?>4H7o*_BVP;9{W$BSFQ*ij`?#iDHnHQRG6jGO#M#PZp5 zF;@^ncoJznCW$U3d;!7p&Zj4OyiP6ovNOtc3CasLBT_7qcO7B>12t5b`Z*ujfyU&%8$y=<00d0Jwuqky@F{3mr(9&D?5(qHJrn{E{c&m}EX@Jfq84Mahg{ zR8uVb+<3T6p(6%IPmU8MA`4Tu8%*Qjlm}BAitU6k72P1X$H4m)`3<7L0nBisrOE@= zH=UY*W}tJLu6lib(FX+9Lu9BBwL)f((09g?Bl5e*u-^TK&;wJ&ZQFCdYeGe!wz7$~ zbsAq?CQhAG_N5TBh1)O#23s9c;XSu58KRpoJSsHwj*>Cyk&Kfv>k`rS9yt;>+Z+SD z}IY;*wq_KwLtM(|>ZU~$(J!Ot#I&-E`hqvrI{5*!M| zErjfm_??i7_6p0RSc%pcLv(ILY_P3tu!b>eO;|4hR8VKZ7R1R@rQw52%s6C z7=L1#oyzkB1>_q@#~?zP8LOj)b%-?U=o!csP<*){(<7Z_Lr+(W;K}FpM+%xQV8i<% zUY%r1(AbZjiU2Ep-{J;PLZRpTvV#mxgv%MSe!+Pohr!Mt5HTHNR7I%$)r>)z*@Q}0it$Zv> z-E*6(s)F>^ocFOKnDl29g1tE$wiEAB1j|mOL{Cq7+fPpSsdWNx$>^sB&+r3w&Tp=W z%2Y^xi@teXN`&H^%nGT$eNxIE4R~mQX!o6`S%!S{79~m4OO7&RF5}^XO^?Psl*IW9 zeJim(q*&)8C~>r{Bvf9ihmHLSrA252+ZHCoi+1p?X88_51QhwTN0d_&lBBwUoC1_D zgqc<0g>JWqvE)Q77m73c?1-UlpMe{6I5WeY(BO#$L9^eKv5VbpZG>R4(9?fdvp4@4 z8~i88dJ8;LOx|FhI7WwYoa%tPlbG063jw!rP-PjKH6N+WTaH3*+b@=pVY{hSlt6SepuFB&D3ER|!Y$5eOvu3eXbgEx8q zP=R($%Tdd{TymiQ;`ZqsquXe4s*(4@#y8vNx71pU-5xY9ay{|V1FWiI@_q#{?E^r6 zlZfNmkoXfeek6Bti~R|A8B&$5P%(HKHpLd-Nw#W>s27v5bD5Ky!=A?jSsL|dMlrdj zy`o-z>}ERCFFdV|Drn<=K7dCgc~)KM=V<2DPT>mz!f^;N`dOR@BMAMUva$(}4MuH| zPMOkq+OzUXeYW}Ly!S2K)4>5IiKv?bCxiI%1Ze|*vJmxHIac(ykL0&ZU>G~4u!$^F zKgVIwG7%2CX`Je)q_>Jz#qZF4@?Q)sFzC0+&gTU8!Wzd)%6O=k9$7JtZ7Tps3g; zFp=Q7K9R_nTlm|!#-cV<=?+}okfb5yGj<@j&TD3T;TBf%dDVSw8jy3{jy8x0hzth~ z7yQaO4I~6Fiv6g3ierq#2YByWBU%QN*{f|97=iUbp$#_o|IJKp7E&pmCdwuqmUUvH zRyFN`E0BLA7H`o-zbq=eQgozC8(B5@!1ELWR2;J&Gia(d-}Gp~(DQpbQd-vm{f1{d z5ng#ob@+nC03mJhedVI1S%vivv!k8BU$poo6wHjkuOL}cfc0Kba!dKst%0}}2K(@0 ziO7QaMb11DipG4oN%rje^=&dgp2%Z#bLbCWQ>5FXREe^!r)^adFZv46tzlErW%JkM z29QXeFqW~y32u|g?M6^ zn<86Q_`0EDy0WPTmn=<|WTFk^mWr<{O2%}w6^MD4H`|2bZ;YE9P#7>!S`Ltq;I~co z&<3*mtR@hUo$44h26e)QZq&?YG`%o;GRN;=Xpj5-NIY{110Y)i@bb4E)oE9)MQHn+?;k;luH(rw(Mf+v0JT? zWJee5tF$*b!ttqqAMmq2?A6kL-Qe_YeD$^m=>yI!rr^&99R&(L&l49n^u|7;G9_4s zwzf|`k9{UIeVsK5!o(#O%}sZ3elYMA^A+-WP`Fsf#{R&UxBaV1-FZdFe1-EU=lf}( z71+w6yQ8dZ>s`oGhFQYi-{Y(%a+rZ#Rn<8axgr0a_rOV1Hz))~9Se_!3TJ@lxF*8@ zcV+9xfT)T>=t;PRx@2(CQTsO<+{mk1@T25CQlCaTMzr6XwPZ$Yeed}k2&MFR%p;Fl zUag%+A&9XqHW_jCybl*w6PI|8TtJ2ahMTKJPmbrs<5~Fq)c`L2Y9h5VZHbVe47%4$ zQP`gbS?FX$rVOgt<<{`W=U59vb~kcfjkbai9|O@fN<&7 z(5OpxYH%Y0w@@Sd#o=7k_9Qk!(%nPhd>POpJmT8dWfq7)TfS^w{iivSoema-jT!LR zrz90#9ksB5PWK`$?4!~8-;32)y5h4~tP~cis&DP{eJ_)QD6DJx>mOS?s!cZ7letC& zI|gX-&Y&-wtIOtnUo9yu%LVwxkyV0Di=<{iV1*dIQE&URn~ z=Y$q~80Vk`+1yz|v5wliv#IVrRvE~@&RUq59I-#mObjD zGs;n0csa!wm}^!a-@N-@U~2K@&c8^~zo%cYkpLK42>?P7OSzfv5FO59(1h zB{c}^!}h88H<)2w*knLr1|?kV*yqHsFm%4lOdhd=B6(t$d#YGwrK$32CzI$=k=L&z zJ1D16=~h%yIz*gTA7!!=n!!e;gp2cDjNSLW1$Vt)2|5h0At`J>A31g$fzMZ?dR|?s z^aHDlT=(fCO4;_c9u6bFCSH$wNDsXX5*QP3RY-pnoIKg|ilL$NZ{TLujvMjYqvZ&b z%_BttAt$dQU;}Q#4jJJah>4|CUh(ow#!b$qp1NeOkvD1T66Ux>U}-f5M~Nv3E;aoEsqq*v13}6tWM3ASN8U=eZrmfFT5oauZo?8-82zEI zbg=o8a7^o-Uo;8abJTYa>!j&Z=#d(A%(CEU8Yp9>PYMQ(LiLe-+HTE+Maxt|4FUOF z8l!fa)xU>mA+yi^^x<@Y4kGG9buG)a(twy~ys2W(4;o~{7twMncHO^f&C-CeavLLc zqLxBv+$cNTjoyz_oP*P^&TH(KhSaoku9TkA8LLljLBcL2*DT_v{FW^_UIvJVPQjU~ z;#cTB<$1I+YPZ-!a;A0ROrQl>OhFJ2<_Ag?p+IS)M``IczN9~{lJ~<&dO4-<5DA&S zk9Y26EAQFx2&{AXMHImG&l&gzzXqFVZC|gAL1wd{>zR1%;xk(pUnXDbJ$(U(982yl z>r2ZI(BrVut57;lJ#J(j;`9N7F#CZzC|r%m)*Q6)q)C>2(cr4jekpcCbRt?QPi@dU zs4}{b)&UgnQ)LTR_sWEBY^`wW%Oxaw4iI-!2RwD6YkD-ok7nj z=(oGNdnZ)eHHp-zSvvLWN^ELZ!i!gh&PiEwVUFj`1rL!cS9T+In4)+gv3p z7Gw*sD}kgOMEwSTK1n=1KyOav<$Vu+JI&iBrfjMc$50qpKGWd+qBH^R#R6-eN)oqG zS!uJR^_F1qU7WtuhtPC9*jLSqP*Cv2{leTd{84xT<}V}sHS2jJ@D&fNl=bTn~gdp?gp5giR<&Ha=7iC?M?s8@eod6R;OsG z1Xkmp7Mk_w@(0PKkP$%uDGqk1ML$}419ZmvQ0iwKPX=J=!nJ$(z>hpkIfZFuTv6=q ziGMK2N)K=7vCQJloGrfS^mg;+CN0P_H`THbUfIb^`B71S==l9IKro(O$6Zeq*}Zvh zRkAVWAT~pO)+lQU_y^4UecAuE9&ir@j~=a%7{~GXumsXXwxbtHXLynKfxI598S#ZX zK{i2l+x`=KLx0kAb$>3g>eJ@=cmd+;X2gdla9qXQ8=^45RIryy4ZB=lGBAND1D!l+ z^?54g%XgAfw_jJct+vwG!~|4pU#`QoHoOhm^XSe73_u$LQuGj}?jfv69)|~|DB;J! z{jaTpnxDr&$pJ`9N|6$7GYa6yZy#$&Qb&$>8X#B*|R==vN z+q&%0EAzWl8-Op6KXk{TN2;{^L#%MUsX6bD%f74cPox!|q|xo414u9fE@YNuzORok zbzZ`sag6Q^n7#R=#!!5tE$`Veg~pG(zuD)Kb_02%PRQdia5Oe*NCK_AxJ_v-G43^G zq&E2QC0fcR2s&`hkdX|AYsPiXrHE97GE3@pAg!<4-E-l`Wy@JWbna?b732bmE1t$Q z%u%IM@hrhxgQCtKav{t@!vtLxJ>eJ`i)QWHE!~N!%FO}Il=i=bPJGGrHYr9TCzzM( zZ?F4OIt)rKIQg|L%Ms1$@By@fH|1?iSC?-g8$)Pzx;L2o4p!Cxc>`PFk!vVvnw6N_6>%tIV4KLd54ddY(_%Hdr%YYF{z%#2Z0M zUe=DBDKMoZcO!NlPfQ_j9!}&xnlM%iiSkd2W9xtMeu01le~Osv0R#+j}s>JV2Nl?)6i~2-Ug^FF!bUJX%rFn3u@erhZv5s zE;C3@xq=U#=BRIz?O!Jz_&N*s^BTb~3d90Au^Qb>qVx^J43GO+#&YV_YM=X!v<*D& zPtV(U0*N+3Njw%+i9EXPMN%?y2mL%l%JfNO26}vE+BizQ(;6X-i`VzHB0OcA99XPA zzYX{gt7IA3PV3$J@#gb6MY{ZR2Bk|)dqf2t;WjhpjfO#yK(7|w6hl;(%I6v?U7tR? z!lLwTY0pD82v0(kR`#auiiITp3;hKX`p3_Lx1g;FNC#Wy zW25&nQZ=1-N(3Wp*qvXrl8z2qGr;s7ER2pJ_mHih8pPjx`=EhjR_io($_fiz!X(i_ z@Q3{~9@7M+l9=K}@oi;Sp`+%6A%(BX>`#ZzIR6{wCP=`qFJ6F4z)>BK@5wx8Lmx?e z1OEKDz7_xNEXRme3MPNiPFG*GI??5XG_t51maEI~qv@1qaDsGzwpwwho_aa2{lEcN zqu^_DZTP3v*D9^W-VmED@z>RGAF3B`6LTh!ZyY`o%xJvN+R>#W$E2ZaC*`wnE27>~ z+23Gg(u^K;L>#tSg4IPhBTp_;II2;l-vwselG}$US3ZB(N4rTJpFzj4C*`p62`k$* zCeF4nhxbk1>h_=5QLUf2?{nfM?MBr}%iiL6hL;C$U zlK!K%{5~D`luE2vH_7pVe19&Lfv?4elcCOw;VPXA zj0)Z>BbY~Vx026rKF1_VSo-huZZICgF$~%Ls1Nw;dNA401JK@koytFMUxL+B~%neGLAfJE&etqbpJ4$<1KqTMG-{C>B8+g`{SerB@lHS^KucM)(-q- z4NuqeY1C?Oq$p0Ok7pa28aLZj#Tpvi=y0Ijb5@K0G<`?(5S(tR>wk@DibC2<^YsCF z{0kT|c+$0Rh^ztq%-U<>}#>blXq?)oj+5x5%jeVlpt2Lhg!*o%#IJZT&R!Rf zjm<+C zh0fpZ(R#<7ZxmBNVNk?|pR-@~ILi39<4>38g1z8VAjw5WGRD$1LK+%>fGy%dPw!Zd z?L5*hNB(zd#ppli-+wXC5RNM~zp@+7I3cdv^@Fq_L%q zn)=F5l%e8ryas`9g@cIIY^fE zC?2W?5c_!RkoLK;`Y#BES{_dC+?XHB_abQJIlx|^kUXZteah?h^1 zrL^H}yBX78!R^n+yDQic^)svK;IfV4lHG%NpMQ(EYHmGc6e5R~`ccnq>qFy?{-qwc zOnC11 z*!jOzXl$7~tPlU~-Ww1GJ%-CYBgCfGKkGJ9k+7=MC-bcMm7g*)QP45cg236M9!%gy znpL!Y%3GKK*tx+SD%}Ol^}hVA2!$NZu;=Jx?yjxGS{im0(L@zesQ^YZ_fpp{RW%72 ze)Sm$rK~h95IQOW)O?4sMX94@Go!^aad@waT-gxDKw!bTpr$41NH#P<-qMZBFh*0L z_N_rIt{PA;R=bS9ys>Okb=kntTMofxlZV^*GDo>?^I3F_TzjfWjYjCFbx0&Wu=ie) z3*B72WYI4^K@l}mpE`{n@wX&~_w$W5a{Mz}tt>`U930^?)#*R2OO3dC>E#F)!<;g5 zFfn-Ee(+Rh|Gg}g77Zgl2s8ZSj*R`8Wopim@-g}Af!vt)HZgCGfHN-SSC_;>(%2)| zyL&YbNzc__pu!Fx`t^HJSeKLD#(Jy~rakdJVOF*9S@=Ew$-)sju7*Re@s_C#OrF8|+cEj8^e{WjB7pat0#UCn?Xu<>vn2 zME9?@vw!%SJI7v`1oZt?(;Tr)Q)0(0T%_I?*l9e(;kApern?#cF6gy-w7LzLvGFc{ z_Ap+;vC2WV+Ic2FbDBV$*Qu!eZu!FgIpC(`mw-b~MW74h$O9z-&e%El%kV?S8N}Y^ z6{?W)QiA@*8H6jO7+<&MJ1@unf0*F^{JlNm0$3s!C1v}NJ=2Xl&E}XLGVza^L{XqCg27Rkf_SRup1_3&8#J1r`N^F5Jjs~$Uz%|8LkdJ zZnB89$W6C|WE9GD?5IZ!xmt&pF;|pd%yhO&o_s9W531k$o*C3D(h_3ORyolmTsIdYi{(uO@H_dkH2KD6#EDOU zhjPf!-Mv#JEG;66jegtz!DAN=@Kf}_r9p}|ihCo}By0!3;QP2mjCqV3%SpsoKcG!f z{Co3NfFnN6uVeWz(dX*+ml1z5(-tIi_N8g4vHD3q_mVz8m|2geo?mS`ZxeIBGOc-u z8{Sr;XXe2j8K^!a60DL>_FnqS7E^4^_*B2G>=ZHv@6L}q-cPztW7ijlZ~gPmm-p|= z!H@MSd5v6^xU9pMdbBN*;g&6`aK26j7mct`?LWd1`OM6BgR#J!%4M+Sb4G^C6boGW zxfA+u<|Hm!L;*){GoJs%i<+{x*<8MM&sf_`(pTxl7h*F}+$&AI+Vu7E>aZbIQJ$QJ z?x>u}(v=!rJHHbUy`;P2UOUeYOJSGoveDgo3|@5nG2V%w0bkQ=WW@K?-}yX)%f^g6 zg*>>!_TSWQov_nVnNQ~S0#rP#Th5Zv(UarV)CJDFuUrc7@o(6LUpG*u`E7D*G`X2~ zR7T>Np7H>1lAtiR@*bLWW{JIiVFHT1P37;}sB$ky0Yw@Aj}7dM*Ei4ilwEoEmkazC zO89i&gaC-VMXKqM`H{qjINPXRhWf&1+gzc&7??&Qemid~ zX(wN;gQ{hcDHRcEl9lLVF?xdZ!f`bFllmTq&Dx6|VD>fak8}J6ATTEaQ&X*qVOdKxg2aoh_2>DDT{v;A zB`%j>9azK1+82MTz4KEs2B`^g&226wM5}}FGgG&|*qTVfMl%IAtbO{h5j{LI(y%VD zSsBTQO#(?kgisCAfz@+J@)$LIdHHA*!=s?(Bxi}&b(ZBHYC?c%)1XCvWlBsr$~8Iz zFST=mJoGvX<2|`s*0iC5ujzPmyiAE4D393+`%A*grzk5 zp;A#YmoI!)#I+xX&{t;po4x;jlf3m04e?*mi+|r4V0aOz_0xL|pzpNvmo|$zjS1aZ z^tCpUUzX)*^`$}2e!_RfJ?a0nxp%pS0z|uHUs!Q!3m6F|W*H=rm$&9Ll8}N_)Dp_5 zn2IJOZJ(ZMQhjHH``bl*G+hB$vP~fqSH5v)QZ-)Kz+3NkJc=04sfngkK$UtSaEX^S zD2kWVy~vL;;%~5{i6+7*MrcG=l)%3(JC+Yuxfy@gOLK<}k`$>-MgtlAuvv-1(H?`l zm+PcVhSGF6huOdKr0gP1TX6$Y9a$`<#}tQJ4qrRUp0WO+s=T&%rIuwB-<*AqW?kO! zfUz{uTkfadF0Jv~;gjwxaG*B}{A2@Yyz4hHHq#>2BlNrB*Evc>a znVzuD)izQf`mfCyr<6aZ9;}gsxl<+XyR?TzH>KXllzNihtNqDaWpTEc2B@Z`V4QW@J%xkMAx)MyIVIa=oB)sw6F%#J6P|swpyZq@@9X6j?*jPM2jB1 ztvB82v8q8Ss-mBj*op*=X0IOL#=nhWCcu(=Q+)BN7fWhYW=;h)HEkqvkX~bLe}QdL zp-wSyv?Hfg-4wX?Ezw+%Wm;%nm8%Idx}2*r@JV;R|3h2+*ESP)b6zU;x=?M$L?bwI&F%T3DcSGhlOx@8sz48 zd&o(}s`<2zD-RKUPi`};>q;NY=phT~$4Iit(uZs;mtOOejybu+| z*f|Ys*kqLHoMd+z+1bo&?Rv1G$NN?$MU88?%L!=1BM9FoVcT_wFEB&m_Lk37_I%Es z=Qnpux3xKS_KqAOZFOeoi>||~LVcZ}=AaB4yPrs~`lKR`$^3+6neopMLg)Y(-e(uh~s9*gn+N4nlkg{W9&0yDexbi#)jUxLDTEF@>9?MY|!@X}|7CV#%)ewsA ze;%aAW$|H!ndVpz;`4Z2$M{ik<%42D@{=ZP$|5ud^<PC_F%+ip9UPM8!Y)w)!|1k37He+^cCq?zhmV%JkR!xp6!Gt=TD z!J(TeQJDR7ojNwlBgaUT>qH_Oxs{za$(v$#wRKbFBPoN*7-S&Rg5AMeGaMQyRmB9+ z@qy^%IZ6dXNe7xmww(@>;Y!Ll3~JhfN@KbPz43G`*C@sRx#A+3(wav=II& zH&it9(+ppDH?o=K`!rgu|3iOFAdsa6V0t^iOh3&(ja5&L%upX_x?h;_0`f_7-c`<} ze8hyOc+_&Xf-+D@bbjc=528mA*kTGG3zrw3lu{v)Engx^dluv`rDIHPPxYH}YY=+m z;ISy_AVq2In@oFhi;fZdlY!jUF@eFqv}9hJM%l#u2bUF+N!$x%;*)!bwHyE4>W=5r z_|ypT>gG)eWJy13=r-4Ue{lQA@9WMo++ZiBq7*DKq2@qQ_iVpc%pPo(UAF%M*|RRj ziPOc-Wz%O|3{~N}vSIHVwb7cG(8a_vyn!|6UCpUY6j+Gwz#tJj_2x?KbaaSGyd3vx zV{%@Q720kip8qCvhrSYq+010X@?3DiH9^QiVZP^{XEl_ScK>~d*x3HyZ2&jd>6xcF zCRo}f77hQqId&jIDP?%>U;_0TwC=|WcIdGom0aFvo8%$JOX^A(3AnxoH%mT}4>xDC z-aNcJ<;lK+P-`DB0m=WDU0S0Xd_orThVGhX#)lKF@6EK+UVkKt-d<(s8iF%?CZOcj#=IGlgIpBow+{>X+dkWHw>Z z^Rg=r+BryL3AyY?VC+aom){M)F_lhXea}RA9V^GeUzuk({OxN+3~dd|_0{j;k`owW zZ)*i8co`UF23zc~5jvH{tc%}!R@Eoo*atsC05H8CIb&S5Ng^;->rT($f3(Nn&Annc zK%F&|q4wm#8a@krkvwrXHm-MRK0Kk;M`hrlJ4H6$hq%~e>z{La}vxdZo!Z5fQ ziY7sV$X`4M@a?sdzb%^t?J=WOpX`hW3d!HLYFvcq6JrEgy%P0+104K#Vj z@UGRfqwi%JH00@2_TNLWheF8EhSE#d_$laL;jq1R;n2@^`)?no{aD#1V}z3Na+J$ zeisCq<3~(t7|_+rz2SW~RV$zYB%q*W*i>;7`~$VL{Yhi8Usc>(d;DSE^cW$0*z*R@ts@ z#rq(9i=F=dRIAJqyWC1U0QyH(2&MZx@OAFk6*G(Ghp?Z)5@aTWkd&M#d%myEH7(dW zc{W}m$_CDFa?3z=3;Gb6J%$^$lqw49JPPRTnuGr!BgQN7_nn}*uA)N*%8?{pf0NFB zH{cD9vra2$ogq}psx(XEmaU9#V|)P1Ly}0n)v$bo>`B;n{|lW&ku?KgxYhDJYW`c- z&BG4u)C&{2iVp5d1BrJ<<>E(*n#9xN%TX#}?8cRToeOTYKuzFUATj&1WsjxS{H|kf z`PHO&(dq|01^o{@ss`e_i!B9;qo3msxQT|$k2n+3=K^VS`j<^PFb+&#KTpSX!V;5D zVY2&Tl#RUwoljLp#^pVB2Ru$|joduQ4>$H{`5jD4%!b8x6%fgykXMWk*T6i@ABEoZl-(n)*et4lFXz`woc4?a?gZ*a-o21 zrCN_F7%qKUz;k0w$Ss6^Y-2Og0e8ujlXyM0c9!|p=tYI4{38q0ovi_#7z`hj_Wk+& zNa8`Khsq0QHRH3}SSj-Ekr*k#`b(h~7o@-A+INW>f!jvp^6OM7zR}2_yTPZ39Eu9* ziLBNkse!jj#Oi_$=tuck^(Qg2ro?epP6FauMgVvI#WyHDS-@p`7iD0X5$R2YJ?{#- zm0#jk7jV}6lt^SWWJ9OG12^Nb`Rn`iO49w*#M)waXR@D zB;WrznENHE?{oVBJ-p9L%suAYo z$?i@?9f6_WxUAUM0!eMBA|`HjsGD-}u*RbAVG6NG!wUglixSrn@2B_Z^aNljw(5}~!+7K%GB4iT zqIX{FUO3P}$~S;a{yW3m%6X1p3)!p?OjK(#e<0NDX^$h~&eG|~&mas0O7i#t}t@t*cQ5?_XqZ!{??Q(5usvSp4rpGHNJ{Sko%sm(sCD+)+YGqKF}_4i?^>%H3{YGQ zgW(k)E@#}p>=GZS9`;*eqrSV3Li3B{IR6(%`LvTSgUP)f-_!FG#fZv0abhV^T>_7N zPdQIppu4Oq5Vw6`hy^@V>=9X3%8emaGvmrn*4PE}% z4P}X9d=`yiB{1arxSf&UEx8L>q`eL_p!R!=dcrufaTJQHfr}$pL$Lsj`Z`b z7W1if+8vRe6#Z!lwkI`&0$Sp3LXpnGkX%kfk=v6S!jSj=QxWt$4Mq7|L7(_fC3@(o F@;}Ceh&uoP literal 0 HcmV?d00001 From 50eb02b9acfeb98d18518620a49bb3d49e87ca0e Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Jan 2017 23:42:27 -0500 Subject: [PATCH 012/164] Implement decode_element, which uses a Schema trait to turn EBML elements into user types. --- src/lib.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 2ed94e5..dd34b45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,6 +93,38 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { } } +pub trait Schema<'a>: Sized { + fn should_unwrap(element_id: u64) -> bool; + fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; +} + +pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result, Error> { + match decode_tag(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_id, payload_size_tag, tag_size))) => { + let should_unwrap = ::should_unwrap(element_id); + + let payload_size = match (should_unwrap, payload_size_tag) { + (true, _) => 0, + (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Value(size)) => size as usize + }; + + let element_size = tag_size + payload_size; + if element_size > bytes.len() { + // need to read more still + return Ok(None); + } + + match Schema::decode(element_id, &bytes[tag_size..element_size]) { + Ok(element) => Ok(Some((element, element_size))), + Err(error) => Err(error) + } + } + } +} + #[cfg(test)] mod tests { From 8b0021e3eb60b1502f2d7619697417dd93b7b1ef Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 17 Jan 2017 00:11:49 -0500 Subject: [PATCH 013/164] Stub WebM schema, just make all elements unknown. --- src/lib.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index dd34b45..dd5d75e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,6 +98,22 @@ pub trait Schema<'a>: Sized { fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; } +#[derive(Debug, PartialEq)] +pub enum WebmElement<'a> { + Unknown(u64, &'a[u8]) +} + +impl<'a> Schema<'a> for WebmElement<'a> { + fn should_unwrap(element_id: u64) -> bool { + false + } + + fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result, Error> { + // dummy + Ok(WebmElement::Unknown(element_id, bytes)) + } +} + pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), @@ -191,4 +207,19 @@ mod tests { assert_eq!(decode_tag(&[0x80, 0x7F, 0xFF]), Ok(Some((0, Unknown, 3)))); assert_eq!(decode_tag(&[0x85, 0x40, 52]), Ok(Some((5, Value(52), 3)))); } + + const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm"); + + #[test] + fn decode_sanity_test() { + let decoded = decode_element(TEST_FILE); + if let Ok(Some((WebmElement::Unknown(tag, slice), element_size))) = decoded { + assert_eq!(tag, 0x0A45DFA3); // EBML tag, sans the length indicator bit + assert_eq!(slice.len(), 31); // known header payload length + assert_eq!(element_size, 43); // known header total length + } else { + panic!("Did not parse expected EBML header; result: {:?}", decoded); + } + } + } From 6ab32205bba834ce0e089422fd5925c522c2b0b1 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 17 Jan 2017 03:00:09 -0500 Subject: [PATCH 014/164] Change Schema to be implemented by a dedicated type instead of its element type. --- src/lib.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dd5d75e..679d27b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,17 +93,22 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { } } -pub trait Schema<'a>: Sized { +pub trait Schema<'a> { + type Element; fn should_unwrap(element_id: u64) -> bool; - fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; + fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; } +pub struct Webm; + #[derive(Debug, PartialEq)] pub enum WebmElement<'a> { Unknown(u64, &'a[u8]) } -impl<'a> Schema<'a> for WebmElement<'a> { +impl<'a> Schema<'a> for Webm { + type Element = WebmElement<'a>; + fn should_unwrap(element_id: u64) -> bool { false } @@ -114,12 +119,12 @@ impl<'a> Schema<'a> for WebmElement<'a> { } } -pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result, Error> { +pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), Ok(Some((element_id, payload_size_tag, tag_size))) => { - let should_unwrap = ::should_unwrap(element_id); + let should_unwrap = T::should_unwrap(element_id); let payload_size = match (should_unwrap, payload_size_tag) { (true, _) => 0, @@ -133,7 +138,7 @@ pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result Ok(Some((element, element_size))), Err(error) => Err(error) } @@ -212,7 +217,7 @@ mod tests { #[test] fn decode_sanity_test() { - let decoded = decode_element(TEST_FILE); + let decoded = decode_element::(TEST_FILE); if let Ok(Some((WebmElement::Unknown(tag, slice), element_size))) = decoded { assert_eq!(tag, 0x0A45DFA3); // EBML tag, sans the length indicator bit assert_eq!(slice.len(), 31); // known header payload length From 4a42b7ae820ab320060f4d8b8d2ed148c05f611c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 24 Jan 2017 03:20:49 -0500 Subject: [PATCH 015/164] implement WebM iterator --- src/lib.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 679d27b..f3a2652 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,7 +94,8 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { } pub trait Schema<'a> { - type Element; + type Element: 'a; + fn should_unwrap(element_id: u64) -> bool; fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; } @@ -146,6 +147,35 @@ pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result { + slice: &'b[u8], + position: usize, +} + +impl<'b> WebmIterator<'b> { + pub fn new(bytes: &'b[u8]) -> Self { + WebmIterator { + slice: bytes, + position: 0 + } + } +} + +impl<'b> Iterator for WebmIterator<'b> { + type Item = WebmElement<'b>; + + fn next(&mut self) -> Option> { + match decode_element::(&self.slice[self.position..]) { + Err(_) => None, + Ok(None) => None, + Ok(Some((element, element_size))) => { + self.position += element_size; + Some(element) + } + } + } +} + #[cfg(test)] mod tests { @@ -227,4 +257,26 @@ mod tests { } } + fn assert_webm_blob(test: Option, tag: u64, payload_size: usize) { + match test { + Some(WebmElement::Unknown(element_tag, bytes)) => { + assert_eq!(element_tag, tag); + assert_eq!(bytes.len(), payload_size); + }, + None => { + panic!("Did not parse expected WebM element; result: {:?}", test); + } + } + } + + #[test] + fn decode_webm_test1() { + let mut iter = WebmIterator::new(TEST_FILE); + // EBML Header + assert_webm_blob(iter.next(), 0x0A45DFA3, 31); + + // Segment + assert_webm_blob(iter.next(), 0x08538067, 56124); + } + } From 128192c622ad224d683d0a99a4244681d0c8b417 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 24 Jan 2017 19:11:59 -0500 Subject: [PATCH 016/164] genericize EBML iterator --- src/lib.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f3a2652..1908218 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,25 +147,27 @@ pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result { +pub struct EbmlIterator<'b, T: Schema<'b>> { + schema: std::marker::PhantomData, slice: &'b[u8], position: usize, } -impl<'b> WebmIterator<'b> { +impl<'b, T: Schema<'b>> EbmlIterator<'b, T> { pub fn new(bytes: &'b[u8]) -> Self { - WebmIterator { + EbmlIterator { + schema: std::marker::PhantomData, slice: bytes, position: 0 } } } -impl<'b> Iterator for WebmIterator<'b> { - type Item = WebmElement<'b>; +impl<'b, T: Schema<'b>> Iterator for EbmlIterator<'b, T> { + type Item = T::Element; - fn next(&mut self) -> Option> { - match decode_element::(&self.slice[self.position..]) { + fn next(&mut self) -> Option { + match decode_element::(&self.slice[self.position..]) { Err(_) => None, Ok(None) => None, Ok(Some((element, element_size))) => { @@ -271,7 +273,7 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = WebmIterator::new(TEST_FILE); + let mut iter = EbmlIterator::::new(TEST_FILE); // EBML Header assert_webm_blob(iter.next(), 0x0A45DFA3, 31); From fd0fb41acf9969615ec1330151e483044bdd1606 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 26 Jan 2017 01:43:35 -0500 Subject: [PATCH 017/164] move decode_element & iterator creation onto Schema trait --- src/lib.rs | 90 +++++++++++++++++++++++++++--------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1908218..b036d8e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,8 +96,45 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { pub trait Schema<'a> { type Element: 'a; - fn should_unwrap(element_id: u64) -> bool; - fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result; + fn should_unwrap(&self, element_id: u64) -> bool; + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result; + + fn decode_element<'b: 'a>(&self, bytes: &'b[u8]) -> Result, Error> { + match decode_tag(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_id, payload_size_tag, tag_size))) => { + let should_unwrap = self.should_unwrap(element_id); + + let payload_size = match (should_unwrap, payload_size_tag) { + (true, _) => 0, + (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Value(size)) => size as usize + }; + + let element_size = tag_size + payload_size; + if element_size > bytes.len() { + // need to read more still + return Ok(None); + } + + match self.decode(element_id, &bytes[tag_size..element_size]) { + Ok(element) => Ok(Some((element, element_size))), + Err(error) => Err(error) + } + } + } + } + + fn iter_for<'b: 'a>(self, bytes: &'b[u8]) -> EbmlIterator<'a, Self> + where Self: Sized + { + EbmlIterator { + schema: self, + slice: bytes, + position: 0 + } + } } pub struct Webm; @@ -110,64 +147,27 @@ pub enum WebmElement<'a> { impl<'a> Schema<'a> for Webm { type Element = WebmElement<'a>; - fn should_unwrap(element_id: u64) -> bool { + fn should_unwrap(&self, element_id: u64) -> bool { false } - fn decode<'b: 'a>(element_id: u64, bytes: &'b[u8]) -> Result, Error> { + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { // dummy Ok(WebmElement::Unknown(element_id, bytes)) } } -pub fn decode_element<'a, 'b: 'a, T: Schema<'a>>(bytes: &'b[u8]) -> Result, Error> { - match decode_tag(bytes) { - Ok(None) => Ok(None), - Err(err) => Err(err), - Ok(Some((element_id, payload_size_tag, tag_size))) => { - let should_unwrap = T::should_unwrap(element_id); - - let payload_size = match (should_unwrap, payload_size_tag) { - (true, _) => 0, - (false, Varint::Unknown) => return Err(Error::UnknownElementLength), - (false, Varint::Value(size)) => size as usize - }; - - let element_size = tag_size + payload_size; - if element_size > bytes.len() { - // need to read more still - return Ok(None); - } - - match T::decode(element_id, &bytes[tag_size..element_size]) { - Ok(element) => Ok(Some((element, element_size))), - Err(error) => Err(error) - } - } - } -} - pub struct EbmlIterator<'b, T: Schema<'b>> { - schema: std::marker::PhantomData, + schema: T, slice: &'b[u8], position: usize, } -impl<'b, T: Schema<'b>> EbmlIterator<'b, T> { - pub fn new(bytes: &'b[u8]) -> Self { - EbmlIterator { - schema: std::marker::PhantomData, - slice: bytes, - position: 0 - } - } -} - impl<'b, T: Schema<'b>> Iterator for EbmlIterator<'b, T> { type Item = T::Element; fn next(&mut self) -> Option { - match decode_element::(&self.slice[self.position..]) { + match self.schema.decode_element(&self.slice[self.position..]) { Err(_) => None, Ok(None) => None, Ok(Some((element, element_size))) => { @@ -249,7 +249,7 @@ mod tests { #[test] fn decode_sanity_test() { - let decoded = decode_element::(TEST_FILE); + let decoded = Webm.decode_element(TEST_FILE); if let Ok(Some((WebmElement::Unknown(tag, slice), element_size))) = decoded { assert_eq!(tag, 0x0A45DFA3); // EBML tag, sans the length indicator bit assert_eq!(slice.len(), 31); // known header payload length @@ -273,7 +273,7 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = EbmlIterator::::new(TEST_FILE); + let mut iter = Webm.iter_for(TEST_FILE); // EBML Header assert_webm_blob(iter.next(), 0x0A45DFA3, 31); From 3253f32d032d0cdc10f1c0254427dea86ffecaa7 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 27 Jan 2017 01:24:17 -0500 Subject: [PATCH 018/164] Split generic EBML test from WebM test --- src/lib.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b036d8e..ba49a17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -247,16 +247,31 @@ mod tests { const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm"); + struct Dummy; + + #[derive(Debug, PartialEq)] + struct GenericElement(u64, usize); + + impl<'a> Schema<'a> for Dummy { + type Element = GenericElement; + + fn should_unwrap(&self, element_id: u64) -> bool { + match element_id { + _ => false + } + } + + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { + match element_id { + _ => Ok(GenericElement(element_id, bytes.len())) + } + } + } + #[test] fn decode_sanity_test() { - let decoded = Webm.decode_element(TEST_FILE); - if let Ok(Some((WebmElement::Unknown(tag, slice), element_size))) = decoded { - assert_eq!(tag, 0x0A45DFA3); // EBML tag, sans the length indicator bit - assert_eq!(slice.len(), 31); // known header payload length - assert_eq!(element_size, 43); // known header total length - } else { - panic!("Did not parse expected EBML header; result: {:?}", decoded); - } + let decoded = Dummy.decode_element(TEST_FILE); + assert_eq!(decoded, Ok(Some((GenericElement(0x0A45DFA3, 31), 43)))); } fn assert_webm_blob(test: Option, tag: u64, payload_size: usize) { From a23a29bf9bdfe6ce0cd53a60d707faef290beb73 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 27 Jan 2017 02:09:09 -0500 Subject: [PATCH 019/164] Experiment with WebM Schema --- src/lib.rs | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ba49a17..0e0a32a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -140,20 +140,29 @@ pub trait Schema<'a> { pub struct Webm; #[derive(Debug, PartialEq)] -pub enum WebmElement<'a> { - Unknown(u64, &'a[u8]) +pub enum WebmElement { + EbmlHead, + Segment, + Unknown(u64) } impl<'a> Schema<'a> for Webm { - type Element = WebmElement<'a>; + type Element = WebmElement; fn should_unwrap(&self, element_id: u64) -> bool { - false + match element_id { + // Segment + 0x08538067 => true, + _ => false + } } - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { - // dummy - Ok(WebmElement::Unknown(element_id, bytes)) + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { + match element_id { + 0x0A45DFA3 => Ok(WebmElement::EbmlHead), + 0x08538067 => Ok(WebmElement::Segment), + _ => Ok(WebmElement::Unknown(element_id)) + } } } @@ -274,26 +283,17 @@ mod tests { assert_eq!(decoded, Ok(Some((GenericElement(0x0A45DFA3, 31), 43)))); } - fn assert_webm_blob(test: Option, tag: u64, payload_size: usize) { - match test { - Some(WebmElement::Unknown(element_tag, bytes)) => { - assert_eq!(element_tag, tag); - assert_eq!(bytes.len(), payload_size); - }, - None => { - panic!("Did not parse expected WebM element; result: {:?}", test); - } - } - } - #[test] fn decode_webm_test1() { let mut iter = Webm.iter_for(TEST_FILE); // EBML Header - assert_webm_blob(iter.next(), 0x0A45DFA3, 31); + assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); // Segment - assert_webm_blob(iter.next(), 0x08538067, 56124); + assert_eq!(iter.next(), Some(WebmElement::Segment)); + + // SeekHead + assert_eq!(iter.next(), Some(WebmElement::Unknown(0x014D9B74))); } } From bc1569a46168b74243f9ce0dbfdfe01d2bc0b5ae Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 28 Jan 2017 02:34:01 -0500 Subject: [PATCH 020/164] use IntoIterator to get Ebml iterator --- src/lib.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0e0a32a..bd5fd09 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -126,12 +126,19 @@ pub trait Schema<'a> { } } - fn iter_for<'b: 'a>(self, bytes: &'b[u8]) -> EbmlIterator<'a, Self> - where Self: Sized +} + +pub struct Ebml<'b, S: Schema<'b>>(S, &'b[u8]); + +impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S> { + type Item = S::Element; + type IntoIter = EbmlIterator<'b, S>; + + fn into_iter(self) -> EbmlIterator<'b, S> { EbmlIterator { - schema: self, - slice: bytes, + schema: self.0, + slice: self.1, position: 0 } } @@ -285,7 +292,7 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = Webm.iter_for(TEST_FILE); + let mut iter = Ebml(Webm, TEST_FILE).into_iter(); // EBML Header assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); From efe708ca6bd614c392dc7831dd9f471f9acbfc94 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 16 Apr 2017 16:24:15 -0400 Subject: [PATCH 021/164] Make Ebml struct generic over source type --- src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bd5fd09..8e46682 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,9 +128,10 @@ pub trait Schema<'a> { } -pub struct Ebml<'b, S: Schema<'b>>(S, &'b[u8]); +#[derive(Debug, PartialEq)] +pub struct Ebml<'b, S: Schema<'b>, T: 'b>(S, &'b T); -impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S> { +impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S, &'b[u8]> { type Item = S::Element; type IntoIter = EbmlIterator<'b, S>; From c567099b7330d182945ec9638348e644e48bf873 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 16 Apr 2017 16:35:48 -0400 Subject: [PATCH 022/164] Add SeekHead & Cluster to Webm schema --- src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8e46682..c29be82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -148,14 +148,16 @@ impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S, &'b[u8]> { pub struct Webm; #[derive(Debug, PartialEq)] -pub enum WebmElement { +pub enum WebmElement<'b> { EbmlHead, Segment, + SeekHead, + Cluster(&'b[u8]), Unknown(u64) } impl<'a> Schema<'a> for Webm { - type Element = WebmElement; + type Element = WebmElement<'a>; fn should_unwrap(&self, element_id: u64) -> bool { match element_id { @@ -165,10 +167,12 @@ impl<'a> Schema<'a> for Webm { } } - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { match element_id { 0x0A45DFA3 => Ok(WebmElement::EbmlHead), 0x08538067 => Ok(WebmElement::Segment), + 0x014D9B74 => Ok(WebmElement::SeekHead), + 0x0F43B675 => Ok(WebmElement::Cluster(bytes)), _ => Ok(WebmElement::Unknown(element_id)) } } @@ -293,7 +297,8 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = Ebml(Webm, TEST_FILE).into_iter(); + let source = &TEST_FILE; + let mut iter = Ebml(Webm, source).into_iter(); // EBML Header assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); @@ -301,7 +306,7 @@ mod tests { assert_eq!(iter.next(), Some(WebmElement::Segment)); // SeekHead - assert_eq!(iter.next(), Some(WebmElement::Unknown(0x014D9B74))); + assert_eq!(iter.next(), Some(WebmElement::SeekHead)); } } From 466f0c46f015049641d735477293e03283e09d1b Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 30 Apr 2017 00:19:38 -0400 Subject: [PATCH 023/164] Parse whole test file, using constants for ids. --- src/lib.rs | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c29be82..22a0460 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,6 +128,9 @@ pub trait Schema<'a> { } +pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; +pub const VOID_ID: u64 = 0x6C; + #[derive(Debug, PartialEq)] pub struct Ebml<'b, S: Schema<'b>, T: 'b>(S, &'b T); @@ -145,13 +148,23 @@ impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S, &'b[u8]> { } } +const SEGMENT_ID: u64 = 0x08538067; +const SEEK_HEAD_ID: u64 = 0x014D9B74; +const SEGMENT_INFO_ID: u64 = 0x0549A966; +const CUES_ID: u64 = 0x0C53BB6B; +const TRACKS_ID: u64 = 0x0654AE6B; +const CLUSTER_ID: u64 = 0x0F43B675; pub struct Webm; #[derive(Debug, PartialEq)] pub enum WebmElement<'b> { EbmlHead, + Void, Segment, SeekHead, + Info, + Cues, + Tracks(&'b[u8]), Cluster(&'b[u8]), Unknown(u64) } @@ -162,17 +175,21 @@ impl<'a> Schema<'a> for Webm { fn should_unwrap(&self, element_id: u64) -> bool { match element_id { // Segment - 0x08538067 => true, + SEGMENT_ID => true, _ => false } } fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { match element_id { - 0x0A45DFA3 => Ok(WebmElement::EbmlHead), - 0x08538067 => Ok(WebmElement::Segment), - 0x014D9B74 => Ok(WebmElement::SeekHead), - 0x0F43B675 => Ok(WebmElement::Cluster(bytes)), + EBML_HEAD_ID => Ok(WebmElement::EbmlHead), + VOID_ID => Ok(WebmElement::Void), + SEGMENT_ID => Ok(WebmElement::Segment), + SEEK_HEAD_ID => Ok(WebmElement::SeekHead), + SEGMENT_INFO_ID => Ok(WebmElement::Info), + CUES_ID => Ok(WebmElement::Cues), + TRACKS_ID => Ok(WebmElement::Tracks(bytes)), + CLUSTER_ID => Ok(WebmElement::Cluster(bytes)), _ => Ok(WebmElement::Unknown(element_id)) } } @@ -299,14 +316,19 @@ mod tests { fn decode_webm_test1() { let source = &TEST_FILE; let mut iter = Ebml(Webm, source).into_iter(); - // EBML Header + + // test that we match the structure of the test file assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); - - // Segment assert_eq!(iter.next(), Some(WebmElement::Segment)); - - // SeekHead assert_eq!(iter.next(), Some(WebmElement::SeekHead)); + assert_eq!(iter.next(), Some(WebmElement::Void)); + assert_eq!(iter.next(), Some(WebmElement::Info)); + assert_eq!(iter.next(), Some(WebmElement::Tracks(&TEST_FILE[358..421]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[433..13739]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[13751..34814]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[34826..56114]))); + assert_eq!(iter.next(), Some(WebmElement::Cues)); + assert_eq!(iter.next(), None); } } From 13660fe8026c93c0aef9ce564263950df4843970 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 30 Apr 2017 14:35:11 -0400 Subject: [PATCH 024/164] Streamline adapting schema to source. --- src/lib.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 22a0460..3141784 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -126,15 +126,19 @@ pub trait Schema<'a> { } } + fn parse(self, source: T) -> Ebml where Self: Sized { + Ebml(self, source) + } + } pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; #[derive(Debug, PartialEq)] -pub struct Ebml<'b, S: Schema<'b>, T: 'b>(S, &'b T); +pub struct Ebml(S, T); -impl<'b, S: Schema<'b>> IntoIterator for Ebml<'b, S, &'b[u8]> { +impl<'b, S: Schema<'b>> IntoIterator for Ebml { type Item = S::Element; type IntoIter = EbmlIterator<'b, S>; @@ -314,8 +318,7 @@ mod tests { #[test] fn decode_webm_test1() { - let source = &TEST_FILE; - let mut iter = Ebml(Webm, source).into_iter(); + let mut iter = Webm.parse(TEST_FILE).into_iter(); // test that we match the structure of the test file assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); From 124536d703e1fe76bb05e188cf8b2d7c9957f28e Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 1 Jun 2017 02:29:12 -0400 Subject: [PATCH 025/164] Create dump tool for debugging/research --- src/bin/dump.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/bin/dump.rs diff --git a/src/bin/dump.rs b/src/bin/dump.rs new file mode 100644 index 0000000..7ff1dba --- /dev/null +++ b/src/bin/dump.rs @@ -0,0 +1,29 @@ +extern crate lab_ebml; + +use std::env::args; +use std::fs::File; +use std::io::Read; +use std::path::Path; +use lab_ebml::{Schema, Webm}; +use lab_ebml::WebmElement::*; + +pub fn main() { + let mut args = args(); + let _ = args.next(); + let filename = args.next().expect("Reading filename"); + + let mut buffer = Vec::new(); + let mut file = File::open(Path::new(&filename)).expect("Opening file"); + + file.read_to_end(&mut buffer).expect("Reading file contents"); + + for element in Webm.parse(buffer.as_slice()) { + match element { + // suppress printing byte arrays + Tracks(slice) => println!("Tracks[{}]", slice.len()), + Cluster(slice) => println!("Cluster[{}]", slice.len()), + other => println!("{:?}", other) + } + } + +} From 8411363035ac4c2b400033480452e2f8f1d4ba02 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 27 Jun 2017 02:11:29 -0400 Subject: [PATCH 026/164] Break code into modules --- src/bin/dump.rs | 5 +- src/ebml.rs | 135 +++++++++++++++++++++++++++++ src/iterator.rs | 36 ++++++++ src/lib.rs | 221 +----------------------------------------------- src/webm.rs | 48 +++++++++++ 5 files changed, 226 insertions(+), 219 deletions(-) create mode 100644 src/ebml.rs create mode 100644 src/iterator.rs create mode 100644 src/webm.rs diff --git a/src/bin/dump.rs b/src/bin/dump.rs index 7ff1dba..1251c9a 100644 --- a/src/bin/dump.rs +++ b/src/bin/dump.rs @@ -4,8 +4,9 @@ use std::env::args; use std::fs::File; use std::io::Read; use std::path::Path; -use lab_ebml::{Schema, Webm}; -use lab_ebml::WebmElement::*; +use lab_ebml::Schema; +use lab_ebml::webm::Webm; +use lab_ebml::webm::WebmElement::*; pub fn main() { let mut args = args(); diff --git a/src/ebml.rs b/src/ebml.rs new file mode 100644 index 0000000..30079e5 --- /dev/null +++ b/src/ebml.rs @@ -0,0 +1,135 @@ +pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; +pub const VOID_ID: u64 = 0x6C; + +#[derive(Debug, PartialEq)] +pub enum Error { + CorruptVarint, + UnknownElementId, + UnknownElementLength, + CorruptPayload, +} + +#[derive(Debug, PartialEq)] +pub enum Varint { + /// a numeric value + Value(u64), + /// the reserved "unknown" value + Unknown +} + +/// Try to parse an EBML varint at the start of the given slice. +/// Returns an Err() if the format is corrupt. +/// Returns Ok(None) if more bytes are needed to get a result. +/// Returns Ok(Some((varint, size))) to return a varint value and +/// the size of the parsed varint. +pub fn decode_varint(bytes: &[u8]) -> Result, Error> { + let mut value: u64 = 0; + let mut value_length = 1; + let mut mask: u8 = 0x80; + let mut unknown_marker: u64 = !0; + + if bytes.len() == 0 { + return Ok(None) + } + + // get length marker bit from first byte & parse first byte + while mask > 0 { + if (mask & bytes[0]) != 0 { + value = (bytes[0] & !mask) as u64; + unknown_marker = (mask - 1) as u64; + break + } + value_length += 1; + mask = mask >> 1; + } + + if mask == 0 { + return Err(Error::CorruptVarint) + } + + // check we have enough data to parse + if value_length > bytes.len() { + return Ok(None) + } + + // decode remaining bytes + for i in 1..value_length { + value = (value << 8) + (bytes[i] as u64); + unknown_marker = (unknown_marker << 8) + 0xFF; + } + + // determine result + if value == unknown_marker { + Ok(Some((Varint::Unknown, value_length))) + } else { + Ok(Some((Varint::Value(value), value_length))) + } +} + +/// Try to parse an EBML element header at the start of the given slice. +/// Returns an Err() if the format is corrupt. +/// Returns Ok(None) if more bytes are needed to get a result. +/// Returns Ok(Some((id, varint, size))) to return the element id, +/// the size of the payload, and the size of the parsed header. +pub fn decode_tag(bytes: &[u8]) -> Result, Error> { + // parse element ID + match decode_varint(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((Varint::Unknown, _))) => Err(Error::UnknownElementId), + Ok(Some((Varint::Value(element_id), id_size))) => { + // parse payload size + match decode_varint(&bytes[id_size..]) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_length, length_size))) => + Ok(Some(( + element_id, + element_length, + id_size + length_size + ))) + } + } + } +} + +#[derive(Debug, PartialEq)] +pub struct Ebml(pub S, pub T); + +pub trait Schema<'a> { + type Element: 'a; + + fn should_unwrap(&self, element_id: u64) -> bool; + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result; + + fn decode_element<'b: 'a>(&self, bytes: &'b[u8]) -> Result, Error> { + match decode_tag(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_id, payload_size_tag, tag_size))) => { + let should_unwrap = self.should_unwrap(element_id); + + let payload_size = match (should_unwrap, payload_size_tag) { + (true, _) => 0, + (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Value(size)) => size as usize + }; + + let element_size = tag_size + payload_size; + if element_size > bytes.len() { + // need to read more still + return Ok(None); + } + + match self.decode(element_id, &bytes[tag_size..element_size]) { + Ok(element) => Ok(Some((element, element_size))), + Err(error) => Err(error) + } + } + } + } + + fn parse(self, source: T) -> Ebml where Self: Sized { + Ebml(self, source) + } +} diff --git a/src/iterator.rs b/src/iterator.rs new file mode 100644 index 0000000..4283252 --- /dev/null +++ b/src/iterator.rs @@ -0,0 +1,36 @@ +use ebml::*; + +pub struct EbmlIterator<'b, T: Schema<'b>> { + schema: T, + slice: &'b[u8], + position: usize, +} + +impl<'b, S: Schema<'b>> IntoIterator for Ebml { + type Item = S::Element; + type IntoIter = EbmlIterator<'b, S>; + + fn into_iter(self) -> EbmlIterator<'b, S> + { + EbmlIterator { + schema: self.0, + slice: self.1, + position: 0 + } + } +} + +impl<'b, T: Schema<'b>> Iterator for EbmlIterator<'b, T> { + type Item = T::Element; + + fn next(&mut self) -> Option { + match self.schema.decode_element(&self.slice[self.position..]) { + Err(_) => None, + Ok(None) => None, + Ok(Some((element, element_size))) => { + self.position += element_size; + Some(element) + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 3141784..a6c7517 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,224 +1,11 @@ extern crate futures; -#[derive(Debug, PartialEq)] -pub enum Error { - CorruptVarint, - UnknownElementId, - UnknownElementLength, - CorruptPayload, -} +pub mod ebml; +mod iterator; +pub mod webm; -#[derive(Debug, PartialEq)] -pub enum Varint { - /// a numeric value - Value(u64), - /// the reserved "unknown" value - Unknown -} - -/// Try to parse an EBML varint at the start of the given slice. -/// Returns an Err() if the format is corrupt. -/// Returns Ok(None) if more bytes are needed to get a result. -/// Returns Ok(Some((varint, size))) to return a varint value and -/// the size of the parsed varint. -pub fn decode_varint(bytes: &[u8]) -> Result, Error> { - let mut value: u64 = 0; - let mut value_length = 1; - let mut mask: u8 = 0x80; - let mut unknown_marker: u64 = !0; - - if bytes.len() == 0 { - return Ok(None) - } - - // get length marker bit from first byte & parse first byte - while mask > 0 { - if (mask & bytes[0]) != 0 { - value = (bytes[0] & !mask) as u64; - unknown_marker = (mask - 1) as u64; - break - } - value_length += 1; - mask = mask >> 1; - } - - if mask == 0 { - return Err(Error::CorruptVarint) - } - - // check we have enough data to parse - if value_length > bytes.len() { - return Ok(None) - } - - // decode remaining bytes - for i in 1..value_length { - value = (value << 8) + (bytes[i] as u64); - unknown_marker = (unknown_marker << 8) + 0xFF; - } - - // determine result - if value == unknown_marker { - Ok(Some((Varint::Unknown, value_length))) - } else { - Ok(Some((Varint::Value(value), value_length))) - } -} - -/// Try to parse an EBML element header at the start of the given slice. -/// Returns an Err() if the format is corrupt. -/// Returns Ok(None) if more bytes are needed to get a result. -/// Returns Ok(Some((id, varint, size))) to return the element id, -/// the size of the payload, and the size of the parsed header. -pub fn decode_tag(bytes: &[u8]) -> Result, Error> { - // parse element ID - match decode_varint(bytes) { - Ok(None) => Ok(None), - Err(err) => Err(err), - Ok(Some((Varint::Unknown, _))) => Err(Error::UnknownElementId), - Ok(Some((Varint::Value(element_id), id_size))) => { - // parse payload size - match decode_varint(&bytes[id_size..]) { - Ok(None) => Ok(None), - Err(err) => Err(err), - Ok(Some((element_length, length_size))) => - Ok(Some(( - element_id, - element_length, - id_size + length_size - ))) - } - } - } -} - -pub trait Schema<'a> { - type Element: 'a; - - fn should_unwrap(&self, element_id: u64) -> bool; - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result; - - fn decode_element<'b: 'a>(&self, bytes: &'b[u8]) -> Result, Error> { - match decode_tag(bytes) { - Ok(None) => Ok(None), - Err(err) => Err(err), - Ok(Some((element_id, payload_size_tag, tag_size))) => { - let should_unwrap = self.should_unwrap(element_id); - - let payload_size = match (should_unwrap, payload_size_tag) { - (true, _) => 0, - (false, Varint::Unknown) => return Err(Error::UnknownElementLength), - (false, Varint::Value(size)) => size as usize - }; - - let element_size = tag_size + payload_size; - if element_size > bytes.len() { - // need to read more still - return Ok(None); - } - - match self.decode(element_id, &bytes[tag_size..element_size]) { - Ok(element) => Ok(Some((element, element_size))), - Err(error) => Err(error) - } - } - } - } - - fn parse(self, source: T) -> Ebml where Self: Sized { - Ebml(self, source) - } - -} - -pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; -pub const VOID_ID: u64 = 0x6C; - -#[derive(Debug, PartialEq)] -pub struct Ebml(S, T); - -impl<'b, S: Schema<'b>> IntoIterator for Ebml { - type Item = S::Element; - type IntoIter = EbmlIterator<'b, S>; - - fn into_iter(self) -> EbmlIterator<'b, S> - { - EbmlIterator { - schema: self.0, - slice: self.1, - position: 0 - } - } -} - -const SEGMENT_ID: u64 = 0x08538067; -const SEEK_HEAD_ID: u64 = 0x014D9B74; -const SEGMENT_INFO_ID: u64 = 0x0549A966; -const CUES_ID: u64 = 0x0C53BB6B; -const TRACKS_ID: u64 = 0x0654AE6B; -const CLUSTER_ID: u64 = 0x0F43B675; -pub struct Webm; - -#[derive(Debug, PartialEq)] -pub enum WebmElement<'b> { - EbmlHead, - Void, - Segment, - SeekHead, - Info, - Cues, - Tracks(&'b[u8]), - Cluster(&'b[u8]), - Unknown(u64) -} - -impl<'a> Schema<'a> for Webm { - type Element = WebmElement<'a>; - - fn should_unwrap(&self, element_id: u64) -> bool { - match element_id { - // Segment - SEGMENT_ID => true, - _ => false - } - } - - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { - match element_id { - EBML_HEAD_ID => Ok(WebmElement::EbmlHead), - VOID_ID => Ok(WebmElement::Void), - SEGMENT_ID => Ok(WebmElement::Segment), - SEEK_HEAD_ID => Ok(WebmElement::SeekHead), - SEGMENT_INFO_ID => Ok(WebmElement::Info), - CUES_ID => Ok(WebmElement::Cues), - TRACKS_ID => Ok(WebmElement::Tracks(bytes)), - CLUSTER_ID => Ok(WebmElement::Cluster(bytes)), - _ => Ok(WebmElement::Unknown(element_id)) - } - } -} - -pub struct EbmlIterator<'b, T: Schema<'b>> { - schema: T, - slice: &'b[u8], - position: usize, -} - -impl<'b, T: Schema<'b>> Iterator for EbmlIterator<'b, T> { - type Item = T::Element; - - fn next(&mut self) -> Option { - match self.schema.decode_element(&self.slice[self.position..]) { - Err(_) => None, - Ok(None) => None, - Ok(Some((element, element_size))) => { - self.position += element_size; - Some(element) - } - } - } -} +pub use ebml::{Error, Schema}; #[cfg(test)] mod tests { diff --git a/src/webm.rs b/src/webm.rs new file mode 100644 index 0000000..8e3fbef --- /dev/null +++ b/src/webm.rs @@ -0,0 +1,48 @@ +use ebml::*; + +const SEGMENT_ID: u64 = 0x08538067; +const SEEK_HEAD_ID: u64 = 0x014D9B74; +const SEGMENT_INFO_ID: u64 = 0x0549A966; +const CUES_ID: u64 = 0x0C53BB6B; +const TRACKS_ID: u64 = 0x0654AE6B; +const CLUSTER_ID: u64 = 0x0F43B675; +pub struct Webm; + +#[derive(Debug, PartialEq)] +pub enum WebmElement<'b> { + EbmlHead, + Void, + Segment, + SeekHead, + Info, + Cues, + Tracks(&'b[u8]), + Cluster(&'b[u8]), + Unknown(u64) +} + +impl<'a> Schema<'a> for Webm { + type Element = WebmElement<'a>; + + fn should_unwrap(&self, element_id: u64) -> bool { + match element_id { + // Segment + SEGMENT_ID => true, + _ => false + } + } + + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { + match element_id { + EBML_HEAD_ID => Ok(WebmElement::EbmlHead), + VOID_ID => Ok(WebmElement::Void), + SEGMENT_ID => Ok(WebmElement::Segment), + SEEK_HEAD_ID => Ok(WebmElement::SeekHead), + SEGMENT_INFO_ID => Ok(WebmElement::Info), + CUES_ID => Ok(WebmElement::Cues), + TRACKS_ID => Ok(WebmElement::Tracks(bytes)), + CLUSTER_ID => Ok(WebmElement::Cluster(bytes)), + _ => Ok(WebmElement::Unknown(element_id)) + } + } +} From f8695cfc97f94e41be07cb35c6b1a6550d9404de Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 31 May 2017 02:45:16 -0400 Subject: [PATCH 027/164] wip junk notes --- notes.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 notes.md diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..9daaa4b --- /dev/null +++ b/notes.md @@ -0,0 +1,3 @@ +* element, as utf8, as binary, as int, as container of subelements (body left "open" then) +* support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) +* rustfmt modules From ba9d00d0b477ed8aa77c67a4e6d884703c92179d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 28 Jun 2017 01:54:30 -0400 Subject: [PATCH 028/164] Move tests under appropriate modules --- src/ebml.rs | 84 ++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 103 +--------------------------------------------------- src/webm.rs | 24 ++++++++++++ 3 files changed, 110 insertions(+), 101 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 30079e5..fe6d56f 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -133,3 +133,87 @@ pub trait Schema<'a> { Ebml(self, source) } } + +#[cfg(test)] +mod tests { + use ebml::*; + use ebml::Error::{CorruptVarint, UnknownElementId}; + use ebml::Varint::{Unknown, Value}; + use tests::TEST_FILE; + + #[test] + fn fail_corrupted_varints() { + assert_eq!(decode_varint(&[0]), Err(CorruptVarint)); + assert_eq!(decode_varint(&[0, 0, 0]), Err(CorruptVarint)); + } + + #[test] + fn incomplete_varints() { + assert_eq!(decode_varint(&[]), Ok(None)); + assert_eq!(decode_varint(&[0x40]), Ok(None)); + assert_eq!(decode_varint(&[0x01, 0, 0]), Ok(None)); + } + + #[test] + fn parse_varints() { + assert_eq!(decode_varint(&[0xFF]), Ok(Some((Unknown, 1)))); + assert_eq!(decode_varint(&[0x7F, 0xFF]), Ok(Some((Unknown, 2)))); + assert_eq!(decode_varint(&[0x80]), Ok(Some((Value(0), 1)))); + assert_eq!(decode_varint(&[0x81]), Ok(Some((Value(1), 1)))); + assert_eq!(decode_varint(&[0x40, 52]), Ok(Some((Value(52), 2)))); + + // test extra data in buffer + assert_eq!(decode_varint(&[0x83, 0x11]), Ok(Some((Value(3), 1)))); + } + + #[test] + fn fail_corrupted_tags() { + assert_eq!(decode_tag(&[0]), Err(CorruptVarint)); + assert_eq!(decode_tag(&[0x80, 0]), Err(CorruptVarint)); + assert_eq!(decode_tag(&[0xFF, 0x80]), Err(UnknownElementId)); + assert_eq!(decode_tag(&[0x7F, 0xFF, 0x40, 0]), Err(UnknownElementId)); + } + + #[test] + fn incomplete_tags() { + assert_eq!(decode_tag(&[]), Ok(None)); + assert_eq!(decode_tag(&[0x80]), Ok(None)); + assert_eq!(decode_tag(&[0x40, 0, 0x40]), Ok(None)); + } + + #[test] + fn parse_tags() { + assert_eq!(decode_tag(&[0x80, 0x80]), Ok(Some((0, Value(0), 2)))); + assert_eq!(decode_tag(&[0x81, 0x85]), Ok(Some((1, Value(5), 2)))); + assert_eq!(decode_tag(&[0x80, 0xFF]), Ok(Some((0, Unknown, 2)))); + assert_eq!(decode_tag(&[0x80, 0x7F, 0xFF]), Ok(Some((0, Unknown, 3)))); + assert_eq!(decode_tag(&[0x85, 0x40, 52]), Ok(Some((5, Value(52), 3)))); + } + + struct Dummy; + + #[derive(Debug, PartialEq)] + struct GenericElement(u64, usize); + + impl<'a> Schema<'a> for Dummy { + type Element = GenericElement; + + fn should_unwrap(&self, element_id: u64) -> bool { + match element_id { + _ => false + } + } + + fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { + match element_id { + _ => Ok(GenericElement(element_id, bytes.len())) + } + } + } + + #[test] + fn decode_sanity_test() { + let decoded = Dummy.decode_element(TEST_FILE); + assert_eq!(decoded, Ok(Some((GenericElement(0x0A45DFA3, 31), 43)))); + } +} diff --git a/src/lib.rs b/src/lib.rs index a6c7517..f30276a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,11 +9,9 @@ pub use ebml::{Error, Schema}; #[cfg(test)] mod tests { - use futures::future::{ok, Future}; - use super::*; - use super::Error::{CorruptVarint, UnknownElementId}; - use super::Varint::{Unknown, Value}; + + pub const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm"); #[test] fn hello_futures() { @@ -24,101 +22,4 @@ mod tests { assert_eq!(string_result, "Hello, Futures!"); } - - #[test] - fn fail_corrupted_varints() { - assert_eq!(decode_varint(&[0]), Err(CorruptVarint)); - assert_eq!(decode_varint(&[0, 0, 0]), Err(CorruptVarint)); - } - - #[test] - fn incomplete_varints() { - assert_eq!(decode_varint(&[]), Ok(None)); - assert_eq!(decode_varint(&[0x40]), Ok(None)); - assert_eq!(decode_varint(&[0x01, 0, 0]), Ok(None)); - } - - #[test] - fn parse_varints() { - assert_eq!(decode_varint(&[0xFF]), Ok(Some((Unknown, 1)))); - assert_eq!(decode_varint(&[0x7F, 0xFF]), Ok(Some((Unknown, 2)))); - assert_eq!(decode_varint(&[0x80]), Ok(Some((Value(0), 1)))); - assert_eq!(decode_varint(&[0x81]), Ok(Some((Value(1), 1)))); - assert_eq!(decode_varint(&[0x40, 52]), Ok(Some((Value(52), 2)))); - - // test extra data in buffer - assert_eq!(decode_varint(&[0x83, 0x11]), Ok(Some((Value(3), 1)))); - } - - #[test] - fn fail_corrupted_tags() { - assert_eq!(decode_tag(&[0]), Err(CorruptVarint)); - assert_eq!(decode_tag(&[0x80, 0]), Err(CorruptVarint)); - assert_eq!(decode_tag(&[0xFF, 0x80]), Err(UnknownElementId)); - assert_eq!(decode_tag(&[0x7F, 0xFF, 0x40, 0]), Err(UnknownElementId)); - } - - #[test] - fn incomplete_tags() { - assert_eq!(decode_tag(&[]), Ok(None)); - assert_eq!(decode_tag(&[0x80]), Ok(None)); - assert_eq!(decode_tag(&[0x40, 0, 0x40]), Ok(None)); - } - - #[test] - fn parse_tags() { - assert_eq!(decode_tag(&[0x80, 0x80]), Ok(Some((0, Value(0), 2)))); - assert_eq!(decode_tag(&[0x81, 0x85]), Ok(Some((1, Value(5), 2)))); - assert_eq!(decode_tag(&[0x80, 0xFF]), Ok(Some((0, Unknown, 2)))); - assert_eq!(decode_tag(&[0x80, 0x7F, 0xFF]), Ok(Some((0, Unknown, 3)))); - assert_eq!(decode_tag(&[0x85, 0x40, 52]), Ok(Some((5, Value(52), 3)))); - } - - const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm"); - - struct Dummy; - - #[derive(Debug, PartialEq)] - struct GenericElement(u64, usize); - - impl<'a> Schema<'a> for Dummy { - type Element = GenericElement; - - fn should_unwrap(&self, element_id: u64) -> bool { - match element_id { - _ => false - } - } - - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { - match element_id { - _ => Ok(GenericElement(element_id, bytes.len())) - } - } - } - - #[test] - fn decode_sanity_test() { - let decoded = Dummy.decode_element(TEST_FILE); - assert_eq!(decoded, Ok(Some((GenericElement(0x0A45DFA3, 31), 43)))); - } - - #[test] - fn decode_webm_test1() { - let mut iter = Webm.parse(TEST_FILE).into_iter(); - - // test that we match the structure of the test file - assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); - assert_eq!(iter.next(), Some(WebmElement::Segment)); - assert_eq!(iter.next(), Some(WebmElement::SeekHead)); - assert_eq!(iter.next(), Some(WebmElement::Void)); - assert_eq!(iter.next(), Some(WebmElement::Info)); - assert_eq!(iter.next(), Some(WebmElement::Tracks(&TEST_FILE[358..421]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[433..13739]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[13751..34814]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[34826..56114]))); - assert_eq!(iter.next(), Some(WebmElement::Cues)); - assert_eq!(iter.next(), None); - } - } diff --git a/src/webm.rs b/src/webm.rs index 8e3fbef..4775e95 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -46,3 +46,27 @@ impl<'a> Schema<'a> for Webm { } } } + +#[cfg(test)] +mod tests { + use tests::TEST_FILE; + use webm::*; + + #[test] + fn decode_webm_test1() { + let mut iter = Webm.parse(TEST_FILE).into_iter(); + + // test that we match the structure of the test file + assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); + assert_eq!(iter.next(), Some(WebmElement::Segment)); + assert_eq!(iter.next(), Some(WebmElement::SeekHead)); + assert_eq!(iter.next(), Some(WebmElement::Void)); + assert_eq!(iter.next(), Some(WebmElement::Info)); + assert_eq!(iter.next(), Some(WebmElement::Tracks(&TEST_FILE[358..421]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[433..13739]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[13751..34814]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[34826..56114]))); + assert_eq!(iter.next(), Some(WebmElement::Cues)); + assert_eq!(iter.next(), None); + } +} From 8c88ff25d0ee0f40892e937b8f413f10d89ec327 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 29 Jun 2017 02:08:19 -0400 Subject: [PATCH 029/164] Define Timecode & SimpleBlock tags (tests broken) --- src/webm.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/webm.rs b/src/webm.rs index 4775e95..3908c31 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -6,6 +6,8 @@ const SEGMENT_INFO_ID: u64 = 0x0549A966; const CUES_ID: u64 = 0x0C53BB6B; const TRACKS_ID: u64 = 0x0654AE6B; const CLUSTER_ID: u64 = 0x0F43B675; +const TIMECODE_ID: u64 = 0x67; +const SIMPLE_BLOCK_ID: u64 = 0x23; pub struct Webm; #[derive(Debug, PartialEq)] @@ -17,7 +19,9 @@ pub enum WebmElement<'b> { Info, Cues, Tracks(&'b[u8]), - Cluster(&'b[u8]), + Cluster, + Timecode(u64), + SimpleBlock(&'b[u8]), Unknown(u64) } @@ -28,6 +32,7 @@ impl<'a> Schema<'a> for Webm { match element_id { // Segment SEGMENT_ID => true, + CLUSTER_ID => true, _ => false } } @@ -41,7 +46,9 @@ impl<'a> Schema<'a> for Webm { SEGMENT_INFO_ID => Ok(WebmElement::Info), CUES_ID => Ok(WebmElement::Cues), TRACKS_ID => Ok(WebmElement::Tracks(bytes)), - CLUSTER_ID => Ok(WebmElement::Cluster(bytes)), + CLUSTER_ID => Ok(WebmElement::Cluster), + TIMECODE_ID => Ok(WebmElement::Timecode(0)), + SIMPLE_BLOCK_ID => Ok(WebmElement::SimpleBlock(bytes)), _ => Ok(WebmElement::Unknown(element_id)) } } @@ -63,9 +70,9 @@ mod tests { assert_eq!(iter.next(), Some(WebmElement::Void)); assert_eq!(iter.next(), Some(WebmElement::Info)); assert_eq!(iter.next(), Some(WebmElement::Tracks(&TEST_FILE[358..421]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[433..13739]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[13751..34814]))); - assert_eq!(iter.next(), Some(WebmElement::Cluster(&TEST_FILE[34826..56114]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster)); + assert_eq!(iter.next(), Some(WebmElement::Cluster)); + assert_eq!(iter.next(), Some(WebmElement::Cluster)); assert_eq!(iter.next(), Some(WebmElement::Cues)); assert_eq!(iter.next(), None); } From 51156d9fde2a6de10d08f8a1224003b8e13e3a73 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 29 Jun 2017 02:38:08 -0400 Subject: [PATCH 030/164] implement decode_uint --- src/ebml.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/ebml.rs b/src/ebml.rs index fe6d56f..d3749a5 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -93,6 +93,18 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { } } +pub fn decode_uint(bytes: &[u8]) -> Result { + if bytes.len() < 1 || bytes.len() > 8 { + return Err(Error::CorruptPayload); + } + + let mut value: u64 = 0; + for byte in bytes { + value = (value << 8) + (*byte as u64); + } + Ok(value) +} + #[derive(Debug, PartialEq)] pub struct Ebml(pub S, pub T); @@ -190,6 +202,25 @@ mod tests { assert_eq!(decode_tag(&[0x85, 0x40, 52]), Ok(Some((5, Value(52), 3)))); } + #[test] + fn bad_uints() { + assert_eq!(decode_uint(&[]), Err(Error::CorruptPayload)); + assert_eq!(decode_uint(&[0; 9]), Err(Error::CorruptPayload)); + } + + #[test] + fn parse_uints() { + assert_eq!(decode_uint(&[0]), Ok(0)); + assert_eq!(decode_uint(&[0; 8]), Ok(0)); + assert_eq!(decode_uint(&[1]), Ok(1)); + assert_eq!(decode_uint(&[0,0,0,0,0,0,0,1]), Ok(1)); + assert_eq!(decode_uint(&[38]), Ok(38)); + assert_eq!(decode_uint(&[0,0,0,0,0,0,0,38]), Ok(38)); + assert_eq!(decode_uint(&[0x7F,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF]), Ok(9223372036854775807)); + assert_eq!(decode_uint(&[0x80,0,0,0,0,0,0,0]), Ok(9223372036854775808)); + assert_eq!(decode_uint(&[0x80,0,0,0,0,0,0,1]), Ok(9223372036854775809)); + } + struct Dummy; #[derive(Debug, PartialEq)] From a00d43e124b2d8b5c49c59e65911ef93607413c9 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 1 Jul 2017 01:30:43 -0400 Subject: [PATCH 031/164] Implement decode_simple_block --- Cargo.toml | 1 + src/bin/dump.rs | 2 +- src/lib.rs | 1 + src/webm.rs | 31 ++++++++++++++++++++++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 861f132..e961060 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" authors = ["Tangent 128 "] [dependencies] +byteorder = "1" futures = "^0.1.7" diff --git a/src/bin/dump.rs b/src/bin/dump.rs index 1251c9a..8443dcd 100644 --- a/src/bin/dump.rs +++ b/src/bin/dump.rs @@ -22,7 +22,7 @@ pub fn main() { match element { // suppress printing byte arrays Tracks(slice) => println!("Tracks[{}]", slice.len()), - Cluster(slice) => println!("Cluster[{}]", slice.len()), + SimpleBlock{timecode, ..} => println!("SimpleBlock@{}", timecode), other => println!("{:?}", other) } } diff --git a/src/lib.rs b/src/lib.rs index f30276a..9c5fe78 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ +extern crate byteorder; extern crate futures; pub mod ebml; diff --git a/src/webm.rs b/src/webm.rs index 3908c31..75c2573 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,3 +1,4 @@ +use byteorder::{BigEndian, ByteOrder}; use ebml::*; const SEGMENT_ID: u64 = 0x08538067; @@ -21,7 +22,12 @@ pub enum WebmElement<'b> { Tracks(&'b[u8]), Cluster, Timecode(u64), - SimpleBlock(&'b[u8]), + SimpleBlock { + track: u64, + timecode: i16, + flags: u8, + data: &'b[u8] + }, Unknown(u64) } @@ -47,13 +53,32 @@ impl<'a> Schema<'a> for Webm { CUES_ID => Ok(WebmElement::Cues), TRACKS_ID => Ok(WebmElement::Tracks(bytes)), CLUSTER_ID => Ok(WebmElement::Cluster), - TIMECODE_ID => Ok(WebmElement::Timecode(0)), - SIMPLE_BLOCK_ID => Ok(WebmElement::SimpleBlock(bytes)), + TIMECODE_ID => decode_uint(bytes).map(WebmElement::Timecode), + SIMPLE_BLOCK_ID => decode_simple_block(bytes), _ => Ok(WebmElement::Unknown(element_id)) } } } +fn decode_simple_block(bytes: &[u8]) -> Result { + if let Ok(Some((Varint::Value(track), track_field_len))) = decode_varint(bytes) { + let header_len = track_field_len + 2 + 1; + if bytes.len() < header_len { + return Err(Error::CorruptPayload); + } + let timecode = BigEndian::read_i16(&bytes[track_field_len..]); + let flags = bytes[track_field_len + 2]; + return Ok(WebmElement::SimpleBlock { + track: track, + timecode: timecode, + flags: flags, + data: &bytes[header_len..], + }) + } else { + return Err(Error::CorruptPayload); + } +} + #[cfg(test)] mod tests { use tests::TEST_FILE; From 6383f12ad0e5c7fe63aae30ade42872cb45ff28f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 1 Jul 2017 16:55:24 -0400 Subject: [PATCH 032/164] Make tests pass again --- src/webm.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/webm.rs b/src/webm.rs index 75c2573..09f7017 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -95,9 +95,46 @@ mod tests { assert_eq!(iter.next(), Some(WebmElement::Void)); assert_eq!(iter.next(), Some(WebmElement::Info)); assert_eq!(iter.next(), Some(WebmElement::Tracks(&TEST_FILE[358..421]))); + assert_eq!(iter.next(), Some(WebmElement::Cluster)); + assert_eq!(iter.next(), Some(WebmElement::Timecode(0))); + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + track: 1, + timecode: 0, + flags: 0b10000000, + data: &TEST_FILE[443..3683] + })); + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + track: 1, + timecode: 33, + flags: 0b00000000, + data: &TEST_FILE[3690..4735] + })); + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + track: 1, + timecode: 67, + flags: 0b00000000, + data: &TEST_FILE[4741..4801] + })); + for _ in 3..30 { + // skip remaining contents for brevity + iter.next(); + } + assert_eq!(iter.next(), Some(WebmElement::Cluster)); + assert_eq!(iter.next(), Some(WebmElement::Timecode(1000))); + for _ in 0..30 { + // skip contents for brevity + iter.next(); + } + assert_eq!(iter.next(), Some(WebmElement::Cluster)); + assert_eq!(iter.next(), Some(WebmElement::Timecode(2000))); + for _ in 0..30 { + // skip contents for brevity + iter.next(); + } + assert_eq!(iter.next(), Some(WebmElement::Cues)); assert_eq!(iter.next(), None); } From 7109e30dcb5a60bc3db3f5e834649fa52f05761c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 3 Jul 2017 01:44:11 -0400 Subject: [PATCH 033/164] Use ByteOrder function in decode_uint --- src/ebml.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index d3749a5..022463c 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,3 +1,5 @@ +use byteorder::{BigEndian, ByteOrder}; + pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; @@ -98,9 +100,7 @@ pub fn decode_uint(bytes: &[u8]) -> Result { return Err(Error::CorruptPayload); } - let mut value: u64 = 0; - for byte in bytes { - value = (value << 8) + (*byte as u64); + Ok(BigEndian::read_uint(bytes, bytes.len())) } Ok(value) } From 6c99eca673bd03493100b4b7e417aceef7629f74 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 7 Jul 2017 01:25:32 -0400 Subject: [PATCH 034/164] Initial ebml varint encoder --- src/ebml.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/src/ebml.rs b/src/ebml.rs index 022463c..6b59293 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -11,6 +11,12 @@ pub enum Error { CorruptPayload, } +#[derive(Debug, PartialEq)] +pub enum WriteError { + BufferTooSmall, + OutOfRange, +} + #[derive(Debug, PartialEq)] pub enum Varint { /// a numeric value @@ -101,8 +107,33 @@ pub fn decode_uint(bytes: &[u8]) -> Result { } Ok(BigEndian::read_uint(bytes, bytes.len())) +} + +const SMALL_FLAG: u64 = 0x80; +const SMALL_MAX: u64 = SMALL_FLAG - 2; +const TWO_FLAG: u64 = 0x60 << (8*1); +const TWO_MAX: u64 = TWO_FLAG - 2; +const EIGHT_FLAG: u64 = 0x01 << (8*7); +const EIGHT_MAX: u64 = EIGHT_FLAG - 2; + +/// Tries to write an EBML varint to the buffer +pub fn encode_varint(varint: Varint, buffer: &mut [u8]) -> Result { + let (size, number) = match varint { + Varint::Unknown => (1, 0xFF), + Varint::Value(too_big) if too_big >= EIGHT_MAX => { + return Err(WriteError::OutOfRange) + }, + Varint::Value(small @ 0 ... SMALL_MAX) => (1, small | SMALL_FLAG), + Varint::Value(two @ 0x7F ... TWO_MAX) => (2, two | TWO_FLAG), + Varint::Value(eight) => (8, eight | EIGHT_FLAG), + }; + + if buffer.len() < size { + Err(WriteError::BufferTooSmall) + } else { + BigEndian::write_uint(buffer, number, size); + Ok(size) } - Ok(value) } #[derive(Debug, PartialEq)] @@ -178,6 +209,32 @@ mod tests { assert_eq!(decode_varint(&[0x83, 0x11]), Ok(Some((Value(3), 1)))); } + #[test] + fn encode_varints() { + let mut buffer = [0; 10]; + let mut no_space = [0; 0]; + + assert_eq!(encode_varint(Varint::Unknown, &mut buffer), Ok(1)); + assert_eq!(buffer[0], 0xFF); + assert_eq!(encode_varint(Varint::Unknown, &mut no_space), Err(WriteError::BufferTooSmall)); + + assert_eq!(encode_varint(Varint::Value(0), &mut buffer), Ok(1)); + assert_eq!(buffer[0], 0x80 | 0); + assert_eq!(encode_varint(Varint::Value(0), &mut no_space), Err(WriteError::BufferTooSmall)); + + assert_eq!(encode_varint(Varint::Value(1), &mut buffer), Ok(1)); + assert_eq!(buffer[0], 0x80 | 1); + assert_eq!(encode_varint(Varint::Value(1), &mut no_space), Err(WriteError::BufferTooSmall)); + + assert_eq!(encode_varint(Varint::Value(126), &mut buffer), Ok(1)); + assert_eq!(buffer[0], 0xF0 | 126); + assert_eq!(encode_varint(Varint::Value(126), &mut no_space), Err(WriteError::BufferTooSmall)); + + assert_eq!(encode_varint(Varint::Value(127), &mut buffer), Ok(2)); + assert_eq!(&buffer[0..2], &[0x60, 127]); + assert_eq!(encode_varint(Varint::Value(127), &mut no_space), Err(WriteError::BufferTooSmall)); + } + #[test] fn fail_corrupted_tags() { assert_eq!(decode_tag(&[0]), Err(CorruptVarint)); From b1b157c6be1a3c3e1c063a53ce25e48ceaa298a9 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 7 Jul 2017 02:27:13 -0400 Subject: [PATCH 035/164] Make varint sizing logic use smallest working size; fix bad constant/test. --- src/ebml.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 6b59293..ea7bf10 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -110,9 +110,6 @@ pub fn decode_uint(bytes: &[u8]) -> Result { } const SMALL_FLAG: u64 = 0x80; -const SMALL_MAX: u64 = SMALL_FLAG - 2; -const TWO_FLAG: u64 = 0x60 << (8*1); -const TWO_MAX: u64 = TWO_FLAG - 2; const EIGHT_FLAG: u64 = 0x01 << (8*7); const EIGHT_MAX: u64 = EIGHT_FLAG - 2; @@ -123,9 +120,19 @@ pub fn encode_varint(varint: Varint, buffer: &mut [u8]) -> Result= EIGHT_MAX => { return Err(WriteError::OutOfRange) }, - Varint::Value(small @ 0 ... SMALL_MAX) => (1, small | SMALL_FLAG), - Varint::Value(two @ 0x7F ... TWO_MAX) => (2, two | TWO_FLAG), - Varint::Value(eight) => (8, eight | EIGHT_FLAG), + Varint::Value(value) => { + let mut flag = SMALL_FLAG; + let mut size = 1; + // flag bit - 1 = UNKNOWN representation once OR'd with the flag; + // if we're less than that, we can OR with the flag bit to get a valid Varint + while value >= (flag - 1) { + // right shift length bit by 1 to indicate adding a new byte; + // left shift by 8 because there's a new byte at the end + flag = flag << (8 - 1); + size += 1; + }; + (size, flag | value) + } }; if buffer.len() < size { @@ -214,6 +221,7 @@ mod tests { let mut buffer = [0; 10]; let mut no_space = [0; 0]; + // 1 byte assert_eq!(encode_varint(Varint::Unknown, &mut buffer), Ok(1)); assert_eq!(buffer[0], 0xFF); assert_eq!(encode_varint(Varint::Unknown, &mut no_space), Err(WriteError::BufferTooSmall)); @@ -230,9 +238,14 @@ mod tests { assert_eq!(buffer[0], 0xF0 | 126); assert_eq!(encode_varint(Varint::Value(126), &mut no_space), Err(WriteError::BufferTooSmall)); + // 2 bytes assert_eq!(encode_varint(Varint::Value(127), &mut buffer), Ok(2)); - assert_eq!(&buffer[0..2], &[0x60, 127]); + assert_eq!(&buffer[0..2], &[0x40, 127]); assert_eq!(encode_varint(Varint::Value(127), &mut no_space), Err(WriteError::BufferTooSmall)); + + assert_eq!(encode_varint(Varint::Value(128), &mut buffer), Ok(2)); + assert_eq!(&buffer[0..2], &[0x40, 128]); + assert_eq!(encode_varint(Varint::Value(128), &mut no_space), Err(WriteError::BufferTooSmall)); } #[test] From 8d73a2e577d9bba7572f7bffd5c9b41f0468db1d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 8 Jul 2017 01:08:21 -0400 Subject: [PATCH 036/164] Fix range of varint encoder; add more boundary tests. --- src/ebml.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/ebml.rs b/src/ebml.rs index ea7bf10..b23ee57 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -117,7 +117,7 @@ const EIGHT_MAX: u64 = EIGHT_FLAG - 2; pub fn encode_varint(varint: Varint, buffer: &mut [u8]) -> Result { let (size, number) = match varint { Varint::Unknown => (1, 0xFF), - Varint::Value(too_big) if too_big >= EIGHT_MAX => { + Varint::Value(too_big) if too_big > EIGHT_MAX => { return Err(WriteError::OutOfRange) }, Varint::Value(value) => { @@ -246,6 +246,33 @@ mod tests { assert_eq!(encode_varint(Varint::Value(128), &mut buffer), Ok(2)); assert_eq!(&buffer[0..2], &[0x40, 128]); assert_eq!(encode_varint(Varint::Value(128), &mut no_space), Err(WriteError::BufferTooSmall)); + + // 6 bytes + assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut buffer), Ok(6)); + assert_eq!(&buffer[0..6], &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + + // 7 bytes + assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer), Ok(7)); + assert_eq!(&buffer[0..7], &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + + assert_eq!(encode_varint(Varint::Value(0x01000000000000), &mut buffer), Ok(7)); + assert_eq!(&buffer[0..7], &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer), Ok(7)); + assert_eq!(&buffer[0..7], &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer[0..6]), Err(WriteError::BufferTooSmall)); + + // 8 bytes + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer), Ok(8)); + assert_eq!(&buffer[0..8], &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + + assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer), Ok(8)); + assert_eq!(&buffer[0..8], &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + + assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFF), &mut buffer), Err(WriteError::OutOfRange)); + assert_eq!(encode_varint(Varint::Value(u64::max_value()), &mut buffer), Err(WriteError::OutOfRange)); } #[test] From a1e8aef5d3dfc06cd13237ce7a9b136e45491885 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 10 Jul 2017 19:36:20 -0400 Subject: [PATCH 037/164] Swap to "bytes" crate --- Cargo.toml | 2 +- src/ebml.rs | 2 +- src/lib.rs | 2 +- src/webm.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e961060..6b6f0fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,5 @@ version = "0.1.0" authors = ["Tangent 128 "] [dependencies] -byteorder = "1" +bytes = "0.4" futures = "^0.1.7" diff --git a/src/ebml.rs b/src/ebml.rs index b23ee57..2ce8fc1 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,4 +1,4 @@ -use byteorder::{BigEndian, ByteOrder}; +use bytes::{BigEndian, ByteOrder}; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; diff --git a/src/lib.rs b/src/lib.rs index 9c5fe78..db405a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -extern crate byteorder; +extern crate bytes; extern crate futures; pub mod ebml; diff --git a/src/webm.rs b/src/webm.rs index 09f7017..6a2d645 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,4 +1,4 @@ -use byteorder::{BigEndian, ByteOrder}; +use bytes::{BigEndian, ByteOrder}; use ebml::*; const SEGMENT_ID: u64 = 0x08538067; From da81b741a30356030bc50961cd9f8ff5ea32eb19 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 10 Jul 2017 21:26:00 -0400 Subject: [PATCH 038/164] Change encode_varint to write into a Bufmut --- src/ebml.rs | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 2ce8fc1..8ffc944 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,4 +1,4 @@ -use bytes::{BigEndian, ByteOrder}; +use bytes::{BigEndian, ByteOrder, BufMut}; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; @@ -114,7 +114,7 @@ const EIGHT_FLAG: u64 = 0x01 << (8*7); const EIGHT_MAX: u64 = EIGHT_FLAG - 2; /// Tries to write an EBML varint to the buffer -pub fn encode_varint(varint: Varint, buffer: &mut [u8]) -> Result { +pub fn encode_varint(varint: Varint, buffer: &mut T) -> Result { let (size, number) = match varint { Varint::Unknown => (1, 0xFF), Varint::Value(too_big) if too_big > EIGHT_MAX => { @@ -135,10 +135,10 @@ pub fn encode_varint(varint: Varint, buffer: &mut [u8]) -> Result(number, size); Ok(size) } } @@ -186,9 +186,11 @@ pub trait Schema<'a> { #[cfg(test)] mod tests { + use bytes::{BytesMut, Buf}; use ebml::*; use ebml::Error::{CorruptVarint, UnknownElementId}; use ebml::Varint::{Unknown, Value}; + use std::io::Cursor; use tests::TEST_FILE; #[test] @@ -218,58 +220,66 @@ mod tests { #[test] fn encode_varints() { - let mut buffer = [0; 10]; - let mut no_space = [0; 0]; + let mut buffer = BytesMut::with_capacity(10); + + let mut no_space = Cursor::new([0; 0]); + assert_eq!(no_space.remaining_mut(), 0); + + let mut six_buffer = Cursor::new([0; 6]); + assert_eq!(six_buffer.remaining_mut(), 6); // 1 byte assert_eq!(encode_varint(Varint::Unknown, &mut buffer), Ok(1)); - assert_eq!(buffer[0], 0xFF); + assert_eq!(buffer.split_to(1), &[0xFF].as_ref()); assert_eq!(encode_varint(Varint::Unknown, &mut no_space), Err(WriteError::BufferTooSmall)); assert_eq!(encode_varint(Varint::Value(0), &mut buffer), Ok(1)); - assert_eq!(buffer[0], 0x80 | 0); + assert_eq!(buffer.split_to(1), &[0x80 | 0].as_ref()); assert_eq!(encode_varint(Varint::Value(0), &mut no_space), Err(WriteError::BufferTooSmall)); assert_eq!(encode_varint(Varint::Value(1), &mut buffer), Ok(1)); - assert_eq!(buffer[0], 0x80 | 1); + assert_eq!(buffer.split_to(1), &[0x80 | 1].as_ref()); assert_eq!(encode_varint(Varint::Value(1), &mut no_space), Err(WriteError::BufferTooSmall)); assert_eq!(encode_varint(Varint::Value(126), &mut buffer), Ok(1)); - assert_eq!(buffer[0], 0xF0 | 126); + assert_eq!(buffer.split_to(1), &[0xF0 | 126].as_ref()); assert_eq!(encode_varint(Varint::Value(126), &mut no_space), Err(WriteError::BufferTooSmall)); // 2 bytes assert_eq!(encode_varint(Varint::Value(127), &mut buffer), Ok(2)); - assert_eq!(&buffer[0..2], &[0x40, 127]); + assert_eq!(&buffer.split_to(2), &[0x40, 127].as_ref()); assert_eq!(encode_varint(Varint::Value(127), &mut no_space), Err(WriteError::BufferTooSmall)); assert_eq!(encode_varint(Varint::Value(128), &mut buffer), Ok(2)); - assert_eq!(&buffer[0..2], &[0x40, 128]); + assert_eq!(&buffer.split_to(2), &[0x40, 128].as_ref()); assert_eq!(encode_varint(Varint::Value(128), &mut no_space), Err(WriteError::BufferTooSmall)); // 6 bytes - assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut buffer), Ok(6)); - assert_eq!(&buffer[0..6], &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + assert_eq!(six_buffer.remaining_mut(), 6); + assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut six_buffer), Ok(6)); + assert_eq!(six_buffer.remaining_mut(), 0); + assert_eq!(&six_buffer.get_ref(), &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); + six_buffer = Cursor::new([0; 6]); // 7 bytes assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer), Ok(7)); - assert_eq!(&buffer[0..7], &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + assert_eq!(&buffer.split_to(7), &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); assert_eq!(encode_varint(Varint::Value(0x01000000000000), &mut buffer), Ok(7)); - assert_eq!(&buffer[0..7], &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + assert_eq!(&buffer.split_to(7), &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00].as_ref()); assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer), Ok(7)); - assert_eq!(&buffer[0..7], &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + assert_eq!(&buffer.split_to(7), &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut no_space), Err(WriteError::BufferTooSmall)); - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer[0..6]), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut six_buffer), Err(WriteError::BufferTooSmall)); // 8 bytes assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer), Ok(8)); - assert_eq!(&buffer[0..8], &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + assert_eq!(&buffer.split_to(8), &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer), Ok(8)); - assert_eq!(&buffer[0..8], &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE]); + assert_eq!(&buffer.split_to(8), &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFF), &mut buffer), Err(WriteError::OutOfRange)); assert_eq!(encode_varint(Varint::Value(u64::max_value()), &mut buffer), Err(WriteError::OutOfRange)); From cf4821599ec10a38d096794afa29f24532cb433c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 30 Jul 2017 01:43:09 -0400 Subject: [PATCH 039/164] Make encode_varint use a generic Writer --- src/ebml.rs | 122 +++++++++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 54 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 8ffc944..14248a9 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,4 +1,7 @@ use bytes::{BigEndian, ByteOrder, BufMut}; +use std::error::Error as ErrorTrait; +use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write}; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; @@ -13,8 +16,21 @@ pub enum Error { #[derive(Debug, PartialEq)] pub enum WriteError { - BufferTooSmall, - OutOfRange, + OutOfRange +} +impl Display for WriteError { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + &WriteError::OutOfRange => write!(f, "EBML Varint out of range") + } + } +} +impl ErrorTrait for WriteError { + fn description(&self) -> &str { + match self { + &WriteError::OutOfRange => "EBML Varint out of range" + } + } } #[derive(Debug, PartialEq)] @@ -113,12 +129,12 @@ const SMALL_FLAG: u64 = 0x80; const EIGHT_FLAG: u64 = 0x01 << (8*7); const EIGHT_MAX: u64 = EIGHT_FLAG - 2; -/// Tries to write an EBML varint to the buffer -pub fn encode_varint(varint: Varint, buffer: &mut T) -> Result { +/// Tries to write an EBML varint +pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult { let (size, number) = match varint { Varint::Unknown => (1, 0xFF), Varint::Value(too_big) if too_big > EIGHT_MAX => { - return Err(WriteError::OutOfRange) + return Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) }, Varint::Value(value) => { let mut flag = SMALL_FLAG; @@ -135,12 +151,10 @@ pub fn encode_varint(varint: Varint, buffer: &mut T) -> Result(number, size); - Ok(size) - } + let mut buffer = Cursor::new([0; 8]); + buffer.put_uint::(number, size); + + return output.write_all(&buffer.get_ref()[..size]).map(|()| size); } #[derive(Debug, PartialEq)] @@ -186,7 +200,7 @@ pub trait Schema<'a> { #[cfg(test)] mod tests { - use bytes::{BytesMut, Buf}; + use bytes::{BytesMut}; use ebml::*; use ebml::Error::{CorruptVarint, UnknownElementId}; use ebml::Varint::{Unknown, Value}; @@ -220,69 +234,69 @@ mod tests { #[test] fn encode_varints() { - let mut buffer = BytesMut::with_capacity(10); + let mut buffer = BytesMut::with_capacity(10).writer(); - let mut no_space = Cursor::new([0; 0]); - assert_eq!(no_space.remaining_mut(), 0); + let mut no_space = Cursor::new([0; 0]).writer(); + assert_eq!(no_space.get_ref().remaining_mut(), 0); - let mut six_buffer = Cursor::new([0; 6]); - assert_eq!(six_buffer.remaining_mut(), 6); + let mut six_buffer = Cursor::new([0; 6]).writer(); + assert_eq!(six_buffer.get_ref().remaining_mut(), 6); // 1 byte - assert_eq!(encode_varint(Varint::Unknown, &mut buffer), Ok(1)); - assert_eq!(buffer.split_to(1), &[0xFF].as_ref()); - assert_eq!(encode_varint(Varint::Unknown, &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Unknown, &mut buffer).unwrap(), 1); + assert_eq!(buffer.get_mut().split_to(1), &[0xFF].as_ref()); + assert_eq!(encode_varint(Varint::Unknown, &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(0), &mut buffer), Ok(1)); - assert_eq!(buffer.split_to(1), &[0x80 | 0].as_ref()); - assert_eq!(encode_varint(Varint::Value(0), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(0), &mut buffer).unwrap(), 1); + assert_eq!(buffer.get_mut().split_to(1), &[0x80 | 0].as_ref()); + assert_eq!(encode_varint(Varint::Value(0), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(1), &mut buffer), Ok(1)); - assert_eq!(buffer.split_to(1), &[0x80 | 1].as_ref()); - assert_eq!(encode_varint(Varint::Value(1), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(1), &mut buffer).unwrap(), 1); + assert_eq!(buffer.get_mut().split_to(1), &[0x80 | 1].as_ref()); + assert_eq!(encode_varint(Varint::Value(1), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(126), &mut buffer), Ok(1)); - assert_eq!(buffer.split_to(1), &[0xF0 | 126].as_ref()); - assert_eq!(encode_varint(Varint::Value(126), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(126), &mut buffer).unwrap(), 1); + assert_eq!(buffer.get_mut().split_to(1), &[0xF0 | 126].as_ref()); + assert_eq!(encode_varint(Varint::Value(126), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); // 2 bytes - assert_eq!(encode_varint(Varint::Value(127), &mut buffer), Ok(2)); - assert_eq!(&buffer.split_to(2), &[0x40, 127].as_ref()); - assert_eq!(encode_varint(Varint::Value(127), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(127), &mut buffer).unwrap(), 2); + assert_eq!(&buffer.get_mut().split_to(2), &[0x40, 127].as_ref()); + assert_eq!(encode_varint(Varint::Value(127), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(128), &mut buffer), Ok(2)); - assert_eq!(&buffer.split_to(2), &[0x40, 128].as_ref()); - assert_eq!(encode_varint(Varint::Value(128), &mut no_space), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(128), &mut buffer).unwrap(), 2); + assert_eq!(&buffer.get_mut().split_to(2), &[0x40, 128].as_ref()); + assert_eq!(encode_varint(Varint::Value(128), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); // 6 bytes - assert_eq!(six_buffer.remaining_mut(), 6); - assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut six_buffer), Ok(6)); - assert_eq!(six_buffer.remaining_mut(), 0); - assert_eq!(&six_buffer.get_ref(), &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); - six_buffer = Cursor::new([0; 6]); + assert_eq!(six_buffer.get_ref().remaining_mut(), 6); + assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut six_buffer).unwrap(), 6); + assert_eq!(six_buffer.get_ref().remaining_mut(), 0); + assert_eq!(&six_buffer.get_ref().get_ref(), &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); + six_buffer = Cursor::new([0; 6]).writer(); // 7 bytes - assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer), Ok(7)); - assert_eq!(&buffer.split_to(7), &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); + assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer).unwrap(), 7); + assert_eq!(&buffer.get_mut().split_to(7), &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); - assert_eq!(encode_varint(Varint::Value(0x01000000000000), &mut buffer), Ok(7)); - assert_eq!(&buffer.split_to(7), &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00].as_ref()); + assert_eq!(encode_varint(Varint::Value(0x01000000000000), &mut buffer).unwrap(), 7); + assert_eq!(&buffer.get_mut().split_to(7), &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00].as_ref()); - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer), Ok(7)); - assert_eq!(&buffer.split_to(7), &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer).unwrap(), 7); + assert_eq!(&buffer.get_mut().split_to(7), &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut no_space), Err(WriteError::BufferTooSmall)); - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut six_buffer), Err(WriteError::BufferTooSmall)); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut six_buffer).unwrap_err().kind(), ErrorKind::WriteZero); // 8 bytes - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer), Ok(8)); - assert_eq!(&buffer.split_to(8), &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); + assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer).unwrap(), 8); + assert_eq!(&buffer.get_mut().split_to(8), &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); - assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer), Ok(8)); - assert_eq!(&buffer.split_to(8), &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); + assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer).unwrap(), 8); + assert_eq!(&buffer.get_mut().split_to(8), &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); - assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFF), &mut buffer), Err(WriteError::OutOfRange)); - assert_eq!(encode_varint(Varint::Value(u64::max_value()), &mut buffer), Err(WriteError::OutOfRange)); + assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFF), &mut buffer).unwrap_err().kind(), ErrorKind::InvalidInput); + assert_eq!(encode_varint(Varint::Value(u64::max_value()), &mut buffer).unwrap_err().kind(), ErrorKind::InvalidInput); } #[test] From b5bbf952b0583f09374c55e1efa667bbbbc0b957 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 2 Aug 2017 01:30:14 -0400 Subject: [PATCH 040/164] Stub some EBML encoding funcs --- src/ebml.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ebml.rs b/src/ebml.rs index 14248a9..23d24d1 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -157,6 +157,19 @@ pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult(tag: u64, size: Varint, output: &mut T) -> IoResult { + let id_size = encode_varint(Varint::Value(tag), output)?; + let size_size = encode_varint(size, output)?; + Ok(id_size + size_size) +} + +/// Tries to write a simple EBML tag with a string value +pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResult { + let tag_size = encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; + let string_size = output.write_all(string.as_ref()).map(|()| string.len())?; + Ok(tag_size + string_size) +} + #[derive(Debug, PartialEq)] pub struct Ebml(pub S, pub T); From 95fcc6c86b7197ba6b882433276a02bc82fa6019 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 6 Aug 2017 01:20:51 -0400 Subject: [PATCH 041/164] Add fixed-size varint encoder function to simplify writing complex EBML element sizes --- src/ebml.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 23d24d1..793ad13 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,7 +1,7 @@ use bytes::{BigEndian, ByteOrder, BufMut}; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; -use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write}; +use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const VOID_ID: u64 = 0x6C; @@ -129,7 +129,7 @@ const SMALL_FLAG: u64 = 0x80; const EIGHT_FLAG: u64 = 0x01 << (8*7); const EIGHT_MAX: u64 = EIGHT_FLAG - 2; -/// Tries to write an EBML varint +/// Tries to write an EBML varint using minimal space pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult { let (size, number) = match varint { Varint::Unknown => (1, 0xFF), @@ -157,6 +157,40 @@ pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult(varint: Varint, output: &mut T) -> IoResult { + let number = match varint { + Varint::Unknown => FOUR_FLAG | (FOUR_FLAG - 1), + Varint::Value(too_big) if too_big > FOUR_MAX => { + return Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) + }, + Varint::Value(value) => FOUR_FLAG | value + }; + + let mut buffer = Cursor::new([0; 4]); + buffer.put_u32::(number as u32); + + return output.write_all(&buffer.get_ref()[..]).map(|()| 4); +} + +pub fn encode_element IoResult, X>(tag: u64, output: &mut T, content: F) -> IoResult<()> { + encode_varint(Varint::Value(tag), output)?; + encode_varint_4(Varint::Unknown, output)?; + + let start = output.seek(SeekFrom::Current(0))?; + content(output)?; + let end = output.seek(SeekFrom::Current(0))?; + + output.seek(SeekFrom::Start(start - 4))?; + encode_varint_4(Varint::Value(end - start), output)?; + output.seek(SeekFrom::Start(end))?; + + Ok(()) +} + pub fn encode_tag_header(tag: u64, size: Varint, output: &mut T) -> IoResult { let id_size = encode_varint(Varint::Value(tag), output)?; let size_size = encode_varint(size, output)?; From 45fe373ee62168cdc9f99fb1583acd64568db934 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 6 Aug 2017 01:21:53 -0400 Subject: [PATCH 042/164] Implement writing a barebones EBML header from scratch --- src/bin/stub.rs | 12 ++++++++++++ src/ebml.rs | 1 + src/webm.rs | 13 +++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 src/bin/stub.rs diff --git a/src/bin/stub.rs b/src/bin/stub.rs new file mode 100644 index 0000000..509ebe8 --- /dev/null +++ b/src/bin/stub.rs @@ -0,0 +1,12 @@ +extern crate lab_ebml; + +use std::io::{Cursor, stdout, Write}; +use lab_ebml::webm::*; + +pub fn main() { + let mut cursor = Cursor::new(Vec::new()); + + encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); + + stdout().write_all(&cursor.get_ref()).unwrap(); +} diff --git a/src/ebml.rs b/src/ebml.rs index 793ad13..e29e5ec 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -4,6 +4,7 @@ use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; +pub const DOC_TYPE_ID: u64 = 0x0282; pub const VOID_ID: u64 = 0x6C; #[derive(Debug, PartialEq)] diff --git a/src/webm.rs b/src/webm.rs index 6a2d645..22b0885 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,3 +1,4 @@ +use std::io::{Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; use bytes::{BigEndian, ByteOrder}; use ebml::*; @@ -79,6 +80,18 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } } +pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { + let mut start = 0; + match element { + WebmElement::EbmlHead => { + encode_element(EBML_HEAD_ID, output, |output| { + encode_string(DOC_TYPE_ID, "webm", output) + }) + } + _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) + } +} + #[cfg(test)] mod tests { use tests::TEST_FILE; From ff4c2d796e4771822095257ff5fe2d6d90528182 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 7 Aug 2017 00:55:57 -0400 Subject: [PATCH 043/164] Remove IoResult return, value not needed and confuses ? operator --- src/ebml.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index e29e5ec..25c6705 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -192,17 +192,17 @@ pub fn encode_element IoResult, X>(tag: u64 Ok(()) } -pub fn encode_tag_header(tag: u64, size: Varint, output: &mut T) -> IoResult { - let id_size = encode_varint(Varint::Value(tag), output)?; - let size_size = encode_varint(size, output)?; - Ok(id_size + size_size) +pub fn encode_tag_header(tag: u64, size: Varint, output: &mut T) -> IoResult<()> { + encode_varint(Varint::Value(tag), output)?; + encode_varint(size, output)?; + Ok(()) } /// Tries to write a simple EBML tag with a string value -pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResult { - let tag_size = encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; - let string_size = output.write_all(string.as_ref()).map(|()| string.len())?; - Ok(tag_size + string_size) +pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResult<()> { + encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; + output.write_all(string.as_ref()).map(|()| string.len())?; + Ok(()) } #[derive(Debug, PartialEq)] From 2ecedf795d077492353d5a2c738fdaa192fdeef5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 7 Aug 2017 00:56:18 -0400 Subject: [PATCH 044/164] Cleanup dead line --- src/webm.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webm.rs b/src/webm.rs index 22b0885..5330145 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -81,7 +81,6 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { - let mut start = 0; match element { WebmElement::EbmlHead => { encode_element(EBML_HEAD_ID, output, |output| { From c969627510c9cb2718e5a906a3908e90b8e14417 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 7 Aug 2017 01:03:20 -0400 Subject: [PATCH 045/164] Extend stub WebM structure --- src/bin/stub.rs | 3 +++ src/webm.rs | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/bin/stub.rs b/src/bin/stub.rs index 509ebe8..3b26f35 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -7,6 +7,9 @@ pub fn main() { let mut cursor = Cursor::new(Vec::new()); encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); + + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); stdout().write_all(&cursor.get_ref()).unwrap(); } diff --git a/src/webm.rs b/src/webm.rs index 5330145..b2f134c 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -82,11 +82,13 @@ fn decode_simple_block(bytes: &[u8]) -> Result { pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { match element { - WebmElement::EbmlHead => { - encode_element(EBML_HEAD_ID, output, |output| { - encode_string(DOC_TYPE_ID, "webm", output) - }) - } + WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { + encode_string(DOC_TYPE_ID, "webm", output) + }), + WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), + WebmElement::SeekHead => Ok(()), + WebmElement::Cues => Ok(()), + WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From 120fe26b6b8187b3da3341fc213eed80a05c57f2 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 7 Aug 2017 01:11:43 -0400 Subject: [PATCH 046/164] Cleanup results more --- src/ebml.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 25c6705..4a39625 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -131,7 +131,7 @@ const EIGHT_FLAG: u64 = 0x01 << (8*7); const EIGHT_MAX: u64 = EIGHT_FLAG - 2; /// Tries to write an EBML varint using minimal space -pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult { +pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult<()> { let (size, number) = match varint { Varint::Unknown => (1, 0xFF), Varint::Value(too_big) if too_big > EIGHT_MAX => { @@ -155,14 +155,14 @@ pub fn encode_varint(varint: Varint, output: &mut T) -> IoResult(number, size); - return output.write_all(&buffer.get_ref()[..size]).map(|()| size); + return output.write_all(&buffer.get_ref()[..size]); } const FOUR_FLAG: u64 = 0x10 << (8*3); const FOUR_MAX: u64 = FOUR_FLAG - 2; // tries to write a varint with a fixed 4-byte representation -pub fn encode_varint_4(varint: Varint, output: &mut T) -> IoResult { +pub fn encode_varint_4(varint: Varint, output: &mut T) -> IoResult<()> { let number = match varint { Varint::Unknown => FOUR_FLAG | (FOUR_FLAG - 1), Varint::Value(too_big) if too_big > FOUR_MAX => { @@ -174,7 +174,7 @@ pub fn encode_varint_4(varint: Varint, output: &mut T) -> IoResult(number as u32); - return output.write_all(&buffer.get_ref()[..]).map(|()| 4); + output.write_all(&buffer.get_ref()[..]) } pub fn encode_element IoResult, X>(tag: u64, output: &mut T, content: F) -> IoResult<()> { @@ -194,15 +194,14 @@ pub fn encode_element IoResult, X>(tag: u64 pub fn encode_tag_header(tag: u64, size: Varint, output: &mut T) -> IoResult<()> { encode_varint(Varint::Value(tag), output)?; - encode_varint(size, output)?; - Ok(()) + encode_varint(size, output) } /// Tries to write a simple EBML tag with a string value pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResult<()> { encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; - output.write_all(string.as_ref()).map(|()| string.len())?; - Ok(()) + output.write_all(string.as_ref()) +} } #[derive(Debug, PartialEq)] From b76f24402298c2cd44562171c181be1ec3cf1d29 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 7 Aug 2017 01:12:18 -0400 Subject: [PATCH 047/164] Encode Timecodes into WebM stub --- src/bin/stub.rs | 4 ++++ src/ebml.rs | 9 +++++++++ src/webm.rs | 1 + 3 files changed, 14 insertions(+) diff --git a/src/bin/stub.rs b/src/bin/stub.rs index 3b26f35..617437c 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -10,6 +10,10 @@ pub fn main() { encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); + + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); stdout().write_all(&cursor.get_ref()).unwrap(); } diff --git a/src/ebml.rs b/src/ebml.rs index 4a39625..e4b07b3 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -202,6 +202,15 @@ pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResu encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; output.write_all(string.as_ref()) } + +/// Tries to write a simple EBML tag with an integer value +pub fn encode_integer(tag: u64, value: u64, output: &mut T) -> IoResult<()> { + encode_tag_header(tag, Varint::Value(8), output)?; + + let mut buffer = Cursor::new([0; 8]); + buffer.put_u64::(value); + + output.write_all(&buffer.get_ref()[..]) } #[derive(Debug, PartialEq)] diff --git a/src/webm.rs b/src/webm.rs index b2f134c..9868e49 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -89,6 +89,7 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T WebmElement::SeekHead => Ok(()), WebmElement::Cues => Ok(()), WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), + WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From 509dcd3827c3051332cc6ef8fd2aec21913635a9 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 9 Aug 2017 01:49:44 -0400 Subject: [PATCH 048/164] Generalize string writing to binary writing --- src/ebml.rs | 8 ++++---- src/webm.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index e4b07b3..691c5f6 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -197,10 +197,10 @@ pub fn encode_tag_header(tag: u64, size: Varint, output: &mut T) -> Io encode_varint(size, output) } -/// Tries to write a simple EBML tag with a string value -pub fn encode_string(tag: u64, string: &str, output: &mut T) -> IoResult<()> { - encode_tag_header(tag, Varint::Value(string.len() as u64), output)?; - output.write_all(string.as_ref()) +/// Tries to write a simple EBML tag with a string or binary value +pub fn encode_bytes(tag: u64, bytes: &[u8], output: &mut T) -> IoResult<()> { + encode_tag_header(tag, Varint::Value(bytes.len() as u64), output)?; + output.write_all(bytes) } /// Tries to write a simple EBML tag with an integer value diff --git a/src/webm.rs b/src/webm.rs index 9868e49..03b2c96 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -83,7 +83,7 @@ fn decode_simple_block(bytes: &[u8]) -> Result { pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { match element { WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { - encode_string(DOC_TYPE_ID, "webm", output) + encode_bytes(DOC_TYPE_ID, "webm".as_bytes(), output) }), WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), WebmElement::SeekHead => Ok(()), From 01e81c6a9fa3cd01b37ffaebdbc03c634ddebef9 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 10 Aug 2017 00:14:42 -0400 Subject: [PATCH 049/164] Fix tests --- src/ebml.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 691c5f6..8a5ab78 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -299,56 +299,56 @@ mod tests { assert_eq!(six_buffer.get_ref().remaining_mut(), 6); // 1 byte - assert_eq!(encode_varint(Varint::Unknown, &mut buffer).unwrap(), 1); + encode_varint(Varint::Unknown, &mut buffer).unwrap(); assert_eq!(buffer.get_mut().split_to(1), &[0xFF].as_ref()); assert_eq!(encode_varint(Varint::Unknown, &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(0), &mut buffer).unwrap(), 1); + encode_varint(Varint::Value(0), &mut buffer).unwrap(); assert_eq!(buffer.get_mut().split_to(1), &[0x80 | 0].as_ref()); assert_eq!(encode_varint(Varint::Value(0), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(1), &mut buffer).unwrap(), 1); + encode_varint(Varint::Value(1), &mut buffer).unwrap(); assert_eq!(buffer.get_mut().split_to(1), &[0x80 | 1].as_ref()); assert_eq!(encode_varint(Varint::Value(1), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(126), &mut buffer).unwrap(), 1); + encode_varint(Varint::Value(126), &mut buffer).unwrap(); assert_eq!(buffer.get_mut().split_to(1), &[0xF0 | 126].as_ref()); assert_eq!(encode_varint(Varint::Value(126), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); // 2 bytes - assert_eq!(encode_varint(Varint::Value(127), &mut buffer).unwrap(), 2); + encode_varint(Varint::Value(127), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(2), &[0x40, 127].as_ref()); assert_eq!(encode_varint(Varint::Value(127), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); - assert_eq!(encode_varint(Varint::Value(128), &mut buffer).unwrap(), 2); + encode_varint(Varint::Value(128), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(2), &[0x40, 128].as_ref()); assert_eq!(encode_varint(Varint::Value(128), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); // 6 bytes assert_eq!(six_buffer.get_ref().remaining_mut(), 6); - assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut six_buffer).unwrap(), 6); + encode_varint(Varint::Value(0x03FFFFFFFFFE), &mut six_buffer).unwrap(); assert_eq!(six_buffer.get_ref().remaining_mut(), 0); assert_eq!(&six_buffer.get_ref().get_ref(), &[0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); six_buffer = Cursor::new([0; 6]).writer(); // 7 bytes - assert_eq!(encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer).unwrap(), 7); + encode_varint(Varint::Value(0x03FFFFFFFFFF), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(7), &[0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); - assert_eq!(encode_varint(Varint::Value(0x01000000000000), &mut buffer).unwrap(), 7); + encode_varint(Varint::Value(0x01000000000000), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(7), &[0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00].as_ref()); - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer).unwrap(), 7); + encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(7), &[0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut no_space).unwrap_err().kind(), ErrorKind::WriteZero); assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFE), &mut six_buffer).unwrap_err().kind(), ErrorKind::WriteZero); // 8 bytes - assert_eq!(encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer).unwrap(), 8); + encode_varint(Varint::Value(0x01FFFFFFFFFFFF), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(8), &[0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF].as_ref()); - assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer).unwrap(), 8); + encode_varint(Varint::Value(0xFFFFFFFFFFFFFE), &mut buffer).unwrap(); assert_eq!(&buffer.get_mut().split_to(8), &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE].as_ref()); assert_eq!(encode_varint(Varint::Value(0xFFFFFFFFFFFFFF), &mut buffer).unwrap_err().kind(), ErrorKind::InvalidInput); From dd376e146eb2ccc8d6070bbe38085323076b87cd Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 12 Aug 2017 16:48:24 -0400 Subject: [PATCH 050/164] Implement encoding SimpleBlocks --- src/bin/stub.rs | 7 +++++++ src/webm.rs | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/bin/stub.rs b/src/bin/stub.rs index 617437c..c545f56 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -12,6 +12,13 @@ pub fn main() { encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); + encode_webm_element(WebmElement::SimpleBlock { + track: 3, + flags: 0x0, + timecode: 123, + data: "Hello, World".as_bytes() + }, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); diff --git a/src/webm.rs b/src/webm.rs index 03b2c96..7f2f7a4 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,5 +1,5 @@ -use std::io::{Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; -use bytes::{BigEndian, ByteOrder}; +use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; +use bytes::{BigEndian, BufMut, ByteOrder}; use ebml::*; const SEGMENT_ID: u64 = 0x08538067; @@ -80,6 +80,33 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } } +pub fn encode_simple_block(element: WebmElement, output: &mut T) -> IoResult<()> { + if let WebmElement::SimpleBlock{ + track, + timecode, + flags, + data + } = element { + // limiting number of tracks for now + if track > 31 { + return Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)); + } + let header_len = 1 + 2 + 1; + encode_tag_header(SIMPLE_BLOCK_ID, Varint::Value((header_len + data.len()) as u64), output)?; + + encode_varint(Varint::Value(track), output)?; + + let mut buffer = Cursor::new([0; 3]); + buffer.put_i16::(timecode); + buffer.put_u8(flags); + + output.write_all(&buffer.get_ref()[..])?; + output.write_all(data) + } else { + Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) + } +} + pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { match element { WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { @@ -90,6 +117,7 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T WebmElement::Cues => Ok(()), WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), + WebmElement::SimpleBlock {..} => encode_simple_block(element, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From fd8b820eef5b63dfa8ba11732843922c48ead5b8 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 14 Aug 2017 00:58:06 -0400 Subject: [PATCH 051/164] Stub-encode Tracks --- src/bin/stub.rs | 2 ++ src/webm.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/bin/stub.rs b/src/bin/stub.rs index c545f56..a77a1a0 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -9,6 +9,8 @@ pub fn main() { encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Tracks(&[]), &mut cursor).unwrap(); + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); diff --git a/src/webm.rs b/src/webm.rs index 7f2f7a4..52eb318 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -115,6 +115,7 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), WebmElement::SeekHead => Ok(()), WebmElement::Cues => Ok(()), + WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), WebmElement::SimpleBlock {..} => encode_simple_block(element, output), From 93883e268302e04ff9c80a11aa9ad5ca0c3af7fb Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 4 Sep 2017 01:41:48 -0400 Subject: [PATCH 052/164] refactor SimpleBlock struct to be independent of Element enum --- src/bin/dump.rs | 3 ++- src/bin/stub.rs | 4 +-- src/webm.rs | 69 +++++++++++++++++++++++++------------------------ 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/bin/dump.rs b/src/bin/dump.rs index 8443dcd..8e5d333 100644 --- a/src/bin/dump.rs +++ b/src/bin/dump.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::io::Read; use std::path::Path; use lab_ebml::Schema; +use lab_ebml::webm::SimpleBlock; use lab_ebml::webm::Webm; use lab_ebml::webm::WebmElement::*; @@ -22,7 +23,7 @@ pub fn main() { match element { // suppress printing byte arrays Tracks(slice) => println!("Tracks[{}]", slice.len()), - SimpleBlock{timecode, ..} => println!("SimpleBlock@{}", timecode), + SimpleBlock(SimpleBlock {timecode, ..}) => println!("SimpleBlock@{}", timecode), other => println!("{:?}", other) } } diff --git a/src/bin/stub.rs b/src/bin/stub.rs index a77a1a0..b3e0951 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -14,12 +14,12 @@ pub fn main() { encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); - encode_webm_element(WebmElement::SimpleBlock { + encode_webm_element(WebmElement::SimpleBlock(SimpleBlock { track: 3, flags: 0x0, timecode: 123, data: "Hello, World".as_bytes() - }, &mut cursor).unwrap(); + }), &mut cursor).unwrap(); encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); diff --git a/src/webm.rs b/src/webm.rs index 52eb318..ccc313b 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -12,6 +12,14 @@ const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; pub struct Webm; +#[derive(Debug, PartialEq)] +pub struct SimpleBlock<'b> { + pub track: u64, + pub timecode: i16, + pub flags: u8, + pub data: &'b[u8] +} + #[derive(Debug, PartialEq)] pub enum WebmElement<'b> { EbmlHead, @@ -23,12 +31,7 @@ pub enum WebmElement<'b> { Tracks(&'b[u8]), Cluster, Timecode(u64), - SimpleBlock { - track: u64, - timecode: i16, - flags: u8, - data: &'b[u8] - }, + SimpleBlock(SimpleBlock<'b>), Unknown(u64) } @@ -69,42 +72,40 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } let timecode = BigEndian::read_i16(&bytes[track_field_len..]); let flags = bytes[track_field_len + 2]; - return Ok(WebmElement::SimpleBlock { + return Ok(WebmElement::SimpleBlock(SimpleBlock { track: track, timecode: timecode, flags: flags, data: &bytes[header_len..], - }) + })) } else { return Err(Error::CorruptPayload); } } -pub fn encode_simple_block(element: WebmElement, output: &mut T) -> IoResult<()> { - if let WebmElement::SimpleBlock{ +pub fn encode_simple_block(block: SimpleBlock, output: &mut T) -> IoResult<()> { + let SimpleBlock { track, timecode, flags, data - } = element { - // limiting number of tracks for now - if track > 31 { - return Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)); - } - let header_len = 1 + 2 + 1; - encode_tag_header(SIMPLE_BLOCK_ID, Varint::Value((header_len + data.len()) as u64), output)?; + } = block; - encode_varint(Varint::Value(track), output)?; - - let mut buffer = Cursor::new([0; 3]); - buffer.put_i16::(timecode); - buffer.put_u8(flags); - - output.write_all(&buffer.get_ref()[..])?; - output.write_all(data) - } else { - Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) + // limiting number of tracks for now + if track > 31 { + return Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)); } + let header_len = 1 + 2 + 1; + encode_tag_header(SIMPLE_BLOCK_ID, Varint::Value((header_len + data.len()) as u64), output)?; + + encode_varint(Varint::Value(track), output)?; + + let mut buffer = Cursor::new([0; 3]); + buffer.put_i16::(timecode); + buffer.put_u8(flags); + + output.write_all(&buffer.get_ref()[..])?; + output.write_all(data) } pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { @@ -118,7 +119,7 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), - WebmElement::SimpleBlock {..} => encode_simple_block(element, output), + WebmElement::SimpleBlock(block) => encode_simple_block(block, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } @@ -142,24 +143,24 @@ mod tests { assert_eq!(iter.next(), Some(WebmElement::Cluster)); assert_eq!(iter.next(), Some(WebmElement::Timecode(0))); - assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock(SimpleBlock { track: 1, timecode: 0, flags: 0b10000000, data: &TEST_FILE[443..3683] - })); - assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + }))); + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock(SimpleBlock { track: 1, timecode: 33, flags: 0b00000000, data: &TEST_FILE[3690..4735] - })); - assert_eq!(iter.next(), Some(WebmElement::SimpleBlock { + }))); + assert_eq!(iter.next(), Some(WebmElement::SimpleBlock(SimpleBlock { track: 1, timecode: 67, flags: 0b00000000, data: &TEST_FILE[4741..4801] - })); + }))); for _ in 3..30 { // skip remaining contents for brevity iter.next(); From 271782bb913f7b680c0095bbd9dc495e535f04fd Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 5 Sep 2017 00:17:11 -0400 Subject: [PATCH 053/164] Create test tool to deconstruct and reassemble a WebM file --- src/bin/resynth.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/bin/resynth.rs diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs new file mode 100644 index 0000000..8110019 --- /dev/null +++ b/src/bin/resynth.rs @@ -0,0 +1,49 @@ +extern crate lab_ebml; + +use std::io::{Cursor, stdout, Write}; +use lab_ebml::Schema; +use lab_ebml::webm::*; +use lab_ebml::webm::WebmElement::*; + +const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); + +pub fn main() { + + let mut head = Vec::new(); + let mut body = Vec::new(); + + let mut reading_head = true; + + for element in Webm.parse(SRC_FILE) { + match element { + Cluster => reading_head = false, + // TODO: skip elements not required for streaming + Info => continue, + Void => continue, + Unknown(_) => continue, + _ => (), + } + + if reading_head { + head.push(element); + } else { + body.push(element); + } + } + + let mut output = Vec::new(); + let mut cursor = Cursor::new(output); + + for element in head { + encode_webm_element(element, &mut cursor).unwrap(); + } + + for element in body { + encode_webm_element(element, &mut cursor).unwrap(); + } + + output = cursor.into_inner(); + stdout().write_all(&output).unwrap(); + output.clear(); + +} From 84c7ec734be6aee40e8a27a763780746c20478cd Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 5 Sep 2017 00:24:38 -0400 Subject: [PATCH 054/164] Don't require encode_ functions to consume their argument element. --- src/bin/resynth.rs | 4 ++-- src/webm.rs | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index 8110019..0419eb1 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -34,11 +34,11 @@ pub fn main() { let mut output = Vec::new(); let mut cursor = Cursor::new(output); - for element in head { + for element in &head { encode_webm_element(element, &mut cursor).unwrap(); } - for element in body { + for element in &body { encode_webm_element(element, &mut cursor).unwrap(); } diff --git a/src/webm.rs b/src/webm.rs index ccc313b..21ebb61 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -83,8 +83,8 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } } -pub fn encode_simple_block(block: SimpleBlock, output: &mut T) -> IoResult<()> { - let SimpleBlock { +pub fn encode_simple_block(block: &SimpleBlock, output: &mut T) -> IoResult<()> { + let &SimpleBlock { track, timecode, flags, @@ -108,18 +108,18 @@ pub fn encode_simple_block(block: SimpleBlock, output: &mut T) -> IoRe output.write_all(data) } -pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { +pub fn encode_webm_element(element: &WebmElement, output: &mut T) -> IoResult<()> { match element { - WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { + &WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { encode_bytes(DOC_TYPE_ID, "webm".as_bytes(), output) }), - WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), - WebmElement::SeekHead => Ok(()), - WebmElement::Cues => Ok(()), - WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), - WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), - WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), - WebmElement::SimpleBlock(block) => encode_simple_block(block, output), + &WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), + &WebmElement::SeekHead => Ok(()), + &WebmElement::Cues => Ok(()), + &WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), + &WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), + &WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), + &WebmElement::SimpleBlock(ref block) => encode_simple_block(block, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From 9b3a6db07472bcd680f753740ee6b1bed99ee3d4 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 6 Sep 2017 01:53:49 -0400 Subject: [PATCH 055/164] fixup bin for reference refactor. --- src/bin/stub.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/bin/stub.rs b/src/bin/stub.rs index b3e0951..56f84d7 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -6,23 +6,23 @@ use lab_ebml::webm::*; pub fn main() { let mut cursor = Cursor::new(Vec::new()); - encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::EbmlHead, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Segment, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Tracks(&[]), &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Tracks(&[]), &mut cursor).unwrap(); - encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Timecode(0), &mut cursor).unwrap(); - encode_webm_element(WebmElement::SimpleBlock(SimpleBlock { + encode_webm_element(&WebmElement::SimpleBlock(SimpleBlock { track: 3, flags: 0x0, timecode: 123, data: "Hello, World".as_bytes() }), &mut cursor).unwrap(); - encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Timecode(1000), &mut cursor).unwrap(); stdout().write_all(&cursor.get_ref()).unwrap(); } From 13e75a6415b8a92d049c172b25420611e14d72bc Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 6 Sep 2017 02:02:50 -0400 Subject: [PATCH 056/164] Add a stage to resynth test's output pipeline to permit patching timestamps --- src/bin/resynth.rs | 7 +++++-- src/lib.rs | 1 + src/timecode_fixer.rs | 18 ++++++++++++++++++ src/webm.rs | 4 ++-- 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 src/timecode_fixer.rs diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index 0419eb1..70526f3 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -4,6 +4,7 @@ use std::io::{Cursor, stdout, Write}; use lab_ebml::Schema; use lab_ebml::webm::*; use lab_ebml::webm::WebmElement::*; +use lab_ebml::timecode_fixer::TimecodeFixer; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); @@ -34,12 +35,14 @@ pub fn main() { let mut output = Vec::new(); let mut cursor = Cursor::new(output); + let mut fixer = TimecodeFixer::new(); + for element in &head { - encode_webm_element(element, &mut cursor).unwrap(); + encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); } for element in &body { - encode_webm_element(element, &mut cursor).unwrap(); + encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); } output = cursor.into_inner(); diff --git a/src/lib.rs b/src/lib.rs index db405a5..b20111c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ extern crate futures; pub mod ebml; mod iterator; +pub mod timecode_fixer; pub mod webm; pub use ebml::{Error, Schema}; diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs new file mode 100644 index 0000000..17aecab --- /dev/null +++ b/src/timecode_fixer.rs @@ -0,0 +1,18 @@ +// TODO: (iterator? stream?) adapter that fixes SimpleBlock/Cluster timecodes +use webm::WebmElement; + +pub struct TimecodeFixer { +} + +impl TimecodeFixer { + pub fn new() -> TimecodeFixer { + TimecodeFixer { + } + } + + pub fn process<'b>(&mut self, element: &WebmElement<'b>) -> WebmElement<'b> { + match element { + _ => *element + } + } +} diff --git a/src/webm.rs b/src/webm.rs index 21ebb61..3123977 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -12,7 +12,7 @@ const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; pub struct Webm; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Copy, Clone)] pub struct SimpleBlock<'b> { pub track: u64, pub timecode: i16, @@ -20,7 +20,7 @@ pub struct SimpleBlock<'b> { pub data: &'b[u8] } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Copy, Clone)] pub enum WebmElement<'b> { EbmlHead, Void, From e633517d3a461cfabfd28c3d7e8c3ffec69b8459 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 7 Sep 2017 01:42:14 -0400 Subject: [PATCH 057/164] Rewrite timestamps on clusters to allow splicing --- src/bin/resynth.rs | 4 ++++ src/timecode_fixer.rs | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index 70526f3..acaef39 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -45,6 +45,10 @@ pub fn main() { encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); } + for element in &body { + encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); + } + output = cursor.into_inner(); stdout().write_all(&output).unwrap(); output.clear(); diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index 17aecab..b379dd5 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -2,16 +2,41 @@ use webm::WebmElement; pub struct TimecodeFixer { + pub current_offset: u64, + pub last_cluster_base: u64, + pub last_observed_timecode: u64, + pub assumed_duration: u64 } impl TimecodeFixer { pub fn new() -> TimecodeFixer { TimecodeFixer { + current_offset: 0, + last_cluster_base: 0, + last_observed_timecode: 0, + assumed_duration: 33 } } pub fn process<'b>(&mut self, element: &WebmElement<'b>) -> WebmElement<'b> { match element { + &WebmElement::Timecode(timecode) => { + // detect a jump backwards in the source, meaning we need to recalculate our offset + if timecode < self.last_cluster_base { + let next_timecode = self.last_observed_timecode + self.assumed_duration; + self.current_offset = next_timecode - timecode; + } + + // remember the source timecode to detect future jumps + self.last_cluster_base = timecode; + + // return adjusted timecode + WebmElement::Timecode(timecode + self.current_offset) + }, + &WebmElement::SimpleBlock(block) => { + self.last_observed_timecode = self.last_cluster_base + (block.timecode as u64); + *element + }, _ => *element } } From 038a042887fd5a29171115fdbba10c851e146aad Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 8 Sep 2017 01:54:46 -0400 Subject: [PATCH 058/164] Add hyper --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6b6f0fd..2f62d86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,4 +5,5 @@ authors = ["Tangent 128 "] [dependencies] bytes = "0.4" -futures = "^0.1.7" +futures = "0.1.14" +hyper = "0.11.2" From 6f80da47352e7352cb7f0c1d40718b8b19fc91be Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 10 Sep 2017 19:34:08 -0400 Subject: [PATCH 059/164] Create stub Hyper server --- src/bin/loop_server.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/bin/loop_server.rs diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs new file mode 100644 index 0000000..f3fb3d5 --- /dev/null +++ b/src/bin/loop_server.rs @@ -0,0 +1,28 @@ +extern crate futures; +extern crate hyper; +extern crate lab_ebml; + +use futures::future::FutureResult; +use hyper::StatusCode; +use hyper::server::{Http, Request, Response, Service}; +use std::env::args; +use std::net::ToSocketAddrs; + +const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); + +struct WebmServer; + +impl Service for WebmServer { + type Request = Request; + type Response = Response; + type Error = hyper::Error; + type Future = FutureResult; + fn call(&self, req: Request) -> Self::Future { + futures::future::ok(Response::new().with_status(StatusCode::NotFound)) + } +} + +pub fn main() { + let addr = args().nth(1).unwrap().to_socket_addrs().unwrap().next().unwrap(); + Http::new().bind(&addr, || Ok(WebmServer)).unwrap().run().unwrap(); +} From ef457287795b2259b748a63f9ca2b3b5c28ed534 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 10 Sep 2017 19:44:54 -0400 Subject: [PATCH 060/164] Have stub server recognize a URL path --- src/bin/loop_server.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index f3fb3d5..2760854 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,12 +3,12 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; -use hyper::StatusCode; +use hyper::{Get, StatusCode}; use hyper::server::{Http, Request, Response, Service}; use std::env::args; use std::net::ToSocketAddrs; -const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); +//const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); struct WebmServer; @@ -18,7 +18,17 @@ impl Service for WebmServer { type Error = hyper::Error; type Future = FutureResult; fn call(&self, req: Request) -> Self::Future { - futures::future::ok(Response::new().with_status(StatusCode::NotFound)) + let response = match (req.method(), req.path()) { + (&Get, "/loop") => { + Response::new() + .with_body("") + }, + _ => { + Response::new() + .with_status(StatusCode::NotFound) + } + }; + futures::future::ok(response) } } From db1731f776e7fe28872f46aee8d1952cc0e86b71 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 21 Sep 2017 00:59:11 -0400 Subject: [PATCH 061/164] Experiment with using a Stream for a Response Body --- src/bin/loop_server.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 2760854..6bf9669 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,6 +3,7 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; +use futures::stream::{iter, Stream}; use hyper::{Get, StatusCode}; use hyper::server::{Http, Request, Response, Service}; use std::env::args; @@ -12,16 +13,20 @@ use std::net::ToSocketAddrs; struct WebmServer; +type BodyStream = Box>; + impl Service for WebmServer { type Request = Request; - type Response = Response; + type Response = Response; type Error = hyper::Error; - type Future = FutureResult; + type Future = FutureResult; fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { + let pieces = vec!["<", "Insert WebM stream here.", ">"]; + let stream: BodyStream = iter(pieces.into_iter().map(|x| Ok(x))).boxed(); Response::new() - .with_body("") + .with_body(stream) }, _ => { Response::new() From 0a18cb408f652afc77b87be09b6664535644f7b2 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 22 Sep 2017 23:58:03 -0400 Subject: [PATCH 062/164] Create Chunk type for use in packaging WebM streams for clients. --- src/chunk.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 51 insertions(+) create mode 100644 src/chunk.rs diff --git a/src/chunk.rs b/src/chunk.rs new file mode 100644 index 0000000..181bcb2 --- /dev/null +++ b/src/chunk.rs @@ -0,0 +1,50 @@ +use std::rc::Rc; + +pub enum Chunk { + Headers { + bytes: Rc> + }, + ClusterHead { + keyframe: bool, + start: u64, + end: u64, + // space for a Cluster tag and a Timecode tag + bytes: [u8;16] + }, + ClusterBody { + bytes: Rc> + } +} + +impl AsRef<[u8]> for Chunk { + fn as_ref(&self) -> &[u8] { + match self { + &Chunk::Headers {ref bytes, ..} => &bytes, + &Chunk::ClusterHead {ref bytes, ..} => bytes, + &Chunk::ClusterBody {ref bytes, ..} => &bytes + } + } +} + +#[cfg(test)] +mod tests { + + use chunk::*; + use std::io::Cursor; + use webm::*; + + #[test] + fn enough_space_for_header() { + let mut chunk = Chunk::ClusterHead { + keyframe: false, + start: 0, + end: 0, + bytes: [0;16] + }; + if let Chunk::ClusterHead {ref mut bytes, ..} = chunk { + let mut cursor = Cursor::new(bytes as &mut [u8]); + encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Timecode(u64::max_value()), &mut cursor).unwrap(); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index b20111c..4f689c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ extern crate bytes; extern crate futures; +pub mod chunk; pub mod ebml; mod iterator; pub mod timecode_fixer; From 27642f7b14457f2fe275ddbb9cdc43618b093213 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 25 Sep 2017 00:22:41 -0400 Subject: [PATCH 063/164] Make Chunk flexible in the type of backing buffer used --- src/chunk.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 181bcb2..c68fd22 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,8 +1,9 @@ use std::rc::Rc; -pub enum Chunk { +#[derive(Clone)] +pub enum Chunk = Vec> { Headers { - bytes: Rc> + bytes: Rc }, ClusterHead { keyframe: bool, @@ -12,16 +13,16 @@ pub enum Chunk { bytes: [u8;16] }, ClusterBody { - bytes: Rc> + bytes: Rc } } -impl AsRef<[u8]> for Chunk { +impl> AsRef<[u8]> for Chunk { fn as_ref(&self) -> &[u8] { match self { - &Chunk::Headers {ref bytes, ..} => &bytes, + &Chunk::Headers {ref bytes, ..} => bytes.as_ref().as_ref(), &Chunk::ClusterHead {ref bytes, ..} => bytes, - &Chunk::ClusterBody {ref bytes, ..} => &bytes + &Chunk::ClusterBody {ref bytes, ..} => bytes.as_ref().as_ref() } } } @@ -35,7 +36,7 @@ mod tests { #[test] fn enough_space_for_header() { - let mut chunk = Chunk::ClusterHead { + let mut chunk: Chunk = Chunk::ClusterHead { keyframe: false, start: 0, end: 0, From d0f1cde4642bbd26350b9bb9d95e4967078cd1f5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 25 Sep 2017 00:23:20 -0400 Subject: [PATCH 064/164] Stub using Chunk stream in loop_server. --- src/bin/loop_server.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 6bf9669..415141d 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -4,16 +4,18 @@ extern crate lab_ebml; use futures::future::FutureResult; use futures::stream::{iter, Stream}; +use lab_ebml::chunk::Chunk; use hyper::{Get, StatusCode}; use hyper::server::{Http, Request, Response, Service}; use std::env::args; +use std::rc::Rc; use std::net::ToSocketAddrs; //const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); struct WebmServer; -type BodyStream = Box>; +type BodyStream = Box, Error = hyper::Error>>; impl Service for WebmServer { type Request = Request; @@ -23,8 +25,12 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let pieces = vec!["<", "Insert WebM stream here.", ">"]; - let stream: BodyStream = iter(pieces.into_iter().map(|x| Ok(x))).boxed(); + let pieces = vec![ + Chunk::Headers {bytes: Rc::new("<")}, + Chunk::Headers {bytes: Rc::new("Insert WebM stream here")}, + Chunk::ClusterBody {bytes: Rc::new(">")} + ]; + let stream: BodyStream = Box::new(iter(pieces.into_iter().map(|x| Ok(x)))); Response::new() .with_body(stream) }, From c3f0de71fa62732ee945a8e2d07392ebab2c76cf Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 25 Sep 2017 01:41:56 -0400 Subject: [PATCH 065/164] Move Chunk timecode updating into the struct impl proper. --- src/chunk.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index c68fd22..4c46bfd 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,4 +1,6 @@ +use std::io::Cursor; use std::rc::Rc; +use webm::*; #[derive(Clone)] pub enum Chunk = Vec> { @@ -17,6 +19,20 @@ pub enum Chunk = Vec> { } } +impl> Chunk { + pub fn update_timecode(&mut self, timecode: u64) { + if let &mut Chunk::ClusterHead {ref mut start, ref mut end, ref mut bytes, ..} = self { + let delta = *end - *start; + *start = timecode; + *end = *start + delta; + let mut cursor = Cursor::new(bytes as &mut [u8]); + // buffer is sized so these should never fail + encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Timecode(timecode), &mut cursor).unwrap(); + } + } +} + impl> AsRef<[u8]> for Chunk { fn as_ref(&self) -> &[u8] { match self { @@ -31,8 +47,6 @@ impl> AsRef<[u8]> for Chunk { mod tests { use chunk::*; - use std::io::Cursor; - use webm::*; #[test] fn enough_space_for_header() { @@ -42,10 +56,6 @@ mod tests { end: 0, bytes: [0;16] }; - if let Chunk::ClusterHead {ref mut bytes, ..} = chunk { - let mut cursor = Cursor::new(bytes as &mut [u8]); - encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Timecode(u64::max_value()), &mut cursor).unwrap(); - } + chunk.update_timecode(u64::max_value()); } } From d20081d9a1acaab2b1b696eedee91a79018bd611 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 26 Sep 2017 02:21:32 -0400 Subject: [PATCH 066/164] Serve single, unparsed video file. --- src/bin/loop_server.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 415141d..88c85b0 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,19 +3,20 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; -use futures::stream::{iter, Stream}; +use futures::stream::{once, Stream}; use lab_ebml::chunk::Chunk; use hyper::{Get, StatusCode}; +use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; use std::env::args; use std::rc::Rc; use std::net::ToSocketAddrs; -//const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); +const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); struct WebmServer; -type BodyStream = Box, Error = hyper::Error>>; +type BodyStream = Box, Error = hyper::Error>>; impl Service for WebmServer { type Request = Request; @@ -25,13 +26,9 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let pieces = vec![ - Chunk::Headers {bytes: Rc::new("<")}, - Chunk::Headers {bytes: Rc::new("Insert WebM stream here")}, - Chunk::ClusterBody {bytes: Rc::new(">")} - ]; - let stream: BodyStream = Box::new(iter(pieces.into_iter().map(|x| Ok(x)))); + let stream: BodyStream = Box::new(once(Ok(Chunk::Headers {bytes: Rc::new(SRC_FILE)}))); Response::new() + .with_header(ContentType("video/webm".parse().unwrap())) .with_body(stream) }, _ => { From 7f214920aed046fc5481de1eb456c059ba26101b Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 28 Sep 2017 22:11:27 -0400 Subject: [PATCH 067/164] switch Chunks to use Arcs --- src/bin/loop_server.rs | 4 ++-- src/chunk.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 88c85b0..20083b5 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -9,8 +9,8 @@ use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; use std::env::args; -use std::rc::Rc; use std::net::ToSocketAddrs; +use std::sync::Arc; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); @@ -26,7 +26,7 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream = Box::new(once(Ok(Chunk::Headers {bytes: Rc::new(SRC_FILE)}))); + let stream: BodyStream = Box::new(once(Ok(Chunk::Headers {bytes: Arc::new(SRC_FILE)}))); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) .with_body(stream) diff --git a/src/chunk.rs b/src/chunk.rs index 4c46bfd..1d82db7 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,11 +1,11 @@ use std::io::Cursor; -use std::rc::Rc; +use std::sync::Arc; use webm::*; #[derive(Clone)] pub enum Chunk = Vec> { Headers { - bytes: Rc + bytes: Arc }, ClusterHead { keyframe: bool, @@ -15,7 +15,7 @@ pub enum Chunk = Vec> { bytes: [u8;16] }, ClusterBody { - bytes: Rc + bytes: Arc } } From 031d32352adc52bc8c77257996263480054587e1 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 00:07:56 -0400 Subject: [PATCH 068/164] Fix dumping the whole cluster head buffer instead of the used portion --- src/chunk.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 1d82db7..cfa4506 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -12,7 +12,8 @@ pub enum Chunk = Vec> { start: u64, end: u64, // space for a Cluster tag and a Timecode tag - bytes: [u8;16] + bytes: [u8;16], + bytes_used: u8 }, ClusterBody { bytes: Arc @@ -21,7 +22,7 @@ pub enum Chunk = Vec> { impl> Chunk { pub fn update_timecode(&mut self, timecode: u64) { - if let &mut Chunk::ClusterHead {ref mut start, ref mut end, ref mut bytes, ..} = self { + if let &mut Chunk::ClusterHead {ref mut start, ref mut end, ref mut bytes, ref mut bytes_used, ..} = self { let delta = *end - *start; *start = timecode; *end = *start + delta; @@ -29,6 +30,7 @@ impl> Chunk { // buffer is sized so these should never fail encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); encode_webm_element(&WebmElement::Timecode(timecode), &mut cursor).unwrap(); + *bytes_used = cursor.position() as u8; } } } @@ -37,7 +39,7 @@ impl> AsRef<[u8]> for Chunk { fn as_ref(&self) -> &[u8] { match self { &Chunk::Headers {ref bytes, ..} => bytes.as_ref().as_ref(), - &Chunk::ClusterHead {ref bytes, ..} => bytes, + &Chunk::ClusterHead {ref bytes, bytes_used, ..} => bytes[..bytes_used as usize].as_ref(), &Chunk::ClusterBody {ref bytes, ..} => bytes.as_ref().as_ref() } } From a7804891ae1a0be8cf6c37e60953b4f63fdd2150 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 00:12:49 -0400 Subject: [PATCH 069/164] add chunk mutators/utilities --- src/chunk.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/chunk.rs b/src/chunk.rs index cfa4506..072f902 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -21,6 +21,18 @@ pub enum Chunk = Vec> { } impl> Chunk { + pub fn new_cluster_head(timecode: u64) -> Chunk { + let mut chunk = Chunk::ClusterHead { + keyframe: false, + start: 0, + end: 0, + bytes: [0;16], + bytes_used: 0 + }; + chunk.update_timecode(timecode); + chunk + } + pub fn update_timecode(&mut self, timecode: u64) { if let &mut Chunk::ClusterHead {ref mut start, ref mut end, ref mut bytes, ref mut bytes_used, ..} = self { let delta = *end - *start; @@ -33,6 +45,18 @@ impl> Chunk { *bytes_used = cursor.position() as u8; } } + pub fn extend_timespan(&mut self, timecode: u64) { + if let &mut Chunk::ClusterHead {start, ref mut end, ..} = self { + if timecode > start { + *end = timecode; + } + } + } + pub fn mark_keyframe(&mut self, new_keyframe: bool) { + if let &mut Chunk::ClusterHead {ref mut keyframe, ..} = self { + *keyframe = new_keyframe; + } + } } impl> AsRef<[u8]> for Chunk { From 9481d53adccdd02284b7fe9e5c34d64ca67bbd49 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 01:31:06 -0400 Subject: [PATCH 070/164] Parse, chunk, & reconstruct served WebM file. --- src/bin/loop_server.rs | 75 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 20083b5..01ad4f1 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,30 +3,39 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; -use futures::stream::{once, Stream}; +use futures::stream::{once, iter, Stream}; use lab_ebml::chunk::Chunk; +use lab_ebml::Schema; +use lab_ebml::webm::*; +use lab_ebml::webm::WebmElement::*; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; use std::env::args; +use std::io::Cursor; use std::net::ToSocketAddrs; use std::sync::Arc; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); -struct WebmServer; +#[derive(Clone)] +struct WebmServer(Chunk, Vec); -type BodyStream = Box, Error = hyper::Error>>; +type BodyStream = Box, Error = hyper::Error>>; impl Service for WebmServer { type Request = Request; - type Response = Response; + type Response = Response>>; type Error = hyper::Error; type Future = FutureResult; fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream = Box::new(once(Ok(Chunk::Headers {bytes: Arc::new(SRC_FILE)}))); + let results: Vec> = self.1.iter().map(|x| Ok(x.clone())).collect(); + let stream: BodyStream> = Box::new( + once(Ok(self.0.clone())) + .chain(iter(results)) + ); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) .with_body(stream) @@ -42,5 +51,59 @@ impl Service for WebmServer { pub fn main() { let addr = args().nth(1).unwrap().to_socket_addrs().unwrap().next().unwrap(); - Http::new().bind(&addr, || Ok(WebmServer)).unwrap().run().unwrap(); + let webm_service = create_loop(); + Http::new().bind(&addr, move || Ok(webm_service.clone())).unwrap().run().unwrap(); +} + +fn create_loop() -> WebmServer { + let mut header = None; + let mut reading_head = true; + + let mut cluster_header = None; + let mut cluster_timecode = 0; + let mut chunks = Vec::new(); + + let mut buffer = Cursor::new(Vec::new()); + + for element in Webm.parse(SRC_FILE) { + match element { + Cluster => { + if reading_head { + header = Some(Chunk::Headers {bytes: Arc::new(buffer.into_inner())}); + } else { + if let Some(chunk) = cluster_header.take() { + chunks.push(chunk); + } + chunks.push(Chunk::ClusterBody {bytes: Arc::new(buffer.into_inner())}); + } + buffer = Cursor::new(Vec::new()); + reading_head = false; + }, + Timecode(timecode) => { + cluster_timecode = timecode; + cluster_header = Some(Chunk::>::new_cluster_head(timecode)); + }, + SimpleBlock(ref block) => { + if let Some(ref mut chunk) = cluster_header { + if (block.flags & 0b10000000) != 0 { + // TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster + chunk.mark_keyframe(true); + } + chunk.extend_timespan(cluster_timecode + (block.timecode as u64)); + } + encode_webm_element(&SimpleBlock(*block), &mut buffer).unwrap(); + }, + Info => continue, + Void => continue, + Unknown(_) => continue, + ref other => { + encode_webm_element(other, &mut buffer).unwrap(); + }, + } + } + + // finish last cluster + chunks.push(Chunk::ClusterBody {bytes: Arc::new(buffer.into_inner())}); + + WebmServer(header.unwrap(), chunks) } From c4fc13fe0bd29335224af2ffac6b69a6f45fc0a6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 01:54:08 -0400 Subject: [PATCH 071/164] Fix final cluster not displaying right --- src/bin/loop_server.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 01ad4f1..faa7f13 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -103,6 +103,9 @@ fn create_loop() -> WebmServer { } // finish last cluster + if let Some(chunk) = cluster_header.take() { + chunks.push(chunk); + } chunks.push(Chunk::ClusterBody {bytes: Arc::new(buffer.into_inner())}); WebmServer(header.unwrap(), chunks) From 15386e96380f38cc85f4c782ec98deda0136e69f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 02:11:42 -0400 Subject: [PATCH 072/164] Repeat test video body a few times --- src/bin/loop_server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index faa7f13..7dceff5 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -31,10 +31,11 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let results: Vec> = self.1.iter().map(|x| Ok(x.clone())).collect(); + let results: Vec> = self.1.iter().map(|x| Ok(x.clone())).collect(); let stream: BodyStream> = Box::new( once(Ok(self.0.clone())) - .chain(iter(results)) + .chain(iter(results.into_iter().cycle().take(20))) + .map_err(|_| hyper::Error::Incomplete) ); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) From dae4a49481cda2b14f5580e39a63a0e1707b1b0c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 29 Sep 2017 15:30:10 -0400 Subject: [PATCH 073/164] update notes --- notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notes.md b/notes.md index 9daaa4b..f05e36b 100644 --- a/notes.md +++ b/notes.md @@ -1,3 +1,5 @@ -* element, as utf8, as binary, as int, as container of subelements (body left "open" then) * support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) * rustfmt modules +* timestamp fix for chunks (stream modifier/extension method) +* create chunks from webm event stream +* bytestream source for ebml parsing From 3e1bec93eef14095da68c68af5d9ccf76bee93bd Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 30 Sep 2017 02:23:27 -0400 Subject: [PATCH 074/164] Fix tests for chunk.rs --- src/chunk.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 072f902..69390e4 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -76,12 +76,6 @@ mod tests { #[test] fn enough_space_for_header() { - let mut chunk: Chunk = Chunk::ClusterHead { - keyframe: false, - start: 0, - end: 0, - bytes: [0;16] - }; - chunk.update_timecode(u64::max_value()); + Chunk::>::new_cluster_head(u64::max_value()); } } From bad7b42e5bc8eebc582f0b80db3ad9ef0dc85556 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 30 Sep 2017 02:33:36 -0400 Subject: [PATCH 075/164] Create stub stream operator to bolt on fixing chunk timecodes --- src/bin/loop_server.rs | 2 ++ src/timecode_fixer.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 7dceff5..df62043 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -6,6 +6,7 @@ use futures::future::FutureResult; use futures::stream::{once, iter, Stream}; use lab_ebml::chunk::Chunk; use lab_ebml::Schema; +use lab_ebml::timecode_fixer::ChunkStream; use lab_ebml::webm::*; use lab_ebml::webm::WebmElement::*; use hyper::{Get, StatusCode}; @@ -36,6 +37,7 @@ impl Service for WebmServer { once(Ok(self.0.clone())) .chain(iter(results.into_iter().cycle().take(20))) .map_err(|_| hyper::Error::Incomplete) + .fix_timecodes() ); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index b379dd5..136dd36 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -1,4 +1,6 @@ -// TODO: (iterator? stream?) adapter that fixes SimpleBlock/Cluster timecodes +use chunk::Chunk; +use futures::Async; +use futures::stream::Stream; use webm::WebmElement; pub struct TimecodeFixer { @@ -41,3 +43,29 @@ impl TimecodeFixer { } } } + +pub struct ChunkTimecodeFixer { + stream: S +} + +impl> Stream for ChunkTimecodeFixer +{ + type Item = S::Item; + type Error = S::Error; + + fn poll(&mut self) -> Result>, Self::Error> { + self.stream.poll() + } +} + +pub trait ChunkStream { + fn fix_timecodes(self) -> ChunkTimecodeFixer; +} + +impl> ChunkStream for T { + fn fix_timecodes(self) -> ChunkTimecodeFixer { + ChunkTimecodeFixer { + stream: self + } + } +} From 27aff20c464281286461b890bebb9e465e65c833 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 30 Sep 2017 17:36:09 -0400 Subject: [PATCH 076/164] Implement rewriting timestamps for chunks --- notes.md | 1 - src/timecode_fixer.rs | 27 ++++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/notes.md b/notes.md index f05e36b..0ba443a 100644 --- a/notes.md +++ b/notes.md @@ -1,5 +1,4 @@ * support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) * rustfmt modules -* timestamp fix for chunks (stream modifier/extension method) * create chunks from webm event stream * bytestream source for ebml parsing diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index 136dd36..5f380a9 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -45,7 +45,10 @@ impl TimecodeFixer { } pub struct ChunkTimecodeFixer { - stream: S + stream: S, + current_offset: u64, + last_observed_timecode: u64, + assumed_duration: u64, } impl> Stream for ChunkTimecodeFixer @@ -54,7 +57,22 @@ impl> Stream for ChunkTimecodeFixer type Error = S::Error; fn poll(&mut self) -> Result>, Self::Error> { - self.stream.poll() + let mut poll_chunk = self.stream.poll(); + match poll_chunk { + Ok(Async::Ready(Some(Chunk::ClusterHead {start, end, ..}))) => { + if start < self.last_observed_timecode { + let next_timecode = self.last_observed_timecode + self.assumed_duration; + self.current_offset = next_timecode - start; + } + + if let Ok(Async::Ready(Some(ref mut cluster_head))) = poll_chunk { + cluster_head.update_timecode(start + self.current_offset); + } + self.last_observed_timecode = end + self.current_offset; + }, + _ => {} + }; + poll_chunk } } @@ -65,7 +83,10 @@ pub trait ChunkStream { impl> ChunkStream for T { fn fix_timecodes(self) -> ChunkTimecodeFixer { ChunkTimecodeFixer { - stream: self + stream: self, + current_offset: 0, + last_observed_timecode: 0, + assumed_duration: 33 } } } From 52c1843311d138326ded720af5d466168bd5acc5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 1 Oct 2017 00:21:33 -0400 Subject: [PATCH 077/164] Add stub Webm chunking stream operator --- src/chunk.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/chunk.rs b/src/chunk.rs index 69390e4..f4c7599 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,3 +1,4 @@ +use futures::{Async, Stream}; use std::io::Cursor; use std::sync::Arc; use webm::*; @@ -69,6 +70,32 @@ impl> AsRef<[u8]> for Chunk { } } +pub struct WebmChunker { + stream: S +} + +impl<'a, S: Stream>> Stream for WebmChunker +{ + type Item = Chunk; + type Error = S::Error; + + fn poll(&mut self) -> Result>, Self::Error> { + Ok(Async::NotReady) + } +} + +pub trait WebmStream { + fn chunk_webm(self) -> WebmChunker; +} + +impl<'a, T: Stream>> WebmStream for T { + fn chunk_webm(self) -> WebmChunker { + WebmChunker { + stream: self + } + } +} + #[cfg(test)] mod tests { From 0aa2f1cbdd8c3f5069225df8751850552d779b0d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 01:05:23 -0400 Subject: [PATCH 078/164] Implement header chunking for chunk_webm() operator --- src/chunk.rs | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index f4c7599..5a0dc2d 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,5 +1,6 @@ use futures::{Async, Stream}; use std::io::Cursor; +use std::mem; use std::sync::Arc; use webm::*; @@ -70,8 +71,16 @@ impl> AsRef<[u8]> for Chunk { } } +enum ChunkerState { + BuildingHeader(Cursor>), + // WIP ClusterHead & body buffer + BuildingCluster(Chunk, Cursor>), + EmittingClusterBody(Chunk) +} + pub struct WebmChunker { - stream: S + stream: S, + state: ChunkerState } impl<'a, S: Stream>> Stream for WebmChunker @@ -80,7 +89,41 @@ impl<'a, S: Stream>> Stream for WebmChunker type Error = S::Error; fn poll(&mut self) -> Result>, Self::Error> { - Ok(Async::NotReady) + loop { + let (return_value, next_state) = match self.state { + ChunkerState::BuildingHeader(ref mut buffer) => { + match self.stream.poll() { + Err(passthru) => return Err(passthru), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), + Ok(Async::Ready(Some(WebmElement::Cluster))) => { + let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); + let chunk = Chunk::Headers {bytes: Arc::new(liberated_buffer.into_inner())}; + ( + Ok(Async::Ready(Some(chunk))), + ChunkerState::BuildingCluster( + Chunk::>::new_cluster_head(0), + Cursor::new(Vec::new()) + ) + ) + }, + Ok(Async::Ready(Some(element @ _))) => { + encode_webm_element(&element, buffer); + continue; + } + } + }, + ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => { + return Ok(Async::Ready(None)); + }, + ChunkerState::EmittingClusterBody(ref mut buffer) => { + return Ok(Async::Ready(None)); + } + }; + + self.state = next_state; + return return_value; + } } } @@ -91,7 +134,8 @@ pub trait WebmStream { impl<'a, T: Stream>> WebmStream for T { fn chunk_webm(self) -> WebmChunker { WebmChunker { - stream: self + stream: self, + state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())) } } } From 9c9db1c505b868e6d9a697b6fa616afdc93e290d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 01:48:33 -0400 Subject: [PATCH 079/164] Better-adapt extend_timespan to its real use, observe_simpleblock_timecode --- src/bin/loop_server.rs | 2 +- src/chunk.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index df62043..0967182 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -92,7 +92,7 @@ fn create_loop() -> WebmServer { // TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster chunk.mark_keyframe(true); } - chunk.extend_timespan(cluster_timecode + (block.timecode as u64)); + chunk.observe_simpleblock_timecode(block.timecode); } encode_webm_element(&SimpleBlock(*block), &mut buffer).unwrap(); }, diff --git a/src/chunk.rs b/src/chunk.rs index 5a0dc2d..20b29cf 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -47,10 +47,11 @@ impl> Chunk { *bytes_used = cursor.position() as u8; } } - pub fn extend_timespan(&mut self, timecode: u64) { + pub fn observe_simpleblock_timecode(&mut self, timecode: i16) { if let &mut Chunk::ClusterHead {start, ref mut end, ..} = self { - if timecode > start { - *end = timecode; + let absolute_timecode = start + (timecode as u64); + if absolute_timecode > start { + *end = absolute_timecode; } } } From 639dc50c354400945b96e0d4bf712f3eeb665882 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 01:49:41 -0400 Subject: [PATCH 080/164] Implement chunking Clusters, including flushing the last one --- src/chunk.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 20b29cf..b8ce3f2 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -76,7 +76,9 @@ enum ChunkerState { BuildingHeader(Cursor>), // WIP ClusterHead & body buffer BuildingCluster(Chunk, Cursor>), - EmittingClusterBody(Chunk) + EmittingClusterBody(Vec), + EmittingFinalClusterBody(Vec), + End } pub struct WebmChunker { @@ -99,9 +101,9 @@ impl<'a, S: Stream>> Stream for WebmChunker Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), Ok(Async::Ready(Some(WebmElement::Cluster))) => { let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); - let chunk = Chunk::Headers {bytes: Arc::new(liberated_buffer.into_inner())}; + let header_chunk = Chunk::Headers {bytes: Arc::new(liberated_buffer.into_inner())}; ( - Ok(Async::Ready(Some(chunk))), + Ok(Async::Ready(Some(header_chunk))), ChunkerState::BuildingCluster( Chunk::>::new_cluster_head(0), Cursor::new(Vec::new()) @@ -115,11 +117,63 @@ impl<'a, S: Stream>> Stream for WebmChunker } }, ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => { - return Ok(Async::Ready(None)); + match self.stream.poll() { + Err(passthru) => return Err(passthru), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(Some(WebmElement::Cluster))) => { + let cluster_head_chunk = mem::replace(cluster_head, Chunk::>::new_cluster_head(0)); + let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); + ( + Ok(Async::Ready(Some(cluster_head_chunk))), + ChunkerState::EmittingClusterBody(liberated_buffer.into_inner()) + ) + }, + Ok(Async::Ready(Some(WebmElement::Timecode(timecode)))) => { + cluster_head.update_timecode(timecode); + continue; + }, + Ok(Async::Ready(Some(WebmElement::SimpleBlock(ref block)))) => { + cluster_head.observe_simpleblock_timecode(block.timecode); + encode_webm_element(&WebmElement::SimpleBlock(*block), buffer); + continue; + }, + Ok(Async::Ready(Some(WebmElement::Info))) => continue, + Ok(Async::Ready(Some(WebmElement::Void))) => continue, + Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => continue, + Ok(Async::Ready(Some(element @ _))) => { + encode_webm_element(&element, buffer); + continue; + }, + Ok(Async::Ready(None)) => { + // flush final Cluster on end of stream + let cluster_head_chunk = mem::replace(cluster_head, Chunk::>::new_cluster_head(0)); + let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); + ( + Ok(Async::Ready(Some(cluster_head_chunk))), + ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner()) + ) + } + } }, ChunkerState::EmittingClusterBody(ref mut buffer) => { - return Ok(Async::Ready(None)); - } + let liberated_buffer = mem::replace(buffer, Vec::new()); + ( + Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)}))), + ChunkerState::BuildingCluster( + Chunk::>::new_cluster_head(0), + Cursor::new(Vec::new()) + ) + ) + }, + ChunkerState::EmittingFinalClusterBody(ref mut buffer) => { + // flush final Cluster on end of stream + let liberated_buffer = mem::replace(buffer, Vec::new()); + ( + Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)}))), + ChunkerState::End + ) + }, + ChunkerState::End => return Ok(Async::Ready(None)) }; self.state = next_state; From b0b4f204f2e78f861923de34ceb183441224e059 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 02:03:19 -0400 Subject: [PATCH 081/164] strip out extra header chunks in timecode_fixer() --- src/timecode_fixer.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index 5f380a9..f701fd2 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -49,6 +49,7 @@ pub struct ChunkTimecodeFixer { current_offset: u64, last_observed_timecode: u64, assumed_duration: u64, + seen_header: bool } impl> Stream for ChunkTimecodeFixer @@ -70,6 +71,13 @@ impl> Stream for ChunkTimecodeFixer } self.last_observed_timecode = end + self.current_offset; }, + Ok(Async::Ready(Some(Chunk::Headers {..}))) => { + if self.seen_header { + return self.poll(); + } else { + self.seen_header = true; + } + }, _ => {} }; poll_chunk @@ -86,7 +94,8 @@ impl> ChunkStream for T { stream: self, current_offset: 0, last_observed_timecode: 0, - assumed_duration: 33 + assumed_duration: 33, + seen_header: false } } } From e0346ae30a1f412b9ebac78afe8bab91aa912b06 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 02:10:58 -0400 Subject: [PATCH 082/164] leave mark-keyframes reminder in webm chunker --- src/chunk.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/chunk.rs b/src/chunk.rs index b8ce3f2..f85c1f0 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -133,6 +133,10 @@ impl<'a, S: Stream>> Stream for WebmChunker continue; }, Ok(Async::Ready(Some(WebmElement::SimpleBlock(ref block)))) => { + if (block.flags & 0b10000000) != 0 { + // TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster + cluster_head.mark_keyframe(true); + } cluster_head.observe_simpleblock_timecode(block.timecode); encode_webm_element(&WebmElement::SimpleBlock(*block), buffer); continue; From 0902a810e525e9541404f295eaca408a40e3dd81 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 02:11:19 -0400 Subject: [PATCH 083/164] Use webm chunker operator in loop_server --- src/bin/loop_server.rs | 78 +++++------------------------------------- 1 file changed, 8 insertions(+), 70 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 0967182..23a90f2 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,24 +3,21 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; -use futures::stream::{once, iter, Stream}; -use lab_ebml::chunk::Chunk; +use futures::stream::{iter, Stream}; +use lab_ebml::chunk::{Chunk, WebmStream}; use lab_ebml::Schema; use lab_ebml::timecode_fixer::ChunkStream; use lab_ebml::webm::*; -use lab_ebml::webm::WebmElement::*; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; use std::env::args; -use std::io::Cursor; use std::net::ToSocketAddrs; -use std::sync::Arc; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); #[derive(Clone)] -struct WebmServer(Chunk, Vec); +struct WebmServer; type BodyStream = Box, Error = hyper::Error>>; @@ -32,13 +29,11 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let results: Vec> = self.1.iter().map(|x| Ok(x.clone())).collect(); - let stream: BodyStream> = Box::new( - once(Ok(self.0.clone())) - .chain(iter(results.into_iter().cycle().take(20))) - .map_err(|_| hyper::Error::Incomplete) + let stream: BodyStream> = iter(Webm.parse(SRC_FILE).into_iter().map(|x| Ok(x))) + .chunk_webm() + .chain(iter(Webm.parse(SRC_FILE).into_iter().map(|x| Ok(x))).chunk_webm()) .fix_timecodes() - ); + .boxed(); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) .with_body(stream) @@ -54,62 +49,5 @@ impl Service for WebmServer { pub fn main() { let addr = args().nth(1).unwrap().to_socket_addrs().unwrap().next().unwrap(); - let webm_service = create_loop(); - Http::new().bind(&addr, move || Ok(webm_service.clone())).unwrap().run().unwrap(); -} - -fn create_loop() -> WebmServer { - let mut header = None; - let mut reading_head = true; - - let mut cluster_header = None; - let mut cluster_timecode = 0; - let mut chunks = Vec::new(); - - let mut buffer = Cursor::new(Vec::new()); - - for element in Webm.parse(SRC_FILE) { - match element { - Cluster => { - if reading_head { - header = Some(Chunk::Headers {bytes: Arc::new(buffer.into_inner())}); - } else { - if let Some(chunk) = cluster_header.take() { - chunks.push(chunk); - } - chunks.push(Chunk::ClusterBody {bytes: Arc::new(buffer.into_inner())}); - } - buffer = Cursor::new(Vec::new()); - reading_head = false; - }, - Timecode(timecode) => { - cluster_timecode = timecode; - cluster_header = Some(Chunk::>::new_cluster_head(timecode)); - }, - SimpleBlock(ref block) => { - if let Some(ref mut chunk) = cluster_header { - if (block.flags & 0b10000000) != 0 { - // TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster - chunk.mark_keyframe(true); - } - chunk.observe_simpleblock_timecode(block.timecode); - } - encode_webm_element(&SimpleBlock(*block), &mut buffer).unwrap(); - }, - Info => continue, - Void => continue, - Unknown(_) => continue, - ref other => { - encode_webm_element(other, &mut buffer).unwrap(); - }, - } - } - - // finish last cluster - if let Some(chunk) = cluster_header.take() { - chunks.push(chunk); - } - chunks.push(Chunk::ClusterBody {bytes: Arc::new(buffer.into_inner())}); - - WebmServer(header.unwrap(), chunks) + Http::new().bind(&addr, move || Ok(WebmServer)).unwrap().run().unwrap(); } From 59a179f9e19f8def5dd6b1db09aa440487959f91 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Oct 2017 02:12:25 -0400 Subject: [PATCH 084/164] update notes --- notes.md | 1 - 1 file changed, 1 deletion(-) diff --git a/notes.md b/notes.md index 0ba443a..626e18d 100644 --- a/notes.md +++ b/notes.md @@ -1,4 +1,3 @@ * support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) * rustfmt modules -* create chunks from webm event stream * bytestream source for ebml parsing From 972a88c35bde5ee15b359d01a4b057b7755b356f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 6 Oct 2017 00:17:18 -0400 Subject: [PATCH 085/164] Handle errors in chunking code in some fashion --- src/bin/loop_server.rs | 6 +++++- src/chunk.rs | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 23a90f2..2d9a27e 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -4,7 +4,7 @@ extern crate lab_ebml; use futures::future::FutureResult; use futures::stream::{iter, Stream}; -use lab_ebml::chunk::{Chunk, WebmStream}; +use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; use lab_ebml::Schema; use lab_ebml::timecode_fixer::ChunkStream; use lab_ebml::webm::*; @@ -33,6 +33,10 @@ impl Service for WebmServer { .chunk_webm() .chain(iter(Webm.parse(SRC_FILE).into_iter().map(|x| Ok(x))).chunk_webm()) .fix_timecodes() + .map_err(|err| match err { + ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), + ChunkingError::OtherError(otx_err) => otx_err + }) .boxed(); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) diff --git a/src/chunk.rs b/src/chunk.rs index f85c1f0..f42c925 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -81,6 +81,11 @@ enum ChunkerState { End } +pub enum ChunkingError { + IoError(::std::io::Error), + OtherError(E) +} + pub struct WebmChunker { stream: S, state: ChunkerState @@ -89,14 +94,14 @@ pub struct WebmChunker { impl<'a, S: Stream>> Stream for WebmChunker { type Item = Chunk; - type Error = S::Error; + type Error = ChunkingError; fn poll(&mut self) -> Result>, Self::Error> { loop { let (return_value, next_state) = match self.state { ChunkerState::BuildingHeader(ref mut buffer) => { match self.stream.poll() { - Err(passthru) => return Err(passthru), + Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), Ok(Async::Ready(Some(WebmElement::Cluster))) => { @@ -111,14 +116,19 @@ impl<'a, S: Stream>> Stream for WebmChunker ) }, Ok(Async::Ready(Some(element @ _))) => { - encode_webm_element(&element, buffer); - continue; + match encode_webm_element(&element, buffer) { + Ok(_) => continue, + Err(err) => ( + Err(ChunkingError::IoError(err)), + ChunkerState::End + ) + } } } }, ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => { match self.stream.poll() { - Err(passthru) => return Err(passthru), + Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(Some(WebmElement::Cluster))) => { let cluster_head_chunk = mem::replace(cluster_head, Chunk::>::new_cluster_head(0)); @@ -138,15 +148,25 @@ impl<'a, S: Stream>> Stream for WebmChunker cluster_head.mark_keyframe(true); } cluster_head.observe_simpleblock_timecode(block.timecode); - encode_webm_element(&WebmElement::SimpleBlock(*block), buffer); - continue; + match encode_webm_element(&WebmElement::SimpleBlock(*block), buffer) { + Ok(_) => continue, + Err(err) => ( + Err(ChunkingError::IoError(err)), + ChunkerState::End + ) + } }, Ok(Async::Ready(Some(WebmElement::Info))) => continue, Ok(Async::Ready(Some(WebmElement::Void))) => continue, Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => continue, Ok(Async::Ready(Some(element @ _))) => { - encode_webm_element(&element, buffer); - continue; + match encode_webm_element(&element, buffer) { + Ok(_) => continue, + Err(err) => ( + Err(ChunkingError::IoError(err)), + ChunkerState::End + ) + } }, Ok(Async::Ready(None)) => { // flush final Cluster on end of stream From cdcff869aae691b809a2abf80a301e36db77db5b Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 25 Mar 2018 21:33:38 -0400 Subject: [PATCH 086/164] Replace Schema types with a FromEbml trait on the Element type simplify lifetimes --- src/bin/dump.rs | 6 ++---- src/bin/loop_server.rs | 5 ++--- src/bin/resynth.rs | 3 +-- src/ebml.rs | 39 ++++++++++++++++++++------------------- src/iterator.rs | 27 ++++++++++++++------------- src/lib.rs | 2 +- src/webm.rs | 15 ++++++++------- 7 files changed, 48 insertions(+), 49 deletions(-) diff --git a/src/bin/dump.rs b/src/bin/dump.rs index 8e5d333..096b42d 100644 --- a/src/bin/dump.rs +++ b/src/bin/dump.rs @@ -4,9 +4,7 @@ use std::env::args; use std::fs::File; use std::io::Read; use std::path::Path; -use lab_ebml::Schema; -use lab_ebml::webm::SimpleBlock; -use lab_ebml::webm::Webm; +use lab_ebml::webm::{ parse_webm, SimpleBlock }; use lab_ebml::webm::WebmElement::*; pub fn main() { @@ -19,7 +17,7 @@ pub fn main() { file.read_to_end(&mut buffer).expect("Reading file contents"); - for element in Webm.parse(buffer.as_slice()) { + for element in parse_webm(buffer.as_slice()) { match element { // suppress printing byte arrays Tracks(slice) => println!("Tracks[{}]", slice.len()), diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 2d9a27e..d2f18e8 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -5,7 +5,6 @@ extern crate lab_ebml; use futures::future::FutureResult; use futures::stream::{iter, Stream}; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; -use lab_ebml::Schema; use lab_ebml::timecode_fixer::ChunkStream; use lab_ebml::webm::*; use hyper::{Get, StatusCode}; @@ -29,9 +28,9 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream> = iter(Webm.parse(SRC_FILE).into_iter().map(|x| Ok(x))) + let stream: BodyStream> = iter(parse_webm(SRC_FILE).into_iter().map(|x| Ok(x))) .chunk_webm() - .chain(iter(Webm.parse(SRC_FILE).into_iter().map(|x| Ok(x))).chunk_webm()) + .chain(iter(parse_webm(SRC_FILE).into_iter().map(|x| Ok(x))).chunk_webm()) .fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index acaef39..fe71fe5 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -1,7 +1,6 @@ extern crate lab_ebml; use std::io::{Cursor, stdout, Write}; -use lab_ebml::Schema; use lab_ebml::webm::*; use lab_ebml::webm::WebmElement::*; use lab_ebml::timecode_fixer::TimecodeFixer; @@ -15,7 +14,7 @@ pub fn main() { let mut reading_head = true; - for element in Webm.parse(SRC_FILE) { + for element in parse_webm(SRC_FILE) { match element { Cluster => reading_head = false, // TODO: skip elements not required for streaming diff --git a/src/ebml.rs b/src/ebml.rs index 8a5ab78..e0930e1 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -2,6 +2,7 @@ use bytes::{BigEndian, ByteOrder, BufMut}; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; +use std::marker::PhantomData; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const DOC_TYPE_ID: u64 = 0x0282; @@ -214,20 +215,21 @@ pub fn encode_integer(tag: u64, value: u64, output: &mut T) -> IoResul } #[derive(Debug, PartialEq)] -pub struct Ebml(pub S, pub T); +pub struct Ebml { + pub source: Source, + _marker: PhantomData Element> +} -pub trait Schema<'a> { - type Element: 'a; +pub trait FromEbml<'b>: Sized { + fn should_unwrap(element_id: u64) -> bool; + fn decode(element_id: u64, bytes: &'b[u8]) -> Result; - fn should_unwrap(&self, element_id: u64) -> bool; - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result; - - fn decode_element<'b: 'a>(&self, bytes: &'b[u8]) -> Result, Error> { + fn decode_element(bytes: &'b[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), Ok(Some((element_id, payload_size_tag, tag_size))) => { - let should_unwrap = self.should_unwrap(element_id); + let should_unwrap = Self::should_unwrap(element_id); let payload_size = match (should_unwrap, payload_size_tag) { (true, _) => 0, @@ -241,7 +243,7 @@ pub trait Schema<'a> { return Ok(None); } - match self.decode(element_id, &bytes[tag_size..element_size]) { + match Self::decode(element_id, &bytes[tag_size..element_size]) { Ok(element) => Ok(Some((element, element_size))), Err(error) => Err(error) } @@ -249,8 +251,11 @@ pub trait Schema<'a> { } } - fn parse(self, source: T) -> Ebml where Self: Sized { - Ebml(self, source) + fn parse(source: T) -> Ebml { + Ebml { + source: source, + _marker: PhantomData + } } } @@ -398,21 +403,17 @@ mod tests { assert_eq!(decode_uint(&[0x80,0,0,0,0,0,0,1]), Ok(9223372036854775809)); } - struct Dummy; - #[derive(Debug, PartialEq)] struct GenericElement(u64, usize); - impl<'a> Schema<'a> for Dummy { - type Element = GenericElement; - - fn should_unwrap(&self, element_id: u64) -> bool { + impl<'a> FromEbml<'a> for GenericElement { + fn should_unwrap(element_id: u64) -> bool { match element_id { _ => false } } - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result { + fn decode(element_id: u64, bytes: &'a[u8]) -> Result { match element_id { _ => Ok(GenericElement(element_id, bytes.len())) } @@ -421,7 +422,7 @@ mod tests { #[test] fn decode_sanity_test() { - let decoded = Dummy.decode_element(TEST_FILE); + let decoded = GenericElement::decode_element(TEST_FILE); assert_eq!(decoded, Ok(Some((GenericElement(0x0A45DFA3, 31), 43)))); } } diff --git a/src/iterator.rs b/src/iterator.rs index 4283252..de7c961 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,30 +1,31 @@ +use std::marker::PhantomData; use ebml::*; -pub struct EbmlIterator<'b, T: Schema<'b>> { - schema: T, +pub struct EbmlIterator<'b, T: FromEbml<'b>> { slice: &'b[u8], position: usize, + _marker: PhantomData T> } -impl<'b, S: Schema<'b>> IntoIterator for Ebml { - type Item = S::Element; - type IntoIter = EbmlIterator<'b, S>; +impl<'b, E: FromEbml<'b>> IntoIterator for Ebml<&'b[u8], E> { + type Item = E; + type IntoIter = EbmlIterator<'b, E>; - fn into_iter(self) -> EbmlIterator<'b, S> + fn into_iter(self) -> EbmlIterator<'b, E> { EbmlIterator { - schema: self.0, - slice: self.1, - position: 0 + slice: self.source, + position: 0, + _marker: PhantomData } } } -impl<'b, T: Schema<'b>> Iterator for EbmlIterator<'b, T> { - type Item = T::Element; +impl<'b, T: FromEbml<'b>> Iterator for EbmlIterator<'b, T> { + type Item = T; - fn next(&mut self) -> Option { - match self.schema.decode_element(&self.slice[self.position..]) { + fn next(&mut self) -> Option { + match Self::Item::decode_element(&self.slice[self.position..]) { Err(_) => None, Ok(None) => None, Ok(Some((element, element_size))) => { diff --git a/src/lib.rs b/src/lib.rs index 4f689c5..635ee82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,7 @@ mod iterator; pub mod timecode_fixer; pub mod webm; -pub use ebml::{Error, Schema}; +pub use ebml::{Error, FromEbml}; #[cfg(test)] mod tests { diff --git a/src/webm.rs b/src/webm.rs index 3123977..39410ba 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -10,7 +10,10 @@ const TRACKS_ID: u64 = 0x0654AE6B; const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub struct Webm; + +pub fn parse_webm<'a, T: 'a>(source: T) -> Ebml> { + WebmElement::parse(source) +} #[derive(Debug, PartialEq, Copy, Clone)] pub struct SimpleBlock<'b> { @@ -35,10 +38,8 @@ pub enum WebmElement<'b> { Unknown(u64) } -impl<'a> Schema<'a> for Webm { - type Element = WebmElement<'a>; - - fn should_unwrap(&self, element_id: u64) -> bool { +impl<'b> FromEbml<'b> for WebmElement<'b> { + fn should_unwrap(element_id: u64) -> bool { match element_id { // Segment SEGMENT_ID => true, @@ -47,7 +48,7 @@ impl<'a> Schema<'a> for Webm { } } - fn decode<'b: 'a>(&self, element_id: u64, bytes: &'b[u8]) -> Result, Error> { + fn decode(element_id: u64, bytes: &'b[u8]) -> Result, Error> { match element_id { EBML_HEAD_ID => Ok(WebmElement::EbmlHead), VOID_ID => Ok(WebmElement::Void), @@ -131,7 +132,7 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = Webm.parse(TEST_FILE).into_iter(); + let mut iter = parse_webm(TEST_FILE).into_iter(); // test that we match the structure of the test file assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); From f8db95e61ee649c4a8afae3b80b0d968dc730129 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 28 Mar 2018 00:31:58 -0400 Subject: [PATCH 087/164] Make ClusterHead a first-class struct, simplifying a lot of enum wrangling --- src/chunk.rs | 99 +++++++++++++++++++++---------------------- src/timecode_fixer.rs | 9 ++-- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index f42c925..9511b88 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -5,60 +5,59 @@ use std::sync::Arc; use webm::*; #[derive(Clone)] -pub enum Chunk = Vec> { - Headers { - bytes: Arc - }, - ClusterHead { - keyframe: bool, - start: u64, - end: u64, - // space for a Cluster tag and a Timecode tag - bytes: [u8;16], - bytes_used: u8 - }, - ClusterBody { - bytes: Arc - } +pub struct ClusterHead { + pub keyframe: bool, + pub start: u64, + pub end: u64, + // space for a Cluster tag and a Timecode tag + bytes: [u8;16], + bytes_used: u8 } -impl> Chunk { - pub fn new_cluster_head(timecode: u64) -> Chunk { - let mut chunk = Chunk::ClusterHead { +impl ClusterHead { + pub fn new(timecode: u64) -> ClusterHead { + let mut cluster_head = ClusterHead { keyframe: false, start: 0, end: 0, bytes: [0;16], bytes_used: 0 }; - chunk.update_timecode(timecode); - chunk + cluster_head.update_timecode(timecode); + cluster_head } - pub fn update_timecode(&mut self, timecode: u64) { - if let &mut Chunk::ClusterHead {ref mut start, ref mut end, ref mut bytes, ref mut bytes_used, ..} = self { - let delta = *end - *start; - *start = timecode; - *end = *start + delta; - let mut cursor = Cursor::new(bytes as &mut [u8]); - // buffer is sized so these should never fail - encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Timecode(timecode), &mut cursor).unwrap(); - *bytes_used = cursor.position() as u8; - } + let delta = self.end - self.start; + self.start = timecode; + self.end = self.start + delta; + let mut cursor = Cursor::new(self.bytes.as_mut()); + // buffer is sized so these should never fail + encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(&WebmElement::Timecode(timecode), &mut cursor).unwrap(); + self.bytes_used = cursor.position() as u8; } pub fn observe_simpleblock_timecode(&mut self, timecode: i16) { - if let &mut Chunk::ClusterHead {start, ref mut end, ..} = self { - let absolute_timecode = start + (timecode as u64); - if absolute_timecode > start { - *end = absolute_timecode; - } + let absolute_timecode = self.start + (timecode as u64); + if absolute_timecode > self.start { + self.end = absolute_timecode; } } - pub fn mark_keyframe(&mut self, new_keyframe: bool) { - if let &mut Chunk::ClusterHead {ref mut keyframe, ..} = self { - *keyframe = new_keyframe; - } +} + +impl AsRef<[u8]> for ClusterHead { + fn as_ref(&self) -> &[u8] { + self.bytes[..self.bytes_used as usize].as_ref() + } +} + +#[derive(Clone)] +pub enum Chunk = Vec> { + Headers { + bytes: Arc + }, + ClusterHead(ClusterHead), + ClusterBody { + bytes: Arc } } @@ -66,7 +65,7 @@ impl> AsRef<[u8]> for Chunk { fn as_ref(&self) -> &[u8] { match self { &Chunk::Headers {ref bytes, ..} => bytes.as_ref().as_ref(), - &Chunk::ClusterHead {ref bytes, bytes_used, ..} => bytes[..bytes_used as usize].as_ref(), + &Chunk::ClusterHead(ref cluster_head) => cluster_head.as_ref(), &Chunk::ClusterBody {ref bytes, ..} => bytes.as_ref().as_ref() } } @@ -75,7 +74,7 @@ impl> AsRef<[u8]> for Chunk { enum ChunkerState { BuildingHeader(Cursor>), // WIP ClusterHead & body buffer - BuildingCluster(Chunk, Cursor>), + BuildingCluster(ClusterHead, Cursor>), EmittingClusterBody(Vec), EmittingFinalClusterBody(Vec), End @@ -110,7 +109,7 @@ impl<'a, S: Stream>> Stream for WebmChunker ( Ok(Async::Ready(Some(header_chunk))), ChunkerState::BuildingCluster( - Chunk::>::new_cluster_head(0), + ClusterHead::new(0), Cursor::new(Vec::new()) ) ) @@ -131,10 +130,10 @@ impl<'a, S: Stream>> Stream for WebmChunker Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(Some(WebmElement::Cluster))) => { - let cluster_head_chunk = mem::replace(cluster_head, Chunk::>::new_cluster_head(0)); + let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); ( - Ok(Async::Ready(Some(cluster_head_chunk))), + Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))), ChunkerState::EmittingClusterBody(liberated_buffer.into_inner()) ) }, @@ -145,7 +144,7 @@ impl<'a, S: Stream>> Stream for WebmChunker Ok(Async::Ready(Some(WebmElement::SimpleBlock(ref block)))) => { if (block.flags & 0b10000000) != 0 { // TODO: this is incorrect, condition needs to also affirm we're the first video block of the cluster - cluster_head.mark_keyframe(true); + cluster_head.keyframe = true; } cluster_head.observe_simpleblock_timecode(block.timecode); match encode_webm_element(&WebmElement::SimpleBlock(*block), buffer) { @@ -170,10 +169,10 @@ impl<'a, S: Stream>> Stream for WebmChunker }, Ok(Async::Ready(None)) => { // flush final Cluster on end of stream - let cluster_head_chunk = mem::replace(cluster_head, Chunk::>::new_cluster_head(0)); + let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); ( - Ok(Async::Ready(Some(cluster_head_chunk))), + Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))), ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner()) ) } @@ -184,7 +183,7 @@ impl<'a, S: Stream>> Stream for WebmChunker ( Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)}))), ChunkerState::BuildingCluster( - Chunk::>::new_cluster_head(0), + ClusterHead::new(0), Cursor::new(Vec::new()) ) ) @@ -226,6 +225,6 @@ mod tests { #[test] fn enough_space_for_header() { - Chunk::>::new_cluster_head(u64::max_value()); + ClusterHead::new(u64::max_value()); } } diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index f701fd2..e8b6df4 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -60,16 +60,15 @@ impl> Stream for ChunkTimecodeFixer fn poll(&mut self) -> Result>, Self::Error> { let mut poll_chunk = self.stream.poll(); match poll_chunk { - Ok(Async::Ready(Some(Chunk::ClusterHead {start, end, ..}))) => { + Ok(Async::Ready(Some(Chunk::ClusterHead(ref mut cluster_head)))) => { + let start = cluster_head.start; if start < self.last_observed_timecode { let next_timecode = self.last_observed_timecode + self.assumed_duration; self.current_offset = next_timecode - start; } - if let Ok(Async::Ready(Some(ref mut cluster_head))) = poll_chunk { - cluster_head.update_timecode(start + self.current_offset); - } - self.last_observed_timecode = end + self.current_offset; + cluster_head.update_timecode(start + self.current_offset); + self.last_observed_timecode = cluster_head.end + self.current_offset; }, Ok(Async::Ready(Some(Chunk::Headers {..}))) => { if self.seen_header { From 417cbf49c79fc99f2c69e2bacb7f79080295cc95 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 01:33:13 -0400 Subject: [PATCH 088/164] Create EbmlEventSource trait since Iterators/Streams can't return borrows --- src/bin/loop_server.rs | 11 +++++------ src/chunk.rs | 28 ++++++++++++++++------------ src/ebml.rs | 7 +++++++ src/iterator.rs | 17 +++++++++++++++++ 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index d2f18e8..34031a7 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,7 +3,7 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; -use futures::stream::{iter, Stream}; +use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; use lab_ebml::timecode_fixer::ChunkStream; use lab_ebml::webm::*; @@ -28,13 +28,12 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream> = iter(parse_webm(SRC_FILE).into_iter().map(|x| Ok(x))) - .chunk_webm() - .chain(iter(parse_webm(SRC_FILE).into_iter().map(|x| Ok(x))).chunk_webm()) + let stream: BodyStream> = parse_webm(SRC_FILE).into_iter().chunk_webm() + .chain(parse_webm(SRC_FILE).into_iter().chunk_webm()) .fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), - ChunkingError::OtherError(otx_err) => otx_err + ChunkingError::OtherError(_) => hyper::Error::Incomplete }) .boxed(); Response::new() @@ -51,6 +50,6 @@ impl Service for WebmServer { } pub fn main() { - let addr = args().nth(1).unwrap().to_socket_addrs().unwrap().next().unwrap(); + let addr = args().nth(1).expect("Need binding address+port").to_socket_addrs().unwrap().next().unwrap(); Http::new().bind(&addr, move || Ok(WebmServer)).unwrap().run().unwrap(); } diff --git a/src/chunk.rs b/src/chunk.rs index 9511b88..5541c6c 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,7 +1,9 @@ use futures::{Async, Stream}; use std::io::Cursor; +use std::marker::PhantomData; use std::mem; use std::sync::Arc; +use ebml::EbmlEventSource; use webm::*; #[derive(Clone)] @@ -85,12 +87,13 @@ pub enum ChunkingError { OtherError(E) } -pub struct WebmChunker { - stream: S, - state: ChunkerState +pub struct WebmChunker<'a, S: EbmlEventSource<'a>> { + source: S, + state: ChunkerState, + _marker: PhantomData<&'a [u8]> } -impl<'a, S: Stream>> Stream for WebmChunker +impl<'a, S: EbmlEventSource<'a, Event = WebmElement<'a>>> Stream for WebmChunker<'a, S> { type Item = Chunk; type Error = ChunkingError; @@ -99,7 +102,7 @@ impl<'a, S: Stream>> Stream for WebmChunker loop { let (return_value, next_state) = match self.state { ChunkerState::BuildingHeader(ref mut buffer) => { - match self.stream.poll() { + match self.source.poll_event() { Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), @@ -126,7 +129,7 @@ impl<'a, S: Stream>> Stream for WebmChunker } }, ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => { - match self.stream.poll() { + match self.source.poll_event() { Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(Some(WebmElement::Cluster))) => { @@ -205,15 +208,16 @@ impl<'a, S: Stream>> Stream for WebmChunker } } -pub trait WebmStream { - fn chunk_webm(self) -> WebmChunker; +pub trait WebmStream<'a, T: EbmlEventSource<'a, Event = WebmElement<'a>>> { + fn chunk_webm(self) -> WebmChunker<'a, T>; } -impl<'a, T: Stream>> WebmStream for T { - fn chunk_webm(self) -> WebmChunker { +impl<'a, T: EbmlEventSource<'a, Event = WebmElement<'a>>> WebmStream<'a, T> for T { + fn chunk_webm(self) -> WebmChunker<'a, T> { WebmChunker { - stream: self, - state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())) + source: self, + state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())), + _marker: PhantomData } } } diff --git a/src/ebml.rs b/src/ebml.rs index e0930e1..a4ff6fe 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,4 +1,5 @@ use bytes::{BigEndian, ByteOrder, BufMut}; +use futures::Async; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; @@ -259,6 +260,12 @@ pub trait FromEbml<'b>: Sized { } } +pub trait EbmlEventSource<'a> { + type Event: FromEbml<'a>; + type Error; + fn poll_event(&mut self) -> Result>, Self::Error>; +} + #[cfg(test)] mod tests { use bytes::{BytesMut}; diff --git a/src/iterator.rs b/src/iterator.rs index de7c961..7cdd00d 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,4 +1,5 @@ use std::marker::PhantomData; +use futures::Async; use ebml::*; pub struct EbmlIterator<'b, T: FromEbml<'b>> { @@ -35,3 +36,19 @@ impl<'b, T: FromEbml<'b>> Iterator for EbmlIterator<'b, T> { } } } + +impl<'a, T: FromEbml<'a>> EbmlEventSource<'a> for EbmlIterator<'a, T> { + type Event = T; + type Error = Error; + + fn poll_event(&mut self) -> Result>, Error> { + match Self::Event::decode_element(&self.slice[self.position..]) { + Err(err) => Err(err), + Ok(None) => Ok(Async::Ready(None)), + Ok(Some((element, element_size))) => { + self.position += element_size; + Ok(Async::Ready(Some(element))) + } + } + } +} From a8fa2792013c85cd5f63e960b1f966fa3bbbf6f2 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 01:43:44 -0400 Subject: [PATCH 089/164] bump dependency versions --- Cargo.toml | 4 ++-- src/bin/loop_server.rs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2f62d86..280bd11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,5 +5,5 @@ authors = ["Tangent 128 "] [dependencies] bytes = "0.4" -futures = "0.1.14" -hyper = "0.11.2" +futures = "0.1.20" +hyper = "0.11.24" diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 34031a7..ffc1808 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -28,14 +28,15 @@ impl Service for WebmServer { fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream> = parse_webm(SRC_FILE).into_iter().chunk_webm() + let stream: BodyStream> = Box::new( + parse_webm(SRC_FILE).into_iter().chunk_webm() .chain(parse_webm(SRC_FILE).into_iter().chunk_webm()) .fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), ChunkingError::OtherError(_) => hyper::Error::Incomplete }) - .boxed(); + ); Response::new() .with_header(ContentType("video/webm".parse().unwrap())) .with_body(stream) From 86d047e217d756e0a08be3566f03565ee84b529c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 02:26:35 -0400 Subject: [PATCH 090/164] Debug derives --- src/chunk.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chunk.rs b/src/chunk.rs index 5541c6c..4aeebba 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use ebml::EbmlEventSource; use webm::*; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ClusterHead { pub keyframe: bool, pub start: u64, @@ -73,6 +73,7 @@ impl> AsRef<[u8]> for Chunk { } } +#[derive(Debug)] enum ChunkerState { BuildingHeader(Cursor>), // WIP ClusterHead & body buffer @@ -82,6 +83,7 @@ enum ChunkerState { End } +#[derive(Debug)] pub enum ChunkingError { IoError(::std::io::Error), OtherError(E) From d9e197ec7d621852256ddbd9d1ee95c67f8e8ff5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 02:26:52 -0400 Subject: [PATCH 091/164] Skip Info/Void in header for now --- src/chunk.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/chunk.rs b/src/chunk.rs index 4aeebba..af1b241 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -119,6 +119,8 @@ impl<'a, S: EbmlEventSource<'a, Event = WebmElement<'a>>> Stream for WebmChunker ) ) }, + Ok(Async::Ready(Some(WebmElement::Info))) => continue, + Ok(Async::Ready(Some(WebmElement::Void))) => continue, Ok(Async::Ready(Some(element @ _))) => { match encode_webm_element(&element, buffer) { Ok(_) => continue, From 00fc1fcb2a6b857563b8b46f7a1eff12d8e0b54a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 02:44:42 -0400 Subject: [PATCH 092/164] Correct the chunk-based timecode fixer --- src/timecode_fixer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index e8b6df4..07d1047 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -68,7 +68,7 @@ impl> Stream for ChunkTimecodeFixer } cluster_head.update_timecode(start + self.current_offset); - self.last_observed_timecode = cluster_head.end + self.current_offset; + self.last_observed_timecode = cluster_head.end; }, Ok(Async::Ready(Some(Chunk::Headers {..}))) => { if self.seen_header { From f385371bb952574ecdd36a30ef32c9b19a1f0660 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 30 Mar 2018 02:54:02 -0400 Subject: [PATCH 093/164] Make loop server iteration count more flexible than a chain --- src/bin/loop_server.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index ffc1808..74fadf2 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,6 +3,7 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; +use futures::stream::repeat; use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; use lab_ebml::timecode_fixer::ChunkStream; @@ -29,8 +30,10 @@ impl Service for WebmServer { let response = match (req.method(), req.path()) { (&Get, "/loop") => { let stream: BodyStream> = Box::new( - parse_webm(SRC_FILE).into_iter().chunk_webm() - .chain(parse_webm(SRC_FILE).into_iter().chunk_webm()) + repeat(()).take(10) + .map(|()| + parse_webm(SRC_FILE).into_iter().chunk_webm() + ).flatten() .fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), From 495ca0f55544415d19d18162146b58386fc69e9d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 31 Mar 2018 19:36:57 -0400 Subject: [PATCH 094/164] Stream buffer/parser experiment, lifetime issues are snarling though --- src/lib.rs | 1 + src/webm_stream.rs | 93 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 src/webm_stream.rs diff --git a/src/lib.rs b/src/lib.rs index 635ee82..a3a7abe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ extern crate futures; pub mod chunk; pub mod ebml; mod iterator; +pub mod webm_stream; pub mod timecode_fixer; pub mod webm; diff --git a/src/webm_stream.rs b/src/webm_stream.rs new file mode 100644 index 0000000..5e0996d --- /dev/null +++ b/src/webm_stream.rs @@ -0,0 +1,93 @@ +use bytes::BytesMut; +use bytes::BufMut; +use futures::Async; +use futures::stream::Stream; + +use ebml::*; +use webm::*; + +pub enum ParsingError { + EbmlError(::ebml::Error), + OtherError(E) +} + +pub struct WebmStream { + stream: S, + buffer: BytesMut, + last_read: usize +} + +impl, S: Stream> WebmStream { + pub fn new(stream: S) -> Self { + WebmStream { + stream: stream, + buffer: BytesMut::new(), + last_read: 0 + } + } + + pub fn try_decode(&mut self) -> Result>, ParsingError> { + match WebmElement::decode_element(&self.buffer) { + Err(err) => return Err(ParsingError::EbmlError(err)), + Ok(None) => { + // need to refill buffer + return Ok(Async::NotReady); + }, + Ok(Some((element, element_size))) => { + self.last_read += element_size; + return Ok(Async::Ready(Some(element))) + } + }; + } + + pub fn can_decode(&mut self) -> bool { + match self.try_decode() { + Ok(Async::NotReady) => false, + _ => true + } + } + + pub fn poll_event2<'a>(&'a mut self) -> Result>>, ParsingError> { + // release buffer from previous event + self.buffer.advance(self.last_read); + self.last_read = 0; + + loop { + if self.can_decode() { + return self.try_decode() + } + + match self.stream.poll() { + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), + Ok(Async::Ready(Some(chunk))) => { + self.buffer.reserve(chunk.as_ref().len()); + self.buffer.put_slice(chunk.as_ref()); + // ok can retry decoding now + } + Err(err) => return Err(ParsingError::OtherError(err)) + }; + } + } +} + +/*fn umm<'a, I: AsRef<[u8]>, S: Stream>(webm_stream: &'a mut WebmStream) + -> Result>>, ParsingError> +{ + return webmStream.poll_event(); +}*/ + +/*impl<'a, I: AsRef<[u8]>, S: Stream> EbmlEventSource<'a> for WebmStream { + type Event = WebmElement<'a>; + type Error = ParsingError; + + fn poll_event(&'a mut self) -> Result>>, Self::Error> { + return self.poll_event2(); + } +}*/ + +#[cfg(test)] +mod tests { + //#[test] + +} From 11bb7f44123a457282264067fbc155d9c822f65a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 13:55:13 -0400 Subject: [PATCH 095/164] Fix EBML Source lifetimes, breaking the chunker --- src/ebml.rs | 4 ++-- src/iterator.rs | 2 +- src/webm_stream.rs | 16 +++++----------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index a4ff6fe..fee53ab 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -225,7 +225,7 @@ pub trait FromEbml<'b>: Sized { fn should_unwrap(element_id: u64) -> bool; fn decode(element_id: u64, bytes: &'b[u8]) -> Result; - fn decode_element(bytes: &'b[u8]) -> Result, Error> { + fn decode_element<'a: 'b>(bytes: &'a[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), @@ -263,7 +263,7 @@ pub trait FromEbml<'b>: Sized { pub trait EbmlEventSource<'a> { type Event: FromEbml<'a>; type Error; - fn poll_event(&mut self) -> Result>, Self::Error>; + fn poll_event(&'a mut self) -> Result>, Self::Error>; } #[cfg(test)] diff --git a/src/iterator.rs b/src/iterator.rs index 7cdd00d..4b3868e 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -41,7 +41,7 @@ impl<'a, T: FromEbml<'a>> EbmlEventSource<'a> for EbmlIterator<'a, T> { type Event = T; type Error = Error; - fn poll_event(&mut self) -> Result>, Error> { + fn poll_event(&'a mut self) -> Result>, Error> { match Self::Event::decode_element(&self.slice[self.position..]) { Err(err) => Err(err), Ok(None) => Ok(Async::Ready(None)), diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 5e0996d..f23eb54 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -47,7 +47,7 @@ impl, S: Stream> WebmStream { } } - pub fn poll_event2<'a>(&'a mut self) -> Result>>, ParsingError> { + pub fn poll_event<'a>(&'a mut self) -> Result>>, ParsingError> { // release buffer from previous event self.buffer.advance(self.last_read); self.last_read = 0; @@ -71,20 +71,14 @@ impl, S: Stream> WebmStream { } } -/*fn umm<'a, I: AsRef<[u8]>, S: Stream>(webm_stream: &'a mut WebmStream) - -> Result>>, ParsingError> -{ - return webmStream.poll_event(); -}*/ - -/*impl<'a, I: AsRef<[u8]>, S: Stream> EbmlEventSource<'a> for WebmStream { +impl<'a, I: AsRef<[u8]>, S: Stream> EbmlEventSource<'a> for WebmStream { type Event = WebmElement<'a>; type Error = ParsingError; - fn poll_event(&'a mut self) -> Result>>, Self::Error> { - return self.poll_event2(); + fn poll_event(&'a mut self) -> Result>, Self::Error> { + return WebmStream::poll_event(&mut self); } -}*/ +} #[cfg(test)] mod tests { From 56a7284e32946d0fadd9aa71fd14a3eddbdac6f2 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 18:37:47 -0400 Subject: [PATCH 096/164] Focus on Webm case specifically to get working code can retry generic EBML later --- src/bin/loop_server.rs | 2 +- src/chunk.rs | 20 ++++++++------------ src/ebml.rs | 13 ++----------- src/iterator.rs | 39 ++++++++++++++++----------------------- src/webm.rs | 11 +++++++++-- src/webm_stream.rs | 7 +++---- 6 files changed, 39 insertions(+), 53 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 74fadf2..6c0f85d 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -32,7 +32,7 @@ impl Service for WebmServer { let stream: BodyStream> = Box::new( repeat(()).take(10) .map(|()| - parse_webm(SRC_FILE).into_iter().chunk_webm() + parse_webm(SRC_FILE).chunk_webm() ).flatten() .fix_timecodes() .map_err(|err| match err { diff --git a/src/chunk.rs b/src/chunk.rs index af1b241..3c10b09 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,9 +1,7 @@ use futures::{Async, Stream}; use std::io::Cursor; -use std::marker::PhantomData; use std::mem; use std::sync::Arc; -use ebml::EbmlEventSource; use webm::*; #[derive(Clone, Debug)] @@ -89,13 +87,12 @@ pub enum ChunkingError { OtherError(E) } -pub struct WebmChunker<'a, S: EbmlEventSource<'a>> { +pub struct WebmChunker { source: S, - state: ChunkerState, - _marker: PhantomData<&'a [u8]> + state: ChunkerState } -impl<'a, S: EbmlEventSource<'a, Event = WebmElement<'a>>> Stream for WebmChunker<'a, S> +impl<'a, S: WebmEventSource> Stream for WebmChunker { type Item = Chunk; type Error = ChunkingError; @@ -212,16 +209,15 @@ impl<'a, S: EbmlEventSource<'a, Event = WebmElement<'a>>> Stream for WebmChunker } } -pub trait WebmStream<'a, T: EbmlEventSource<'a, Event = WebmElement<'a>>> { - fn chunk_webm(self) -> WebmChunker<'a, T>; +pub trait WebmStream { + fn chunk_webm(self) -> WebmChunker; } -impl<'a, T: EbmlEventSource<'a, Event = WebmElement<'a>>> WebmStream<'a, T> for T { - fn chunk_webm(self) -> WebmChunker<'a, T> { +impl<'a, T: WebmEventSource> WebmStream for T { + fn chunk_webm(self) -> WebmChunker { WebmChunker { source: self, - state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())), - _marker: PhantomData + state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())) } } } diff --git a/src/ebml.rs b/src/ebml.rs index fee53ab..2ef62fe 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -3,7 +3,6 @@ use futures::Async; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; -use std::marker::PhantomData; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const DOC_TYPE_ID: u64 = 0x0282; @@ -216,9 +215,8 @@ pub fn encode_integer(tag: u64, value: u64, output: &mut T) -> IoResul } #[derive(Debug, PartialEq)] -pub struct Ebml { - pub source: Source, - _marker: PhantomData Element> +pub struct Ebml { + pub source: Source } pub trait FromEbml<'b>: Sized { @@ -251,13 +249,6 @@ pub trait FromEbml<'b>: Sized { } } } - - fn parse(source: T) -> Ebml { - Ebml { - source: source, - _marker: PhantomData - } - } } pub trait EbmlEventSource<'a> { diff --git a/src/iterator.rs b/src/iterator.rs index 4b3868e..0c1c477 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,32 +1,26 @@ -use std::marker::PhantomData; use futures::Async; use ebml::*; +use webm::*; -pub struct EbmlIterator<'b, T: FromEbml<'b>> { - slice: &'b[u8], - position: usize, - _marker: PhantomData T> +pub struct EbmlCursor { + source: T, + position: usize } -impl<'b, E: FromEbml<'b>> IntoIterator for Ebml<&'b[u8], E> { - type Item = E; - type IntoIter = EbmlIterator<'b, E>; - - fn into_iter(self) -> EbmlIterator<'b, E> - { - EbmlIterator { - slice: self.source, - position: 0, - _marker: PhantomData +impl EbmlCursor { + pub fn new(source: T) -> Self { + EbmlCursor { + source, + position: 0 } } } -impl<'b, T: FromEbml<'b>> Iterator for EbmlIterator<'b, T> { - type Item = T; +impl<'a> Iterator for EbmlCursor<&'a [u8]> { + type Item = WebmElement<'a>; - fn next(&mut self) -> Option { - match Self::Item::decode_element(&self.slice[self.position..]) { + fn next(&mut self) -> Option> { + match Self::Item::decode_element(&self.source[self.position..]) { Err(_) => None, Ok(None) => None, Ok(Some((element, element_size))) => { @@ -37,12 +31,11 @@ impl<'b, T: FromEbml<'b>> Iterator for EbmlIterator<'b, T> { } } -impl<'a, T: FromEbml<'a>> EbmlEventSource<'a> for EbmlIterator<'a, T> { - type Event = T; +impl<'b, T: AsRef<[u8]>> WebmEventSource for EbmlCursor { type Error = Error; - fn poll_event(&'a mut self) -> Result>, Error> { - match Self::Event::decode_element(&self.slice[self.position..]) { + fn poll_event<'a>(&'a mut self) -> Result>>, Error> { + match WebmElement::decode_element(&self.source.as_ref()[self.position..]) { Err(err) => Err(err), Ok(None) => Ok(Async::Ready(None)), Ok(Some((element, element_size))) => { diff --git a/src/webm.rs b/src/webm.rs index 39410ba..bce0a05 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,6 +1,8 @@ use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; use bytes::{BigEndian, BufMut, ByteOrder}; +use futures::Async; use ebml::*; +use iterator::EbmlCursor; const SEGMENT_ID: u64 = 0x08538067; const SEEK_HEAD_ID: u64 = 0x014D9B74; @@ -11,8 +13,8 @@ const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub fn parse_webm<'a, T: 'a>(source: T) -> Ebml> { - WebmElement::parse(source) +pub fn parse_webm>(source: T) -> EbmlCursor { + EbmlCursor::new(source) } #[derive(Debug, PartialEq, Copy, Clone)] @@ -125,6 +127,11 @@ pub fn encode_webm_element(element: &WebmElement, output: &mut } } +pub trait WebmEventSource { + type Error; + fn poll_event<'a>(&'a mut self) -> Result>>, Self::Error>; +} + #[cfg(test)] mod tests { use tests::TEST_FILE; diff --git a/src/webm_stream.rs b/src/webm_stream.rs index f23eb54..5f50fc0 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -71,12 +71,11 @@ impl, S: Stream> WebmStream { } } -impl<'a, I: AsRef<[u8]>, S: Stream> EbmlEventSource<'a> for WebmStream { - type Event = WebmElement<'a>; +impl, S: Stream> WebmEventSource for WebmStream { type Error = ParsingError; - fn poll_event(&'a mut self) -> Result>, Self::Error> { - return WebmStream::poll_event(&mut self); + fn poll_event<'a>(&'a mut self) -> Result>>, Self::Error> { + return WebmStream::poll_event(self); } } From 0e2e5ad48ad9c0f9a8902709fd2370ba08e1ac22 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 18:46:50 -0400 Subject: [PATCH 097/164] Tidy cursor iteration module a little --- src/iterator.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/iterator.rs b/src/iterator.rs index 0c1c477..29e19a8 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,5 +1,6 @@ use futures::Async; -use ebml::*; +use ebml::Error as EbmlError; +use ebml::FromEbml; use webm::*; pub struct EbmlCursor { @@ -7,8 +8,8 @@ pub struct EbmlCursor { position: usize } -impl EbmlCursor { - pub fn new(source: T) -> Self { +impl EbmlCursor { + pub fn new(source: S) -> Self { EbmlCursor { source, position: 0 @@ -20,7 +21,7 @@ impl<'a> Iterator for EbmlCursor<&'a [u8]> { type Item = WebmElement<'a>; fn next(&mut self) -> Option> { - match Self::Item::decode_element(&self.source[self.position..]) { + match Self::Item::decode_element(&self.source.as_ref()[self.position..]) { Err(_) => None, Ok(None) => None, Ok(Some((element, element_size))) => { @@ -31,10 +32,10 @@ impl<'a> Iterator for EbmlCursor<&'a [u8]> { } } -impl<'b, T: AsRef<[u8]>> WebmEventSource for EbmlCursor { - type Error = Error; +impl<'b, S: AsRef<[u8]>> WebmEventSource for EbmlCursor { + type Error = EbmlError; - fn poll_event<'a>(&'a mut self) -> Result>>, Error> { + fn poll_event<'a>(&'a mut self) -> Result>>, EbmlError> { match WebmElement::decode_element(&self.source.as_ref()[self.position..]) { Err(err) => Err(err), Ok(None) => Ok(Async::Ready(None)), From 90c1892668ccedbda93c0486ecbca7a1f716a919 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 19:12:28 -0400 Subject: [PATCH 098/164] Commonize some code for EbmlCursor --- src/iterator.rs | 33 ++++++++++++++++----------------- src/webm.rs | 4 ++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/iterator.rs b/src/iterator.rs index 29e19a8..a668835 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -17,32 +17,31 @@ impl EbmlCursor { } } +impl<'a> EbmlCursor<&'a [u8]> { + fn decode_element>(&mut self) -> Result, EbmlError> { + match T::decode_element(&self.source.as_ref()[self.position..]) { + Err(err) => Err(err), + Ok(None) => Ok(None), + Ok(Some((element, element_size))) => { + self.position += element_size; + Ok(Some(element)) + } + } + } +} + impl<'a> Iterator for EbmlCursor<&'a [u8]> { type Item = WebmElement<'a>; fn next(&mut self) -> Option> { - match Self::Item::decode_element(&self.source.as_ref()[self.position..]) { - Err(_) => None, - Ok(None) => None, - Ok(Some((element, element_size))) => { - self.position += element_size; - Some(element) - } - } + self.decode_element().unwrap_or(None) } } -impl<'b, S: AsRef<[u8]>> WebmEventSource for EbmlCursor { +impl<'b> WebmEventSource for EbmlCursor<&'b [u8]> { type Error = EbmlError; fn poll_event<'a>(&'a mut self) -> Result>>, EbmlError> { - match WebmElement::decode_element(&self.source.as_ref()[self.position..]) { - Err(err) => Err(err), - Ok(None) => Ok(Async::Ready(None)), - Ok(Some((element, element_size))) => { - self.position += element_size; - Ok(Async::Ready(Some(element))) - } - } + self.decode_element().map(Async::Ready) } } diff --git a/src/webm.rs b/src/webm.rs index bce0a05..338723f 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -13,8 +13,8 @@ const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub fn parse_webm>(source: T) -> EbmlCursor { - EbmlCursor::new(source) +pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlCursor<&'a [u8]> { + EbmlCursor::new(source.as_ref()) } #[derive(Debug, PartialEq, Copy, Clone)] From 719b11dd015a05849f80770ae802717af29de0e9 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 19:37:36 -0400 Subject: [PATCH 099/164] Zap some generic code that wasn't going anywhere --- src/ebml.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 2ef62fe..67400ae 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -1,5 +1,4 @@ use bytes::{BigEndian, ByteOrder, BufMut}; -use futures::Async; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; @@ -251,12 +250,6 @@ pub trait FromEbml<'b>: Sized { } } -pub trait EbmlEventSource<'a> { - type Event: FromEbml<'a>; - type Error; - fn poll_event(&'a mut self) -> Result>, Self::Error>; -} - #[cfg(test)] mod tests { use bytes::{BytesMut}; From c6ab5e8e89b28d6a376b26143319e2e591d594ed Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 20:10:50 -0400 Subject: [PATCH 100/164] Fix name collision --- src/webm_stream.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 5f50fc0..5bf7b25 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -11,15 +11,15 @@ pub enum ParsingError { OtherError(E) } -pub struct WebmStream { +pub struct WebmBuffer { stream: S, buffer: BytesMut, last_read: usize } -impl, S: Stream> WebmStream { +impl, S: Stream> WebmBuffer { pub fn new(stream: S) -> Self { - WebmStream { + WebmBuffer { stream: stream, buffer: BytesMut::new(), last_read: 0 @@ -71,11 +71,11 @@ impl, S: Stream> WebmStream { } } -impl, S: Stream> WebmEventSource for WebmStream { +impl, S: Stream> WebmEventSource for WebmBuffer { type Error = ParsingError; fn poll_event<'a>(&'a mut self) -> Result>>, Self::Error> { - return WebmStream::poll_event(self); + return WebmBuffer::poll_event(self); } } From c5a42afd3284aca868bef6b2a197f47c581b08a0 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 20:11:14 -0400 Subject: [PATCH 101/164] Fix bug with WebmBuffer double-counting reads --- src/webm_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 5bf7b25..8d3ee45 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -34,7 +34,7 @@ impl, S: Stream> WebmBuffer { return Ok(Async::NotReady); }, Ok(Some((element, element_size))) => { - self.last_read += element_size; + self.last_read = element_size; return Ok(Async::Ready(Some(element))) } }; From 12cd1bdd4bb3e6140d144d4b8451d5d13e057a2e Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 20:11:54 -0400 Subject: [PATCH 102/164] Debugging formatting --- src/chunk.rs | 2 +- src/webm_stream.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 3c10b09..449248b 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -50,7 +50,7 @@ impl AsRef<[u8]> for ClusterHead { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum Chunk = Vec> { Headers { bytes: Arc diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 8d3ee45..d2a77b9 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -28,12 +28,17 @@ impl, S: Stream> WebmBuffer { pub fn try_decode(&mut self) -> Result>, ParsingError> { match WebmElement::decode_element(&self.buffer) { - Err(err) => return Err(ParsingError::EbmlError(err)), + Err(err) => { + //println!("EBML error: {:?}", err); + return Err(ParsingError::EbmlError(err)) + }, Ok(None) => { + //println!("Need refill"); // need to refill buffer return Ok(Async::NotReady); }, Ok(Some((element, element_size))) => { + //println!("Parsed element: {:?}", element); self.last_read = element_size; return Ok(Async::Ready(Some(element))) } @@ -63,6 +68,7 @@ impl, S: Stream> WebmBuffer { Ok(Async::Ready(Some(chunk))) => { self.buffer.reserve(chunk.as_ref().len()); self.buffer.put_slice(chunk.as_ref()); + //println!("Read {} into Buffer", chunk.as_ref().len()); // ok can retry decoding now } Err(err) => return Err(ParsingError::OtherError(err)) From 724eabf3263b022a544cea7d7429d9bb435b2199 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 2 Apr 2018 20:12:26 -0400 Subject: [PATCH 103/164] Use WebmBuffer in loop_server --- src/bin/loop_server.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 6c0f85d..12e4eb5 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -3,11 +3,12 @@ extern crate hyper; extern crate lab_ebml; use futures::future::FutureResult; +use futures::stream::once; use futures::stream::repeat; use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; use lab_ebml::timecode_fixer::ChunkStream; -use lab_ebml::webm::*; +use lab_ebml::webm_stream::WebmBuffer; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; @@ -30,10 +31,10 @@ impl Service for WebmServer { let response = match (req.method(), req.path()) { (&Get, "/loop") => { let stream: BodyStream> = Box::new( - repeat(()).take(10) - .map(|()| - parse_webm(SRC_FILE).chunk_webm() - ).flatten() + repeat(()).take(3) + .map(|()| { + WebmBuffer::new(once::<&[u8], ()>(Ok(SRC_FILE))).chunk_webm() + }).flatten() .fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), From bebe593c301286fa4c42ca1655ffd70881a0dac6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 3 Apr 2018 19:07:16 -0400 Subject: [PATCH 104/164] Lifetime simplification reflecting probably-better understanding of them --- src/ebml.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 67400ae..5a9293b 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -218,11 +218,11 @@ pub struct Ebml { pub source: Source } -pub trait FromEbml<'b>: Sized { +pub trait FromEbml<'a>: Sized { fn should_unwrap(element_id: u64) -> bool; - fn decode(element_id: u64, bytes: &'b[u8]) -> Result; + fn decode(element_id: u64, bytes: &'a[u8]) -> Result; - fn decode_element<'a: 'b>(bytes: &'a[u8]) -> Result, Error> { + fn decode_element(bytes: &'a[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), From c92273625e6ece5b1a2759133320d3ffad3569f6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 3 Apr 2018 21:19:46 -0400 Subject: [PATCH 105/164] Cleanup poll_event's try_decode hack. --- src/ebml.rs | 36 +++++++++++++++++++++++++++++++++ src/webm_stream.rs | 50 ++++++++++++++++++++-------------------------- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index 5a9293b..f0797e7 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -219,9 +219,45 @@ pub struct Ebml { } pub trait FromEbml<'a>: Sized { + /// Indicates if this tag's contents should be treated as a blob, + /// or if the tag header should be reported as an event and with further + /// parsing descending into its content. + /// + /// Unknown-size tags can *only* be parsed if unwrapped, and will error otherwise. fn should_unwrap(element_id: u64) -> bool; + + /// Given an element's ID and its binary payload, if any, construct a suitable + /// instance of this type to represent the event. The instance may contain + /// references into the given buffer. fn decode(element_id: u64, bytes: &'a[u8]) -> Result; + /// Check if enough space exists in the given buffer for decode_element() to + /// be successful; parsing errors will be returned eagerly. + fn check_space(bytes: &[u8]) -> Result, Error> { + match decode_tag(bytes) { + Ok(None) => Ok(None), + Err(err) => Err(err), + Ok(Some((element_id, payload_size_tag, tag_size))) => { + let should_unwrap = Self::should_unwrap(element_id); + + let payload_size = match (should_unwrap, payload_size_tag) { + (true, _) => 0, + (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Value(size)) => size as usize + }; + + let element_size = tag_size + payload_size; + if element_size > bytes.len() { + // need to read more still + Ok(None) + } else { + Ok(Some(element_size)) + } + } + } + } + + /// Attempt to construct an instance of this type from the given byte slice fn decode_element(bytes: &'a[u8]) -> Result, Error> { match decode_tag(bytes) { Ok(None) => Ok(None), diff --git a/src/webm_stream.rs b/src/webm_stream.rs index d2a77b9..55d8e6e 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -26,40 +26,34 @@ impl, S: Stream> WebmBuffer { } } - pub fn try_decode(&mut self) -> Result>, ParsingError> { - match WebmElement::decode_element(&self.buffer) { - Err(err) => { - //println!("EBML error: {:?}", err); - return Err(ParsingError::EbmlError(err)) - }, - Ok(None) => { - //println!("Need refill"); - // need to refill buffer - return Ok(Async::NotReady); - }, - Ok(Some((element, element_size))) => { - //println!("Parsed element: {:?}", element); - self.last_read = element_size; - return Ok(Async::Ready(Some(element))) - } - }; - } - - pub fn can_decode(&mut self) -> bool { - match self.try_decode() { - Ok(Async::NotReady) => false, - _ => true - } - } - pub fn poll_event<'a>(&'a mut self) -> Result>>, ParsingError> { // release buffer from previous event self.buffer.advance(self.last_read); self.last_read = 0; loop { - if self.can_decode() { - return self.try_decode() + match WebmElement::check_space(&self.buffer) { + Err(err) => { + return Err(ParsingError::EbmlError(err)) + }, + Ok(None) => { + // need to refill buffer, below + }, + Ok(Some(_)) => { + return match WebmElement::decode_element(&self.buffer) { + Err(err) => { + Err(ParsingError::EbmlError(err)) + }, + Ok(None) => { + // buffer should have the data already + panic!("Buffer was supposed to have enough data to parse element, somehow did not.") + }, + Ok(Some((element, element_size))) => { + self.last_read = element_size; + return Ok(Async::Ready(Some(element))) + } + } + } } match self.stream.poll() { From 413f7759c6704eeb68916dbd71a3f6efcb764c8d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 3 Apr 2018 22:06:09 -0400 Subject: [PATCH 106/164] Make the buffer generic --- src/webm_stream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 55d8e6e..b9124a1 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -26,13 +26,13 @@ impl, S: Stream> WebmBuffer { } } - pub fn poll_event<'a>(&'a mut self) -> Result>>, ParsingError> { + pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, ParsingError> { // release buffer from previous event self.buffer.advance(self.last_read); self.last_read = 0; loop { - match WebmElement::check_space(&self.buffer) { + match T::check_space(&self.buffer) { Err(err) => { return Err(ParsingError::EbmlError(err)) }, @@ -40,7 +40,7 @@ impl, S: Stream> WebmBuffer { // need to refill buffer, below }, Ok(Some(_)) => { - return match WebmElement::decode_element(&self.buffer) { + return match T::decode_element(&self.buffer) { Err(err) => { Err(ParsingError::EbmlError(err)) }, From 0909a20a8c5597467f94550d20bfb4c104c863ac Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 3 Apr 2018 23:23:19 -0400 Subject: [PATCH 107/164] generalize WebmEventSource -> EbmlEventSource --- src/chunk.rs | 9 ++--- src/ebml.rs | 6 ++++ src/iterator.rs | 86 +++++++++++++++++++++++++++++++--------------- src/webm.rs | 8 +---- src/webm_stream.rs | 5 ++- 5 files changed, 73 insertions(+), 41 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 449248b..0de1459 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -2,6 +2,7 @@ use futures::{Async, Stream}; use std::io::Cursor; use std::mem; use std::sync::Arc; +use ebml::EbmlEventSource; use webm::*; #[derive(Clone, Debug)] @@ -87,12 +88,12 @@ pub enum ChunkingError { OtherError(E) } -pub struct WebmChunker { +pub struct WebmChunker { source: S, state: ChunkerState } -impl<'a, S: WebmEventSource> Stream for WebmChunker +impl<'a, S: EbmlEventSource> Stream for WebmChunker { type Item = Chunk; type Error = ChunkingError; @@ -209,11 +210,11 @@ impl<'a, S: WebmEventSource> Stream for WebmChunker } } -pub trait WebmStream { +pub trait WebmStream { fn chunk_webm(self) -> WebmChunker; } -impl<'a, T: WebmEventSource> WebmStream for T { +impl<'a, T: EbmlEventSource> WebmStream for T { fn chunk_webm(self) -> WebmChunker { WebmChunker { source: self, diff --git a/src/ebml.rs b/src/ebml.rs index f0797e7..d780b89 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -2,6 +2,7 @@ use bytes::{BigEndian, ByteOrder, BufMut}; use std::error::Error as ErrorTrait; use std::fmt::{Display, Formatter, Result as FmtResult}; use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek, SeekFrom}; +use futures::Async; pub const EBML_HEAD_ID: u64 = 0x0A45DFA3; pub const DOC_TYPE_ID: u64 = 0x0282; @@ -286,6 +287,11 @@ pub trait FromEbml<'a>: Sized { } } +pub trait EbmlEventSource { + type Error; + fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, Self::Error>; +} + #[cfg(test)] mod tests { use bytes::{BytesMut}; diff --git a/src/iterator.rs b/src/iterator.rs index a668835..d8a6850 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,47 +1,79 @@ use futures::Async; use ebml::Error as EbmlError; +use ebml::EbmlEventSource; use ebml::FromEbml; -use webm::*; +use webm::WebmElement; -pub struct EbmlCursor { - source: T, - position: usize +pub struct EbmlCursor<'a> { + source: &'a [u8] } -impl EbmlCursor { - pub fn new(source: S) -> Self { +impl<'a> EbmlCursor<'a> { + pub fn new(source: &'a [u8]) -> Self { EbmlCursor { - source, - position: 0 + source } } } -impl<'a> EbmlCursor<&'a [u8]> { - fn decode_element>(&mut self) -> Result, EbmlError> { - match T::decode_element(&self.source.as_ref()[self.position..]) { - Err(err) => Err(err), - Ok(None) => Ok(None), - Ok(Some((element, element_size))) => { - self.position += element_size; - Ok(Some(element)) +impl<'a> Iterator for EbmlCursor<'a> { + type Item = WebmElement<'a>; + + fn next(&mut self) -> Option> { + match WebmElement::check_space(self.source) { + Err(err) => { + None + }, + Ok(None) => { + None + }, + Ok(Some(element_size)) => { + let (element_data, rest) = self.source.split_at(element_size); + self.source = rest; + match WebmElement::decode_element(element_data) { + Err(err) => { + None + }, + Ok(None) => { + // buffer should have enough data + panic!("Buffer was supposed to have enough data to parse element, somehow did not.") + }, + Ok(Some((element, _))) => { + Some(element) + } + } } } } } -impl<'a> Iterator for EbmlCursor<&'a [u8]> { - type Item = WebmElement<'a>; - - fn next(&mut self) -> Option> { - self.decode_element().unwrap_or(None) - } -} - -impl<'b> WebmEventSource for EbmlCursor<&'b [u8]> { +impl<'b> EbmlEventSource for EbmlCursor<'b> { type Error = EbmlError; - fn poll_event<'a>(&'a mut self) -> Result>>, EbmlError> { - self.decode_element().map(Async::Ready) + fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { + match T::check_space(self.source) { + Err(err) => { + Err(err) + }, + Ok(None) => { + Ok(None) + }, + Ok(Some(element_size)) => { + let (element_data, rest) = self.source.split_at(element_size); + self.source = rest; + match T::decode_element(element_data) { + Err(err) => { + Err(err) + }, + Ok(None) => { + // buffer should have enough data + panic!("Buffer was supposed to have enough data to parse element, somehow did not.") + }, + Ok(Some((element, _))) => { + Ok(Some(element)) + } + } + } + }.map(Async::Ready) } } diff --git a/src/webm.rs b/src/webm.rs index 338723f..50d76cd 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,6 +1,5 @@ use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; use bytes::{BigEndian, BufMut, ByteOrder}; -use futures::Async; use ebml::*; use iterator::EbmlCursor; @@ -13,7 +12,7 @@ const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlCursor<&'a [u8]> { +pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlCursor<'a> { EbmlCursor::new(source.as_ref()) } @@ -127,11 +126,6 @@ pub fn encode_webm_element(element: &WebmElement, output: &mut } } -pub trait WebmEventSource { - type Error; - fn poll_event<'a>(&'a mut self) -> Result>>, Self::Error>; -} - #[cfg(test)] mod tests { use tests::TEST_FILE; diff --git a/src/webm_stream.rs b/src/webm_stream.rs index b9124a1..2bd2177 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -4,7 +4,6 @@ use futures::Async; use futures::stream::Stream; use ebml::*; -use webm::*; pub enum ParsingError { EbmlError(::ebml::Error), @@ -71,10 +70,10 @@ impl, S: Stream> WebmBuffer { } } -impl, S: Stream> WebmEventSource for WebmBuffer { +impl, S: Stream> EbmlEventSource for WebmBuffer { type Error = ParsingError; - fn poll_event<'a>(&'a mut self) -> Result>>, Self::Error> { + fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, Self::Error> { return WebmBuffer::poll_event(self); } } From 2db227c5f90e89043bc1f88d4bf64f8725238ef5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 3 Apr 2018 23:38:31 -0400 Subject: [PATCH 108/164] Cleanup, use EbmlSlice as name --- src/ebml.rs | 5 ----- src/iterator.rs | 28 +++++++++------------------- src/webm.rs | 6 +++--- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index d780b89..f849dab 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -214,11 +214,6 @@ pub fn encode_integer(tag: u64, value: u64, output: &mut T) -> IoResul output.write_all(&buffer.get_ref()[..]) } -#[derive(Debug, PartialEq)] -pub struct Ebml { - pub source: Source -} - pub trait FromEbml<'a>: Sized { /// Indicates if this tag's contents should be treated as a blob, /// or if the tag header should be reported as an event and with further diff --git a/src/iterator.rs b/src/iterator.rs index d8a6850..0dfa72e 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -4,23 +4,13 @@ use ebml::EbmlEventSource; use ebml::FromEbml; use webm::WebmElement; -pub struct EbmlCursor<'a> { - source: &'a [u8] -} +pub struct EbmlSlice<'a>(pub &'a [u8]); -impl<'a> EbmlCursor<'a> { - pub fn new(source: &'a [u8]) -> Self { - EbmlCursor { - source - } - } -} - -impl<'a> Iterator for EbmlCursor<'a> { +impl<'a> Iterator for EbmlSlice<'a> { type Item = WebmElement<'a>; fn next(&mut self) -> Option> { - match WebmElement::check_space(self.source) { + match WebmElement::check_space(self.0) { Err(err) => { None }, @@ -28,8 +18,8 @@ impl<'a> Iterator for EbmlCursor<'a> { None }, Ok(Some(element_size)) => { - let (element_data, rest) = self.source.split_at(element_size); - self.source = rest; + let (element_data, rest) = self.0.split_at(element_size); + self.0 = rest; match WebmElement::decode_element(element_data) { Err(err) => { None @@ -47,11 +37,11 @@ impl<'a> Iterator for EbmlCursor<'a> { } } -impl<'b> EbmlEventSource for EbmlCursor<'b> { +impl<'b> EbmlEventSource for EbmlSlice<'b> { type Error = EbmlError; fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { - match T::check_space(self.source) { + match T::check_space(self.0) { Err(err) => { Err(err) }, @@ -59,8 +49,8 @@ impl<'b> EbmlEventSource for EbmlCursor<'b> { Ok(None) }, Ok(Some(element_size)) => { - let (element_data, rest) = self.source.split_at(element_size); - self.source = rest; + let (element_data, rest) = self.0.split_at(element_size); + self.0 = rest; match T::decode_element(element_data) { Err(err) => { Err(err) diff --git a/src/webm.rs b/src/webm.rs index 50d76cd..bdc1864 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,7 +1,7 @@ use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; use bytes::{BigEndian, BufMut, ByteOrder}; use ebml::*; -use iterator::EbmlCursor; +use iterator::EbmlSlice; const SEGMENT_ID: u64 = 0x08538067; const SEEK_HEAD_ID: u64 = 0x014D9B74; @@ -12,8 +12,8 @@ const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlCursor<'a> { - EbmlCursor::new(source.as_ref()) +pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlSlice<'a> { + EbmlSlice(source.as_ref()) } #[derive(Debug, PartialEq, Copy, Clone)] From ee068bc8875169c21f3a3e833952be3b6c63f49a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 00:12:55 -0400 Subject: [PATCH 109/164] Shrink EbmlSlice trait implementations --- src/iterator.rs | 63 +++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/src/iterator.rs b/src/iterator.rs index 0dfa72e..5bfbeab 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -10,30 +10,15 @@ impl<'a> Iterator for EbmlSlice<'a> { type Item = WebmElement<'a>; fn next(&mut self) -> Option> { - match WebmElement::check_space(self.0) { - Err(err) => { - None - }, - Ok(None) => { - None - }, - Ok(Some(element_size)) => { - let (element_data, rest) = self.0.split_at(element_size); - self.0 = rest; - match WebmElement::decode_element(element_data) { - Err(err) => { - None - }, - Ok(None) => { - // buffer should have enough data - panic!("Buffer was supposed to have enough data to parse element, somehow did not.") - }, - Ok(Some((element, _))) => { - Some(element) - } - } + WebmElement::check_space(self.0).unwrap_or(None).and_then(|element_size| { + let (element_data, rest) = self.0.split_at(element_size); + self.0 = rest; + match WebmElement::decode_element(element_data) { + Err(_) => None, + Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), + Ok(Some((element, _))) => Some(element) } - } + }) } } @@ -41,29 +26,19 @@ impl<'b> EbmlEventSource for EbmlSlice<'b> { type Error = EbmlError; fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { - match T::check_space(self.0) { - Err(err) => { - Err(err) - }, - Ok(None) => { - Ok(None) - }, - Ok(Some(element_size)) => { - let (element_data, rest) = self.0.split_at(element_size); - self.0 = rest; - match T::decode_element(element_data) { - Err(err) => { - Err(err) - }, - Ok(None) => { - // buffer should have enough data - panic!("Buffer was supposed to have enough data to parse element, somehow did not.") - }, - Ok(Some((element, _))) => { - Ok(Some(element)) + T::check_space(self.0).and_then(|size_option| { + match size_option { + None => Ok(None), + Some(element_size) => { + let (element_data, rest) = self.0.split_at(element_size); + self.0 = rest; + match T::decode_element(element_data) { + Err(err) => Err(err), + Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), + Ok(Some((element, _))) => Ok(Some(element)) } } } - }.map(Async::Ready) + }).map(Async::Ready) } } From 9686eb27137020eb602e8f2f00fcd0454cc81df1 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 00:16:35 -0400 Subject: [PATCH 110/164] Rename EbmlError --- src/ebml.rs | 32 ++++++++++++++++---------------- src/iterator.rs | 2 +- src/lib.rs | 2 +- src/webm.rs | 8 ++++---- src/webm_stream.rs | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/ebml.rs b/src/ebml.rs index f849dab..a9fc150 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -9,7 +9,7 @@ pub const DOC_TYPE_ID: u64 = 0x0282; pub const VOID_ID: u64 = 0x6C; #[derive(Debug, PartialEq)] -pub enum Error { +pub enum EbmlError { CorruptVarint, UnknownElementId, UnknownElementLength, @@ -48,7 +48,7 @@ pub enum Varint { /// Returns Ok(None) if more bytes are needed to get a result. /// Returns Ok(Some((varint, size))) to return a varint value and /// the size of the parsed varint. -pub fn decode_varint(bytes: &[u8]) -> Result, Error> { +pub fn decode_varint(bytes: &[u8]) -> Result, EbmlError> { let mut value: u64 = 0; let mut value_length = 1; let mut mask: u8 = 0x80; @@ -70,7 +70,7 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> { } if mask == 0 { - return Err(Error::CorruptVarint) + return Err(EbmlError::CorruptVarint) } // check we have enough data to parse @@ -97,12 +97,12 @@ pub fn decode_varint(bytes: &[u8]) -> Result, Error> { /// Returns Ok(None) if more bytes are needed to get a result. /// Returns Ok(Some((id, varint, size))) to return the element id, /// the size of the payload, and the size of the parsed header. -pub fn decode_tag(bytes: &[u8]) -> Result, Error> { +pub fn decode_tag(bytes: &[u8]) -> Result, EbmlError> { // parse element ID match decode_varint(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), - Ok(Some((Varint::Unknown, _))) => Err(Error::UnknownElementId), + Ok(Some((Varint::Unknown, _))) => Err(EbmlError::UnknownElementId), Ok(Some((Varint::Value(element_id), id_size))) => { // parse payload size match decode_varint(&bytes[id_size..]) { @@ -119,9 +119,9 @@ pub fn decode_tag(bytes: &[u8]) -> Result, Error> { } } -pub fn decode_uint(bytes: &[u8]) -> Result { +pub fn decode_uint(bytes: &[u8]) -> Result { if bytes.len() < 1 || bytes.len() > 8 { - return Err(Error::CorruptPayload); + return Err(EbmlError::CorruptPayload); } Ok(BigEndian::read_uint(bytes, bytes.len())) @@ -225,11 +225,11 @@ pub trait FromEbml<'a>: Sized { /// Given an element's ID and its binary payload, if any, construct a suitable /// instance of this type to represent the event. The instance may contain /// references into the given buffer. - fn decode(element_id: u64, bytes: &'a[u8]) -> Result; + fn decode(element_id: u64, bytes: &'a[u8]) -> Result; /// Check if enough space exists in the given buffer for decode_element() to /// be successful; parsing errors will be returned eagerly. - fn check_space(bytes: &[u8]) -> Result, Error> { + fn check_space(bytes: &[u8]) -> Result, EbmlError> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), @@ -238,7 +238,7 @@ pub trait FromEbml<'a>: Sized { let payload_size = match (should_unwrap, payload_size_tag) { (true, _) => 0, - (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Unknown) => return Err(EbmlError::UnknownElementLength), (false, Varint::Value(size)) => size as usize }; @@ -254,7 +254,7 @@ pub trait FromEbml<'a>: Sized { } /// Attempt to construct an instance of this type from the given byte slice - fn decode_element(bytes: &'a[u8]) -> Result, Error> { + fn decode_element(bytes: &'a[u8]) -> Result, EbmlError> { match decode_tag(bytes) { Ok(None) => Ok(None), Err(err) => Err(err), @@ -263,7 +263,7 @@ pub trait FromEbml<'a>: Sized { let payload_size = match (should_unwrap, payload_size_tag) { (true, _) => 0, - (false, Varint::Unknown) => return Err(Error::UnknownElementLength), + (false, Varint::Unknown) => return Err(EbmlError::UnknownElementLength), (false, Varint::Value(size)) => size as usize }; @@ -291,7 +291,7 @@ pub trait EbmlEventSource { mod tests { use bytes::{BytesMut}; use ebml::*; - use ebml::Error::{CorruptVarint, UnknownElementId}; + use ebml::EbmlError::{CorruptVarint, UnknownElementId}; use ebml::Varint::{Unknown, Value}; use std::io::Cursor; use tests::TEST_FILE; @@ -414,8 +414,8 @@ mod tests { #[test] fn bad_uints() { - assert_eq!(decode_uint(&[]), Err(Error::CorruptPayload)); - assert_eq!(decode_uint(&[0; 9]), Err(Error::CorruptPayload)); + assert_eq!(decode_uint(&[]), Err(EbmlError::CorruptPayload)); + assert_eq!(decode_uint(&[0; 9]), Err(EbmlError::CorruptPayload)); } #[test] @@ -441,7 +441,7 @@ mod tests { } } - fn decode(element_id: u64, bytes: &'a[u8]) -> Result { + fn decode(element_id: u64, bytes: &'a[u8]) -> Result { match element_id { _ => Ok(GenericElement(element_id, bytes.len())) } diff --git a/src/iterator.rs b/src/iterator.rs index 5bfbeab..bcfcd5e 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,5 +1,5 @@ use futures::Async; -use ebml::Error as EbmlError; +use ebml::EbmlError; use ebml::EbmlEventSource; use ebml::FromEbml; use webm::WebmElement; diff --git a/src/lib.rs b/src/lib.rs index a3a7abe..a5ca2bd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ pub mod webm_stream; pub mod timecode_fixer; pub mod webm; -pub use ebml::{Error, FromEbml}; +pub use ebml::{EbmlError, FromEbml}; #[cfg(test)] mod tests { diff --git a/src/webm.rs b/src/webm.rs index bdc1864..6a57add 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -49,7 +49,7 @@ impl<'b> FromEbml<'b> for WebmElement<'b> { } } - fn decode(element_id: u64, bytes: &'b[u8]) -> Result, Error> { + fn decode(element_id: u64, bytes: &'b[u8]) -> Result, EbmlError> { match element_id { EBML_HEAD_ID => Ok(WebmElement::EbmlHead), VOID_ID => Ok(WebmElement::Void), @@ -66,11 +66,11 @@ impl<'b> FromEbml<'b> for WebmElement<'b> { } } -fn decode_simple_block(bytes: &[u8]) -> Result { +fn decode_simple_block(bytes: &[u8]) -> Result { if let Ok(Some((Varint::Value(track), track_field_len))) = decode_varint(bytes) { let header_len = track_field_len + 2 + 1; if bytes.len() < header_len { - return Err(Error::CorruptPayload); + return Err(EbmlError::CorruptPayload); } let timecode = BigEndian::read_i16(&bytes[track_field_len..]); let flags = bytes[track_field_len + 2]; @@ -81,7 +81,7 @@ fn decode_simple_block(bytes: &[u8]) -> Result { data: &bytes[header_len..], })) } else { - return Err(Error::CorruptPayload); + return Err(EbmlError::CorruptPayload); } } diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 2bd2177..800b311 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -6,7 +6,7 @@ use futures::stream::Stream; use ebml::*; pub enum ParsingError { - EbmlError(::ebml::Error), + EbmlError(EbmlError), OtherError(E) } From 3a92da188318401d81381c47721826c88531dd2a Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 00:43:25 -0400 Subject: [PATCH 111/164] Split apart iterator and slice modules --- src/iterator.rs | 34 ++++++++-------------------------- src/lib.rs | 2 ++ src/slice.rs | 27 +++++++++++++++++++++++++++ src/webm.rs | 7 ++++--- 4 files changed, 41 insertions(+), 29 deletions(-) create mode 100644 src/slice.rs diff --git a/src/iterator.rs b/src/iterator.rs index bcfcd5e..d746b3c 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,12 +1,15 @@ -use futures::Async; -use ebml::EbmlError; -use ebml::EbmlEventSource; +use std::marker::PhantomData; + use ebml::FromEbml; use webm::WebmElement; -pub struct EbmlSlice<'a>(pub &'a [u8]); +pub struct EbmlIterator<'a, T: FromEbml<'a>>(&'a [u8], PhantomData T>); -impl<'a> Iterator for EbmlSlice<'a> { +pub fn ebml_iter<'a, T: FromEbml<'a>>(source: &'a [u8])-> EbmlIterator<'a, T> { + EbmlIterator(source, PhantomData) +} + +impl<'a, T: FromEbml<'a>> Iterator for EbmlIterator<'a, T> { type Item = WebmElement<'a>; fn next(&mut self) -> Option> { @@ -21,24 +24,3 @@ impl<'a> Iterator for EbmlSlice<'a> { }) } } - -impl<'b> EbmlEventSource for EbmlSlice<'b> { - type Error = EbmlError; - - fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { - T::check_space(self.0).and_then(|size_option| { - match size_option { - None => Ok(None), - Some(element_size) => { - let (element_data, rest) = self.0.split_at(element_size); - self.0 = rest; - match T::decode_element(element_data) { - Err(err) => Err(err), - Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), - Ok(Some((element, _))) => Ok(Some(element)) - } - } - } - }).map(Async::Ready) - } -} diff --git a/src/lib.rs b/src/lib.rs index a5ca2bd..89a1de1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,8 @@ extern crate futures; pub mod chunk; pub mod ebml; mod iterator; +pub mod slice; + pub mod webm_stream; pub mod timecode_fixer; pub mod webm; diff --git a/src/slice.rs b/src/slice.rs new file mode 100644 index 0000000..8e126b7 --- /dev/null +++ b/src/slice.rs @@ -0,0 +1,27 @@ +use futures::Async; +use ebml::EbmlError; +use ebml::EbmlEventSource; +use ebml::FromEbml; + +pub struct EbmlSlice<'a>(pub &'a [u8]); + +impl<'b> EbmlEventSource for EbmlSlice<'b> { + type Error = EbmlError; + + fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { + T::check_space(self.0).and_then(|size_option| { + match size_option { + None => Ok(None), + Some(element_size) => { + let (element_data, rest) = self.0.split_at(element_size); + self.0 = rest; + match T::decode_element(element_data) { + Err(err) => Err(err), + Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), + Ok(Some((element, _))) => Ok(Some(element)) + } + } + } + }).map(Async::Ready) + } +} diff --git a/src/webm.rs b/src/webm.rs index 6a57add..c0f1d71 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -1,7 +1,8 @@ use std::io::{Cursor, Error as IoError, ErrorKind, Result as IoResult, Write, Seek}; use bytes::{BigEndian, BufMut, ByteOrder}; use ebml::*; -use iterator::EbmlSlice; +use iterator::ebml_iter; +use iterator::EbmlIterator; const SEGMENT_ID: u64 = 0x08538067; const SEEK_HEAD_ID: u64 = 0x014D9B74; @@ -12,8 +13,8 @@ const CLUSTER_ID: u64 = 0x0F43B675; const TIMECODE_ID: u64 = 0x67; const SIMPLE_BLOCK_ID: u64 = 0x23; -pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlSlice<'a> { - EbmlSlice(source.as_ref()) +pub fn parse_webm<'a, T: AsRef<[u8]> + ?Sized>(source: &'a T) -> EbmlIterator<'a, WebmElement> { + ebml_iter(source.as_ref()) } #[derive(Debug, PartialEq, Copy, Clone)] From d0fd4d4bc98a6a5d8ebd1a657864f08ae225b4a0 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 00:54:58 -0400 Subject: [PATCH 112/164] Simplify EbmlIterator & EbmlSlice's next() methods, fix EbmlIterator generic (they don't have dynamic buffers to worry about) --- src/iterator.rs | 16 +++++----------- src/slice.rs | 18 ++++-------------- src/webm.rs | 2 +- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/iterator.rs b/src/iterator.rs index d746b3c..ed5c2fd 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -1,7 +1,6 @@ use std::marker::PhantomData; use ebml::FromEbml; -use webm::WebmElement; pub struct EbmlIterator<'a, T: FromEbml<'a>>(&'a [u8], PhantomData T>); @@ -10,17 +9,12 @@ pub fn ebml_iter<'a, T: FromEbml<'a>>(source: &'a [u8])-> EbmlIterator<'a, T> { } impl<'a, T: FromEbml<'a>> Iterator for EbmlIterator<'a, T> { - type Item = WebmElement<'a>; + type Item = T; - fn next(&mut self) -> Option> { - WebmElement::check_space(self.0).unwrap_or(None).and_then(|element_size| { - let (element_data, rest) = self.0.split_at(element_size); - self.0 = rest; - match WebmElement::decode_element(element_data) { - Err(_) => None, - Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), - Ok(Some((element, _))) => Some(element) - } + fn next(&mut self) -> Option { + T::decode_element(self.0).unwrap_or(None).and_then(|(element, element_size)| { + self.0 = &self.0[element_size..]; + Some(element) }) } } diff --git a/src/slice.rs b/src/slice.rs index 8e126b7..32733ce 100644 --- a/src/slice.rs +++ b/src/slice.rs @@ -9,19 +9,9 @@ impl<'b> EbmlEventSource for EbmlSlice<'b> { type Error = EbmlError; fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, EbmlError> { - T::check_space(self.0).and_then(|size_option| { - match size_option { - None => Ok(None), - Some(element_size) => { - let (element_data, rest) = self.0.split_at(element_size); - self.0 = rest; - match T::decode_element(element_data) { - Err(err) => Err(err), - Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), - Ok(Some((element, _))) => Ok(Some(element)) - } - } - } - }).map(Async::Ready) + T::decode_element(self.0).map(|option| option.map(|(element, element_size)| { + self.0 = &self.0[element_size..]; + element + })).map(Async::Ready) } } diff --git a/src/webm.rs b/src/webm.rs index c0f1d71..907e0f2 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -134,7 +134,7 @@ mod tests { #[test] fn decode_webm_test1() { - let mut iter = parse_webm(TEST_FILE).into_iter(); + let mut iter = parse_webm(TEST_FILE); // test that we match the structure of the test file assert_eq!(iter.next(), Some(WebmElement::EbmlHead)); From 818979428734c86a5d5da6be1567137e075fb003 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 01:21:13 -0400 Subject: [PATCH 113/164] Tidy some imports --- src/bin/loop_server.rs | 5 +++-- src/slice.rs | 1 + src/webm_stream.rs | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 12e4eb5..9ae2ecd 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -2,6 +2,9 @@ extern crate futures; extern crate hyper; extern crate lab_ebml; +use std::env::args; +use std::net::ToSocketAddrs; + use futures::future::FutureResult; use futures::stream::once; use futures::stream::repeat; @@ -12,8 +15,6 @@ use lab_ebml::webm_stream::WebmBuffer; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; -use std::env::args; -use std::net::ToSocketAddrs; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); diff --git a/src/slice.rs b/src/slice.rs index 32733ce..9b1b547 100644 --- a/src/slice.rs +++ b/src/slice.rs @@ -1,4 +1,5 @@ use futures::Async; + use ebml::EbmlError; use ebml::EbmlEventSource; use ebml::FromEbml; diff --git a/src/webm_stream.rs b/src/webm_stream.rs index 800b311..4730473 100644 --- a/src/webm_stream.rs +++ b/src/webm_stream.rs @@ -3,7 +3,9 @@ use bytes::BufMut; use futures::Async; use futures::stream::Stream; -use ebml::*; +use ebml::EbmlError; +use ebml::EbmlEventSource; +use ebml::FromEbml; pub enum ParsingError { EbmlError(EbmlError), From be92d04c0907bbb92848cfe468b86bd4a643c67c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 01:36:15 -0400 Subject: [PATCH 114/164] rename EbmlStreamingParser to reflect genericity, change construction to Stream extension --- src/bin/loop_server.rs | 4 ++-- src/lib.rs | 2 +- src/{webm_stream.rs => stream_parser.rs} | 20 +++++++++++++------- 3 files changed, 16 insertions(+), 10 deletions(-) rename src/{webm_stream.rs => stream_parser.rs} (81%) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 9ae2ecd..c14e87a 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -11,7 +11,7 @@ use futures::stream::repeat; use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; use lab_ebml::timecode_fixer::ChunkStream; -use lab_ebml::webm_stream::WebmBuffer; +use lab_ebml::stream_parser::StreamEbml; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; @@ -34,7 +34,7 @@ impl Service for WebmServer { let stream: BodyStream> = Box::new( repeat(()).take(3) .map(|()| { - WebmBuffer::new(once::<&[u8], ()>(Ok(SRC_FILE))).chunk_webm() + once::<&[u8], ()>(Ok(SRC_FILE)).parse_ebml().chunk_webm() }).flatten() .fix_timecodes() .map_err(|err| match err { diff --git a/src/lib.rs b/src/lib.rs index 89a1de1..2a39830 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,8 @@ pub mod chunk; pub mod ebml; mod iterator; pub mod slice; +pub mod stream_parser; -pub mod webm_stream; pub mod timecode_fixer; pub mod webm; diff --git a/src/webm_stream.rs b/src/stream_parser.rs similarity index 81% rename from src/webm_stream.rs rename to src/stream_parser.rs index 4730473..de13163 100644 --- a/src/webm_stream.rs +++ b/src/stream_parser.rs @@ -12,21 +12,27 @@ pub enum ParsingError { OtherError(E) } -pub struct WebmBuffer { +pub struct EbmlStreamingParser { stream: S, buffer: BytesMut, last_read: usize } -impl, S: Stream> WebmBuffer { - pub fn new(stream: S) -> Self { - WebmBuffer { - stream: stream, +pub trait StreamEbml, S: Stream> { + fn parse_ebml(self) -> EbmlStreamingParser; +} + +impl, S: Stream> StreamEbml for S { + fn parse_ebml(self) -> EbmlStreamingParser { + EbmlStreamingParser { + stream: self, buffer: BytesMut::new(), last_read: 0 } } +} +impl, S: Stream> EbmlStreamingParser { pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, ParsingError> { // release buffer from previous event self.buffer.advance(self.last_read); @@ -72,11 +78,11 @@ impl, S: Stream> WebmBuffer { } } -impl, S: Stream> EbmlEventSource for WebmBuffer { +impl, S: Stream> EbmlEventSource for EbmlStreamingParser { type Error = ParsingError; fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, Self::Error> { - return WebmBuffer::poll_event(self); + return EbmlStreamingParser::poll_event(self); } } From 45012385c2f7ebbd5d70cc1f7eeb7a4189d41d1b Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 19:46:28 -0400 Subject: [PATCH 115/164] Tidy stream_parser --- src/stream_parser.rs | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/stream_parser.rs b/src/stream_parser.rs index de13163..ee8b120 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -40,40 +40,29 @@ impl, S: Stream> EbmlStreamingParser { loop { match T::check_space(&self.buffer) { - Err(err) => { - return Err(ParsingError::EbmlError(err)) - }, Ok(None) => { // need to refill buffer, below }, - Ok(Some(_)) => { - return match T::decode_element(&self.buffer) { - Err(err) => { - Err(ParsingError::EbmlError(err)) - }, - Ok(None) => { - // buffer should have the data already - panic!("Buffer was supposed to have enough data to parse element, somehow did not.") - }, + other => return other.map_err(ParsingError::EbmlError).and_then(move |_| { + match T::decode_element(&self.buffer) { + Err(err) => Err(ParsingError::EbmlError(err)), + Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), Ok(Some((element, element_size))) => { self.last_read = element_size; - return Ok(Async::Ready(Some(element))) + Ok(Async::Ready(Some(element))) } } - } + }) } - match self.stream.poll() { - Ok(Async::NotReady) => return Ok(Async::NotReady), - Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), + match self.stream.poll().map_err(ParsingError::OtherError) { Ok(Async::Ready(Some(chunk))) => { self.buffer.reserve(chunk.as_ref().len()); self.buffer.put_slice(chunk.as_ref()); - //println!("Read {} into Buffer", chunk.as_ref().len()); // ok can retry decoding now - } - Err(err) => return Err(ParsingError::OtherError(err)) - }; + }, + other => return other.map(|async| async.map(|_| None)) + } } } } From 7b9df1e269700af3b8de58e382cfbf2930759aa4 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 19:55:20 -0400 Subject: [PATCH 116/164] Treat WebmElements as value types --- src/bin/resynth.rs | 6 +++--- src/bin/stub.rs | 16 ++++++++-------- src/chunk.rs | 10 +++++----- src/webm.rs | 22 +++++++++++----------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index fe71fe5..c66e163 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -37,15 +37,15 @@ pub fn main() { let mut fixer = TimecodeFixer::new(); for element in &head { - encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); + encode_webm_element(fixer.process(element), &mut cursor).unwrap(); } for element in &body { - encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); + encode_webm_element(fixer.process(element), &mut cursor).unwrap(); } for element in &body { - encode_webm_element(&fixer.process(element), &mut cursor).unwrap(); + encode_webm_element(fixer.process(element), &mut cursor).unwrap(); } output = cursor.into_inner(); diff --git a/src/bin/stub.rs b/src/bin/stub.rs index 56f84d7..b3e0951 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -6,23 +6,23 @@ use lab_ebml::webm::*; pub fn main() { let mut cursor = Cursor::new(Vec::new()); - encode_webm_element(&WebmElement::EbmlHead, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Segment, &mut cursor).unwrap(); + encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Tracks(&[]), &mut cursor).unwrap(); + encode_webm_element(WebmElement::Tracks(&[]), &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Timecode(0), &mut cursor).unwrap(); + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); - encode_webm_element(&WebmElement::SimpleBlock(SimpleBlock { + encode_webm_element(WebmElement::SimpleBlock(SimpleBlock { track: 3, flags: 0x0, timecode: 123, data: "Hello, World".as_bytes() }), &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Timecode(1000), &mut cursor).unwrap(); + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); stdout().write_all(&cursor.get_ref()).unwrap(); } diff --git a/src/chunk.rs b/src/chunk.rs index 0de1459..566277b 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -33,8 +33,8 @@ impl ClusterHead { self.end = self.start + delta; let mut cursor = Cursor::new(self.bytes.as_mut()); // buffer is sized so these should never fail - encode_webm_element(&WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(&WebmElement::Timecode(timecode), &mut cursor).unwrap(); + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(timecode), &mut cursor).unwrap(); self.bytes_used = cursor.position() as u8; } pub fn observe_simpleblock_timecode(&mut self, timecode: i16) { @@ -120,7 +120,7 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker Ok(Async::Ready(Some(WebmElement::Info))) => continue, Ok(Async::Ready(Some(WebmElement::Void))) => continue, Ok(Async::Ready(Some(element @ _))) => { - match encode_webm_element(&element, buffer) { + match encode_webm_element(element, buffer) { Ok(_) => continue, Err(err) => ( Err(ChunkingError::IoError(err)), @@ -152,7 +152,7 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker cluster_head.keyframe = true; } cluster_head.observe_simpleblock_timecode(block.timecode); - match encode_webm_element(&WebmElement::SimpleBlock(*block), buffer) { + match encode_webm_element(WebmElement::SimpleBlock(*block), buffer) { Ok(_) => continue, Err(err) => ( Err(ChunkingError::IoError(err)), @@ -164,7 +164,7 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker Ok(Async::Ready(Some(WebmElement::Void))) => continue, Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => continue, Ok(Async::Ready(Some(element @ _))) => { - match encode_webm_element(&element, buffer) { + match encode_webm_element(element, buffer) { Ok(_) => continue, Err(err) => ( Err(ChunkingError::IoError(err)), diff --git a/src/webm.rs b/src/webm.rs index 907e0f2..c66bb5d 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -86,8 +86,8 @@ fn decode_simple_block(bytes: &[u8]) -> Result { } } -pub fn encode_simple_block(block: &SimpleBlock, output: &mut T) -> IoResult<()> { - let &SimpleBlock { +pub fn encode_simple_block(block: SimpleBlock, output: &mut T) -> IoResult<()> { + let SimpleBlock { track, timecode, flags, @@ -111,18 +111,18 @@ pub fn encode_simple_block(block: &SimpleBlock, output: &mut T) -> IoR output.write_all(data) } -pub fn encode_webm_element(element: &WebmElement, output: &mut T) -> IoResult<()> { +pub fn encode_webm_element(element: WebmElement, output: &mut T) -> IoResult<()> { match element { - &WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { + WebmElement::EbmlHead => encode_element(EBML_HEAD_ID, output, |output| { encode_bytes(DOC_TYPE_ID, "webm".as_bytes(), output) }), - &WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), - &WebmElement::SeekHead => Ok(()), - &WebmElement::Cues => Ok(()), - &WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), - &WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), - &WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), - &WebmElement::SimpleBlock(ref block) => encode_simple_block(block, output), + WebmElement::Segment => encode_tag_header(SEGMENT_ID, Varint::Unknown, output), + WebmElement::SeekHead => Ok(()), + WebmElement::Cues => Ok(()), + WebmElement::Tracks(data) => encode_bytes(TRACKS_ID, data, output), + WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), + WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), + WebmElement::SimpleBlock(block) => encode_simple_block(block, output), _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From 45e9054f363a106d4e5d6dc5c2b3f57ac03eb4b7 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 20:19:28 -0400 Subject: [PATCH 117/164] Move redundant fixer implementation into example code --- src/bin/resynth.rs | 47 +++++++++++++++++++++++++++++++++++++++---- src/timecode_fixer.rs | 43 +-------------------------------------- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index c66e163..1289585 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -3,7 +3,6 @@ extern crate lab_ebml; use std::io::{Cursor, stdout, Write}; use lab_ebml::webm::*; use lab_ebml::webm::WebmElement::*; -use lab_ebml::timecode_fixer::TimecodeFixer; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); @@ -31,8 +30,7 @@ pub fn main() { } } - let mut output = Vec::new(); - let mut cursor = Cursor::new(output); + let mut cursor = Cursor::new(Vec::new()); let mut fixer = TimecodeFixer::new(); @@ -48,8 +46,49 @@ pub fn main() { encode_webm_element(fixer.process(element), &mut cursor).unwrap(); } - output = cursor.into_inner(); + let mut output = cursor.into_inner(); stdout().write_all(&output).unwrap(); output.clear(); } + +pub struct TimecodeFixer { + pub current_offset: u64, + pub last_cluster_base: u64, + pub last_observed_timecode: u64, + pub assumed_duration: u64 +} + +impl TimecodeFixer { + pub fn new() -> TimecodeFixer { + TimecodeFixer { + current_offset: 0, + last_cluster_base: 0, + last_observed_timecode: 0, + assumed_duration: 33 + } + } + + pub fn process<'b>(&mut self, element: &WebmElement<'b>) -> WebmElement<'b> { + match element { + &WebmElement::Timecode(timecode) => { + // detect a jump backwards in the source, meaning we need to recalculate our offset + if timecode < self.last_cluster_base { + let next_timecode = self.last_observed_timecode + self.assumed_duration; + self.current_offset = next_timecode - timecode; + } + + // remember the source timecode to detect future jumps + self.last_cluster_base = timecode; + + // return adjusted timecode + WebmElement::Timecode(timecode + self.current_offset) + }, + &WebmElement::SimpleBlock(block) => { + self.last_observed_timecode = self.last_cluster_base + (block.timecode as u64); + *element + }, + _ => *element + } + } +} diff --git a/src/timecode_fixer.rs b/src/timecode_fixer.rs index 07d1047..9974bbc 100644 --- a/src/timecode_fixer.rs +++ b/src/timecode_fixer.rs @@ -1,48 +1,7 @@ -use chunk::Chunk; use futures::Async; use futures::stream::Stream; -use webm::WebmElement; -pub struct TimecodeFixer { - pub current_offset: u64, - pub last_cluster_base: u64, - pub last_observed_timecode: u64, - pub assumed_duration: u64 -} - -impl TimecodeFixer { - pub fn new() -> TimecodeFixer { - TimecodeFixer { - current_offset: 0, - last_cluster_base: 0, - last_observed_timecode: 0, - assumed_duration: 33 - } - } - - pub fn process<'b>(&mut self, element: &WebmElement<'b>) -> WebmElement<'b> { - match element { - &WebmElement::Timecode(timecode) => { - // detect a jump backwards in the source, meaning we need to recalculate our offset - if timecode < self.last_cluster_base { - let next_timecode = self.last_observed_timecode + self.assumed_duration; - self.current_offset = next_timecode - timecode; - } - - // remember the source timecode to detect future jumps - self.last_cluster_base = timecode; - - // return adjusted timecode - WebmElement::Timecode(timecode + self.current_offset) - }, - &WebmElement::SimpleBlock(block) => { - self.last_observed_timecode = self.last_cluster_base + (block.timecode as u64); - *element - }, - _ => *element - } - } -} +use chunk::Chunk; pub struct ChunkTimecodeFixer { stream: S, From 2bdbe21a736fc657fb4bb6c619a6738365b33757 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 4 Apr 2018 20:26:02 -0400 Subject: [PATCH 118/164] Rename timecode_fixer to fixers --- src/bin/loop_server.rs | 2 +- src/{timecode_fixer.rs => fixers.rs} | 0 src/lib.rs | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) rename src/{timecode_fixer.rs => fixers.rs} (100%) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index c14e87a..4467d48 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -10,7 +10,7 @@ use futures::stream::once; use futures::stream::repeat; use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; -use lab_ebml::timecode_fixer::ChunkStream; +use lab_ebml::fixers::ChunkStream; use lab_ebml::stream_parser::StreamEbml; use hyper::{Get, StatusCode}; use hyper::header::ContentType; diff --git a/src/timecode_fixer.rs b/src/fixers.rs similarity index 100% rename from src/timecode_fixer.rs rename to src/fixers.rs diff --git a/src/lib.rs b/src/lib.rs index 2a39830..2a0ff22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,13 +2,13 @@ extern crate bytes; extern crate futures; -pub mod chunk; pub mod ebml; -mod iterator; +pub mod iterator; pub mod slice; pub mod stream_parser; -pub mod timecode_fixer; +pub mod chunk; +pub mod fixers; pub mod webm; pub use ebml::{EbmlError, FromEbml}; From 409e6eb029d02809893328c2ae3aa37eb3849cee Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 5 Apr 2018 19:57:34 -0400 Subject: [PATCH 119/164] Make control flow of chunker state machine slightly clearer. --- src/chunk.rs | 105 ++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 566277b..fa703c2 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -100,7 +100,10 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker fn poll(&mut self) -> Result>, Self::Error> { loop { - let (return_value, next_state) = match self.state { + let mut return_value = None; + let mut new_state = None; + + match self.state { ChunkerState::BuildingHeader(ref mut buffer) => { match self.source.poll_event() { Err(passthru) => return Err(ChunkingError::OtherError(passthru)), @@ -109,23 +112,22 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker Ok(Async::Ready(Some(WebmElement::Cluster))) => { let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); let header_chunk = Chunk::Headers {bytes: Arc::new(liberated_buffer.into_inner())}; - ( - Ok(Async::Ready(Some(header_chunk))), - ChunkerState::BuildingCluster( - ClusterHead::new(0), - Cursor::new(Vec::new()) - ) - ) + + return_value = Some(Ok(Async::Ready(Some(header_chunk)))); + new_state = Some(ChunkerState::BuildingCluster( + ClusterHead::new(0), + Cursor::new(Vec::new()) + )); }, - Ok(Async::Ready(Some(WebmElement::Info))) => continue, - Ok(Async::Ready(Some(WebmElement::Void))) => continue, - Ok(Async::Ready(Some(element @ _))) => { + Ok(Async::Ready(Some(WebmElement::Info))) => {}, + Ok(Async::Ready(Some(WebmElement::Void))) => {}, + Ok(Async::Ready(Some(element))) => { match encode_webm_element(element, buffer) { - Ok(_) => continue, - Err(err) => ( - Err(ChunkingError::IoError(err)), - ChunkerState::End - ) + Ok(_) => {}, + Err(err) => { + return_value = Some(Err(ChunkingError::IoError(err))); + new_state = Some(ChunkerState::End); + } } } } @@ -137,14 +139,12 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker Ok(Async::Ready(Some(WebmElement::Cluster))) => { let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); - ( - Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))), - ChunkerState::EmittingClusterBody(liberated_buffer.into_inner()) - ) + + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head))))); + new_state = Some(ChunkerState::EmittingClusterBody(liberated_buffer.into_inner())); }, Ok(Async::Ready(Some(WebmElement::Timecode(timecode)))) => { cluster_head.update_timecode(timecode); - continue; }, Ok(Async::Ready(Some(WebmElement::SimpleBlock(ref block)))) => { if (block.flags & 0b10000000) != 0 { @@ -153,59 +153,60 @@ impl<'a, S: EbmlEventSource> Stream for WebmChunker } cluster_head.observe_simpleblock_timecode(block.timecode); match encode_webm_element(WebmElement::SimpleBlock(*block), buffer) { - Ok(_) => continue, - Err(err) => ( - Err(ChunkingError::IoError(err)), - ChunkerState::End - ) + Ok(_) => {}, + Err(err) => { + return_value = Some(Err(ChunkingError::IoError(err))); + new_state = Some(ChunkerState::End); + } } }, - Ok(Async::Ready(Some(WebmElement::Info))) => continue, - Ok(Async::Ready(Some(WebmElement::Void))) => continue, - Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => continue, - Ok(Async::Ready(Some(element @ _))) => { + Ok(Async::Ready(Some(WebmElement::Info))) => {}, + Ok(Async::Ready(Some(WebmElement::Void))) => {}, + Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {}, + Ok(Async::Ready(Some(element))) => { match encode_webm_element(element, buffer) { - Ok(_) => continue, - Err(err) => ( - Err(ChunkingError::IoError(err)), - ChunkerState::End - ) + Ok(_) => {}, + Err(err) => { + return_value = Some(Err(ChunkingError::IoError(err))); + new_state = Some(ChunkerState::End); + } } }, Ok(Async::Ready(None)) => { // flush final Cluster on end of stream let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); - ( - Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head)))), - ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner()) - ) + + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head))))); + new_state = Some(ChunkerState::EmittingFinalClusterBody(liberated_buffer.into_inner())); } } }, ChunkerState::EmittingClusterBody(ref mut buffer) => { let liberated_buffer = mem::replace(buffer, Vec::new()); - ( - Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)}))), - ChunkerState::BuildingCluster( - ClusterHead::new(0), - Cursor::new(Vec::new()) - ) - ) + + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)})))); + new_state = Some(ChunkerState::BuildingCluster( + ClusterHead::new(0), + Cursor::new(Vec::new()) + )); }, ChunkerState::EmittingFinalClusterBody(ref mut buffer) => { // flush final Cluster on end of stream let liberated_buffer = mem::replace(buffer, Vec::new()); - ( - Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)}))), - ChunkerState::End - ) + + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_buffer)})))); + new_state = Some(ChunkerState::End); }, ChunkerState::End => return Ok(Async::Ready(None)) }; - self.state = next_state; - return return_value; + if let Some(new_state) = new_state { + self.state = new_state; + } + if let Some(return_value) = return_value { + return return_value; + } } } } From 6eca0b923db47f25e9b60cc49b458d16fd4106e0 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 5 Apr 2018 19:59:39 -0400 Subject: [PATCH 120/164] Zap unused lifetime --- src/chunk.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chunk.rs b/src/chunk.rs index fa703c2..09645f4 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -93,7 +93,7 @@ pub struct WebmChunker { state: ChunkerState } -impl<'a, S: EbmlEventSource> Stream for WebmChunker +impl Stream for WebmChunker { type Item = Chunk; type Error = ChunkingError; From 7d4a26dad5401c25c81d2440c61c2001d80965fe Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 5 Apr 2018 20:14:10 -0400 Subject: [PATCH 121/164] Make chunker handle additional headers (such as from concatenated files) --- src/chunk.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/chunk.rs b/src/chunk.rs index 09645f4..7ccdfa4 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -75,9 +75,13 @@ impl> AsRef<[u8]> for Chunk { #[derive(Debug)] enum ChunkerState { BuildingHeader(Cursor>), - // WIP ClusterHead & body buffer + // ClusterHead & body buffer BuildingCluster(ClusterHead, Cursor>), EmittingClusterBody(Vec), + EmittingClusterBodyBeforeNewHeader { + body: Vec, + new_header: Cursor> + }, EmittingFinalClusterBody(Vec), End } @@ -136,6 +140,26 @@ impl Stream for WebmChunker match self.source.poll_event() { Err(passthru) => return Err(ChunkingError::OtherError(passthru)), Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(Some(element @ WebmElement::EbmlHead))) + | Ok(Async::Ready(Some(element @ WebmElement::Segment))) => { + let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); + let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); + + let mut new_header_cursor = Cursor::new(Vec::new()); + match encode_webm_element(element, &mut new_header_cursor) { + Ok(_) => { + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head))))); + new_state = Some(ChunkerState::EmittingClusterBodyBeforeNewHeader{ + body: liberated_buffer.into_inner(), + new_header: new_header_cursor + }); + }, + Err(err) => { + return_value = Some(Err(ChunkingError::IoError(err))); + new_state = Some(ChunkerState::End); + } + } + } Ok(Async::Ready(Some(WebmElement::Cluster))) => { let liberated_cluster_head = mem::replace(cluster_head, ClusterHead::new(0)); let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); @@ -191,6 +215,13 @@ impl Stream for WebmChunker Cursor::new(Vec::new()) )); }, + ChunkerState::EmittingClusterBodyBeforeNewHeader { ref mut body, ref mut new_header } => { + let liberated_body = mem::replace(body, Vec::new()); + let liberated_header_cursor = mem::replace(new_header, Cursor::new(Vec::new())); + + return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterBody {bytes: Arc::new(liberated_body)})))); + new_state = Some(ChunkerState::BuildingHeader(liberated_header_cursor)); + }, ChunkerState::EmittingFinalClusterBody(ref mut buffer) => { // flush final Cluster on end of stream let liberated_buffer = mem::replace(buffer, Vec::new()); From 9ca384f9da2023fc96bd973e32fed7136ef9dd4c Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 5 Apr 2018 20:16:37 -0400 Subject: [PATCH 122/164] streamline loop_server stream construction --- src/bin/loop_server.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 4467d48..cdfca0e 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -6,7 +6,6 @@ use std::env::args; use std::net::ToSocketAddrs; use futures::future::FutureResult; -use futures::stream::once; use futures::stream::repeat; use futures::stream::Stream; use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; @@ -32,11 +31,8 @@ impl Service for WebmServer { let response = match (req.method(), req.path()) { (&Get, "/loop") => { let stream: BodyStream> = Box::new( - repeat(()).take(3) - .map(|()| { - once::<&[u8], ()>(Ok(SRC_FILE)).parse_ebml().chunk_webm() - }).flatten() - .fix_timecodes() + repeat::<&[u8], ()>(SRC_FILE).take(5) + .parse_ebml().chunk_webm().fix_timecodes() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), ChunkingError::OtherError(_) => hyper::Error::Incomplete From b7ee4259057bcc8045efd885666caeb5ef45fdc3 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 7 Apr 2018 01:09:17 -0400 Subject: [PATCH 123/164] Tidy Stream extension trait impl --- notes.md | 1 - src/chunk.rs | 10 ++++------ src/fixers.rs | 10 ++++------ src/stream_parser.rs | 10 ++++------ 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/notes.md b/notes.md index 626e18d..ccac46b 100644 --- a/notes.md +++ b/notes.md @@ -1,3 +1,2 @@ * support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) * rustfmt modules -* bytestream source for ebml parsing diff --git a/src/chunk.rs b/src/chunk.rs index 7ccdfa4..7bfdf35 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -242,12 +242,8 @@ impl Stream for WebmChunker } } -pub trait WebmStream { - fn chunk_webm(self) -> WebmChunker; -} - -impl<'a, T: EbmlEventSource> WebmStream for T { - fn chunk_webm(self) -> WebmChunker { +pub trait WebmStream where Self: Sized + EbmlEventSource { + fn chunk_webm(self) -> WebmChunker { WebmChunker { source: self, state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())) @@ -255,6 +251,8 @@ impl<'a, T: EbmlEventSource> WebmStream for T { } } +impl WebmStream for T {} + #[cfg(test)] mod tests { diff --git a/src/fixers.rs b/src/fixers.rs index 9974bbc..4d73050 100644 --- a/src/fixers.rs +++ b/src/fixers.rs @@ -42,12 +42,8 @@ impl> Stream for ChunkTimecodeFixer } } -pub trait ChunkStream { - fn fix_timecodes(self) -> ChunkTimecodeFixer; -} - -impl> ChunkStream for T { - fn fix_timecodes(self) -> ChunkTimecodeFixer { +pub trait ChunkStream where Self : Sized + Stream { + fn fix_timecodes(self) -> ChunkTimecodeFixer { ChunkTimecodeFixer { stream: self, current_offset: 0, @@ -57,3 +53,5 @@ impl> ChunkStream for T { } } } + +impl> ChunkStream for T {} diff --git a/src/stream_parser.rs b/src/stream_parser.rs index ee8b120..278410a 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -18,12 +18,8 @@ pub struct EbmlStreamingParser { last_read: usize } -pub trait StreamEbml, S: Stream> { - fn parse_ebml(self) -> EbmlStreamingParser; -} - -impl, S: Stream> StreamEbml for S { - fn parse_ebml(self) -> EbmlStreamingParser { +pub trait StreamEbml where Self: Sized + Stream, Self::Item: AsRef<[u8]> { + fn parse_ebml(self) -> EbmlStreamingParser { EbmlStreamingParser { stream: self, buffer: BytesMut::new(), @@ -32,6 +28,8 @@ impl, S: Stream> StreamEbml for S { } } +impl, S: Stream> StreamEbml for S {} + impl, S: Stream> EbmlStreamingParser { pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, ParsingError> { // release buffer from previous event From ff8d4e912661449e1ac7a55d6e9603211c660711 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 8 Apr 2018 02:31:00 -0400 Subject: [PATCH 124/164] Break out "find starting point" fixer operator --- src/bin/loop_server.rs | 2 +- src/fixers.rs | 69 +++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index cdfca0e..2d38daa 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -32,7 +32,7 @@ impl Service for WebmServer { (&Get, "/loop") => { let stream: BodyStream> = Box::new( repeat::<&[u8], ()>(SRC_FILE).take(5) - .parse_ebml().chunk_webm().fix_timecodes() + .parse_ebml().chunk_webm().fix_timecodes().find_starting_point() .map_err(|err| match err { ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), ChunkingError::OtherError(_) => hyper::Error::Incomplete diff --git a/src/fixers.rs b/src/fixers.rs index 4d73050..4957781 100644 --- a/src/fixers.rs +++ b/src/fixers.rs @@ -7,8 +7,7 @@ pub struct ChunkTimecodeFixer { stream: S, current_offset: u64, last_observed_timecode: u64, - assumed_duration: u64, - seen_header: bool + assumed_duration: u64 } impl> Stream for ChunkTimecodeFixer @@ -29,27 +28,75 @@ impl> Stream for ChunkTimecodeFixer cluster_head.update_timecode(start + self.current_offset); self.last_observed_timecode = cluster_head.end; }, - Ok(Async::Ready(Some(Chunk::Headers {..}))) => { - if self.seen_header { - return self.poll(); - } else { - self.seen_header = true; - } - }, _ => {} }; poll_chunk } } +pub struct StartingPointFinder { + stream: S, + seen_header: bool, + seen_keyframe: bool +} + +impl> Stream for StartingPointFinder +{ + type Item = S::Item; + type Error = S::Error; + + fn poll(&mut self) -> Result>, Self::Error> { + loop { + return match self.stream.poll() { + Ok(Async::Ready(Some(Chunk::ClusterHead(cluster_head)))) => { + if cluster_head.keyframe { + self.seen_keyframe = true; + } + + if self.seen_keyframe { + Ok(Async::Ready(Some(Chunk::ClusterHead(cluster_head)))) + } else { + continue; + } + }, + chunk @ Ok(Async::Ready(Some(Chunk::ClusterBody {..}))) => { + if self.seen_keyframe { + chunk + } else { + continue; + } + }, + chunk @ Ok(Async::Ready(Some(Chunk::Headers {..}))) => { + if self.seen_header { + // new stream starting, we don't need a new header but should wait for a safe spot to resume + self.seen_keyframe = false; + continue; + } else { + self.seen_header = true; + chunk + } + }, + chunk => chunk + } + }; + } +} + pub trait ChunkStream where Self : Sized + Stream { fn fix_timecodes(self) -> ChunkTimecodeFixer { ChunkTimecodeFixer { stream: self, current_offset: 0, last_observed_timecode: 0, - assumed_duration: 33, - seen_header: false + assumed_duration: 33 + } + } + + fn find_starting_point(self) -> StartingPointFinder { + StartingPointFinder { + stream: self, + seen_header: false, + seen_keyframe: false } } } From 065e653f86179617b52fde2912508caffabeaf31 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 15:50:51 -0400 Subject: [PATCH 125/164] channel.rs: First stab at core of a relay server --- Cargo.toml | 1 + src/channel.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 ++ 3 files changed, 106 insertions(+) create mode 100644 src/channel.rs diff --git a/Cargo.toml b/Cargo.toml index 280bd11..7c6aff3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,3 +7,4 @@ authors = ["Tangent 128 "] bytes = "0.4" futures = "0.1.20" hyper = "0.11.24" +odds = { version = "0.3.1", features = ["std-vec"] } diff --git a/src/channel.rs b/src/channel.rs new file mode 100644 index 0000000..961b441 --- /dev/null +++ b/src/channel.rs @@ -0,0 +1,102 @@ +use std::sync::{ + Arc, + Mutex +}; + +use futures::{ + Async, + AsyncSink, + Sink, + Stream, + sync::mpsc::{ + channel as mpsc_channel, + Sender, + Receiver + } +}; +use odds::vec::VecExt; + +use chunk::Chunk; + +/// A collection of listeners to a stream of WebM chunks. +/// Sending a chunk may fail due to a client being disconnected, +/// or simply failing to keep up with the stream buffer. In either +/// case, there's nothing practical the server can do to recover, +/// so the failing client is just dropped from the listener list. +pub struct Channel { + header_chunk: Option, + listeners: Vec> +} + +impl Channel { + pub fn new() -> Arc> { + Arc::new(Mutex::new(Channel { + header_chunk: None, + listeners: Vec::new() + })) + } +} + +pub struct Transmitter { + channel: Arc> +} + +impl Sink for Transmitter { + type SinkItem = Chunk; + type SinkError = (); // never errors, slow clients are simply dropped + + fn start_send(&mut self, chunk: Chunk) -> Result, ()> { + let mut channel = self.channel.lock().expect("Locking channel"); + + if let Chunk::Headers { .. } = chunk { + channel.header_chunk = Some(chunk.clone()); + } + + channel.listeners.retain_mut(|listener| listener.start_send(chunk.clone()).is_ok()); + + Ok(AsyncSink::Ready) + } + fn poll_complete(&mut self) -> Result, ()> { + let mut channel = self.channel.lock().expect("Locking channel"); + + channel.listeners.retain_mut(|listener| listener.poll_complete().is_ok()); + + Ok(Async::Ready(())) + } +} + +pub struct Listener { + /// not used in operation, but its refcount keeps the channel alive when there's no Transmitter + _channel: Arc>, + receiver: Receiver +} + +impl Listener { + pub fn new(channel_arc: Arc>) -> Self { + let (mut sender, receiver) = mpsc_channel(5); + + { + let mut channel = channel_arc.lock().expect("Locking channel"); + + if let Some(ref chunk) = channel.header_chunk { + sender.start_send(chunk.clone()).expect("Queuing existing header chunk"); + } + + channel.listeners.push(sender); + } + + Listener { + _channel: channel_arc, + receiver: receiver + } + } +} + +impl Stream for Listener { + type Item = Chunk; + type Error = (); // no transmitter errors are exposed to the listeners + + fn poll(&mut self) -> Result>, ()> { + self.receiver.poll() + } +} diff --git a/src/lib.rs b/src/lib.rs index 2a0ff22..b7cde9f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ extern crate bytes; extern crate futures; +extern crate odds; pub mod ebml; pub mod iterator; @@ -11,6 +12,8 @@ pub mod chunk; pub mod fixers; pub mod webm; +pub mod channel; + pub use ebml::{EbmlError, FromEbml}; #[cfg(test)] From adea1e4389430cfe88984520b950714faf717e45 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 17:53:32 -0400 Subject: [PATCH 126/164] Remove unused generic parameter from Chunk --- src/bin/loop_server.rs | 6 +++--- src/chunk.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 2d38daa..81c4948 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -20,17 +20,17 @@ const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); #[derive(Clone)] struct WebmServer; -type BodyStream = Box, Error = hyper::Error>>; +type BodyStream = Box>; impl Service for WebmServer { type Request = Request; - type Response = Response>>; + type Response = Response; type Error = hyper::Error; type Future = FutureResult; fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { - let stream: BodyStream> = Box::new( + let stream: BodyStream = Box::new( repeat::<&[u8], ()>(SRC_FILE).take(5) .parse_ebml().chunk_webm().fix_timecodes().find_starting_point() .map_err(|err| match err { diff --git a/src/chunk.rs b/src/chunk.rs index 7bfdf35..8e37bc3 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -52,17 +52,17 @@ impl AsRef<[u8]> for ClusterHead { } #[derive(Clone, Debug)] -pub enum Chunk = Vec> { +pub enum Chunk { Headers { - bytes: Arc + bytes: Arc> }, ClusterHead(ClusterHead), ClusterBody { - bytes: Arc + bytes: Arc> } } -impl> AsRef<[u8]> for Chunk { +impl AsRef<[u8]> for Chunk { fn as_ref(&self) -> &[u8] { match self { &Chunk::Headers {ref bytes, ..} => bytes.as_ref().as_ref(), From 075c840a461eea69a7194942d0a16197f6c80dc6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 18:26:52 -0400 Subject: [PATCH 127/164] Use Never type & add missing Transmitter::new() to channel.rs --- src/channel.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/channel.rs b/src/channel.rs index 961b441..80d7725 100644 --- a/src/channel.rs +++ b/src/channel.rs @@ -18,6 +18,8 @@ use odds::vec::VecExt; use chunk::Chunk; +pub enum Never {} + /// A collection of listeners to a stream of WebM chunks. /// Sending a chunk may fail due to a client being disconnected, /// or simply failing to keep up with the stream buffer. In either @@ -41,11 +43,19 @@ pub struct Transmitter { channel: Arc> } +impl Transmitter { + pub fn new(channel_arc: Arc>) -> Self { + Transmitter { + channel: channel_arc + } + } +} + impl Sink for Transmitter { type SinkItem = Chunk; - type SinkError = (); // never errors, slow clients are simply dropped + type SinkError = Never; // never errors, slow clients are simply dropped - fn start_send(&mut self, chunk: Chunk) -> Result, ()> { + fn start_send(&mut self, chunk: Chunk) -> Result, Never> { let mut channel = self.channel.lock().expect("Locking channel"); if let Chunk::Headers { .. } = chunk { @@ -56,7 +66,7 @@ impl Sink for Transmitter { Ok(AsyncSink::Ready) } - fn poll_complete(&mut self) -> Result, ()> { + fn poll_complete(&mut self) -> Result, Never> { let mut channel = self.channel.lock().expect("Locking channel"); channel.listeners.retain_mut(|listener| listener.poll_complete().is_ok()); @@ -94,9 +104,9 @@ impl Listener { impl Stream for Listener { type Item = Chunk; - type Error = (); // no transmitter errors are exposed to the listeners + type Error = Never; // no transmitter errors are exposed to the listeners - fn poll(&mut self) -> Result>, ()> { - self.receiver.poll() + fn poll(&mut self) -> Result>, Never> { + Ok(self.receiver.poll().expect("Channel receiving can't error")) } } From 96f6ec8115aa699690a02fef7eebc264b58397c5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 21:00:36 -0400 Subject: [PATCH 128/164] bit of debug support --- src/stream_parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stream_parser.rs b/src/stream_parser.rs index 278410a..a2c99a5 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -7,6 +7,7 @@ use ebml::EbmlError; use ebml::EbmlEventSource; use ebml::FromEbml; +#[derive(Debug)] pub enum ParsingError { EbmlError(EbmlError), OtherError(E) From bf0f727b03051552cc805f1f4957f312d68b0aed Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 21:00:51 -0400 Subject: [PATCH 129/164] Stub out a one-channel video relay server --- src/bin/loop_server.rs | 8 +-- src/bin/relay_server.rs | 119 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 src/bin/relay_server.rs diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index 81c4948..a01a589 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -17,16 +17,16 @@ use hyper::server::{Http, Request, Response, Service}; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); -#[derive(Clone)] -struct WebmServer; +struct LoopServer; type BodyStream = Box>; -impl Service for WebmServer { +impl Service for LoopServer { type Request = Request; type Response = Response; type Error = hyper::Error; type Future = FutureResult; + fn call(&self, req: Request) -> Self::Future { let response = match (req.method(), req.path()) { (&Get, "/loop") => { @@ -53,5 +53,5 @@ impl Service for WebmServer { pub fn main() { let addr = args().nth(1).expect("Need binding address+port").to_socket_addrs().unwrap().next().unwrap(); - Http::new().bind(&addr, move || Ok(WebmServer)).unwrap().run().unwrap(); + Http::new().bind(&addr, move || Ok(LoopServer)).unwrap().run().unwrap(); } diff --git a/src/bin/relay_server.rs b/src/bin/relay_server.rs new file mode 100644 index 0000000..6f80d01 --- /dev/null +++ b/src/bin/relay_server.rs @@ -0,0 +1,119 @@ +extern crate futures; +extern crate hyper; +extern crate lab_ebml; + +use std::env::args; +use std::io::ErrorKind; +use std::net::ToSocketAddrs; +use std::sync::{ + Arc, + Mutex +}; + +use futures::{ + Future, + Stream, + Sink, + future::{ + FutureResult, + ok + }, + stream::empty +}; +use lab_ebml::{ + channel::{ + Channel, + Listener, + Transmitter + }, + chunk::{Chunk, WebmStream, ChunkingError}, + fixers::ChunkStream, + stream_parser::StreamEbml +}; +use hyper::{ + Error as HyperError, + Get, + Head, + Post, + StatusCode, + header::ContentType, + server::{Http, Request, Response, Service} +}; + +type BodyStream = Box>; + +struct RelayServer(Arc>); + +impl RelayServer { + fn get_channel(&self) -> Arc> { + self.0.clone() + } + + fn get_stream(&self) -> BodyStream { + Box::new( + Listener::new(self.get_channel()) + .fix_timecodes() + .find_starting_point() + .map_err(|err| match err {}) + ) + } + + fn post_stream, S: Stream + 'static>(&self, stream: S) -> BodyStream { + let source = stream.parse_ebml().chunk_webm(); + let sink = Transmitter::new(self.get_channel()); + + Box::new( + source.forward(sink.sink_map_err(|err| match err {})) + .into_stream() + .map(|_| empty()) + .map_err(|err| { + let io_err = match err { + ChunkingError::IoError(io_err) => io_err, + ChunkingError::OtherError(_) => ErrorKind::InvalidData.into() + }; + println!("Post failed: {}", &io_err); + io_err + }) + .flatten() + ) + } +} + +impl Service for RelayServer { + type Request = Request; + type Response = Response; + type Error = HyperError; + type Future = FutureResult; + + fn call(&self, request: Request) -> Self::Future { + let (method, uri, _http_version, _headers, request_body) = request.deconstruct(); + + eprintln!("New {} Request: {}", method, uri.path()); + + ok(match (method, uri.path()) { + (Head, "/live") => { + Response::new() + .with_header(ContentType("video/webm".parse().unwrap())) + }, + (Get, "/live") => { + Response::new() + .with_header(ContentType("video/webm".parse().unwrap())) + .with_body(self.get_stream()) + }, + (Post, "/live") => { + Response::new() + .with_body(self.post_stream(request_body)) + }, + _ => { + Response::new() + .with_status(StatusCode::NotFound) + } + }) + } +} + +pub fn main() { + let single_channel = Channel::new(); + let addr = args().nth(1).expect("Need binding address+port").to_socket_addrs().unwrap().next().unwrap(); + Http::new().bind(&addr, move || Ok(RelayServer(single_channel.clone()))).unwrap().run().unwrap(); +} From 497060323654e13bf54c63ad054a9e68bf69a328 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 9 Apr 2018 22:52:01 -0400 Subject: [PATCH 130/164] Drop unknown elements when building header --- src/chunk.rs | 1 + src/webm.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/chunk.rs b/src/chunk.rs index 8e37bc3..7a220b1 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -125,6 +125,7 @@ impl Stream for WebmChunker }, Ok(Async::Ready(Some(WebmElement::Info))) => {}, Ok(Async::Ready(Some(WebmElement::Void))) => {}, + Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {}, Ok(Async::Ready(Some(element))) => { match encode_webm_element(element, buffer) { Ok(_) => {}, diff --git a/src/webm.rs b/src/webm.rs index c66bb5d..5386b4a 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -123,7 +123,9 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T WebmElement::Cluster => encode_tag_header(CLUSTER_ID, Varint::Unknown, output), WebmElement::Timecode(time) => encode_integer(TIMECODE_ID, time, output), WebmElement::SimpleBlock(block) => encode_simple_block(block, output), - _ => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) + WebmElement::Void => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)), + WebmElement::Info => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)), + WebmElement::Unknown(_) => Err(IoError::new(ErrorKind::InvalidInput, WriteError::OutOfRange)) } } From 2310aabe2faf7d7700a15b065d50255c901e7600 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Tue, 10 Apr 2018 01:53:58 -0400 Subject: [PATCH 131/164] Rename crate to webmetro --- Cargo.toml | 2 +- src/bin/dump.rs | 6 +++--- src/bin/loop_server.rs | 8 ++++---- src/bin/relay_server.rs | 22 +++++++++++----------- src/bin/resynth.rs | 7 ++++--- src/bin/stub.rs | 4 ++-- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c6aff3..a79b6c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "lab_ebml" +name = "webmetro" version = "0.1.0" authors = ["Tangent 128 "] diff --git a/src/bin/dump.rs b/src/bin/dump.rs index 096b42d..4512dc4 100644 --- a/src/bin/dump.rs +++ b/src/bin/dump.rs @@ -1,11 +1,11 @@ -extern crate lab_ebml; +extern crate webmetro; use std::env::args; use std::fs::File; use std::io::Read; use std::path::Path; -use lab_ebml::webm::{ parse_webm, SimpleBlock }; -use lab_ebml::webm::WebmElement::*; +use webmetro::webm::{ parse_webm, SimpleBlock }; +use webmetro::webm::WebmElement::*; pub fn main() { let mut args = args(); diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs index a01a589..c290944 100644 --- a/src/bin/loop_server.rs +++ b/src/bin/loop_server.rs @@ -1,6 +1,6 @@ extern crate futures; extern crate hyper; -extern crate lab_ebml; +extern crate webmetro; use std::env::args; use std::net::ToSocketAddrs; @@ -8,12 +8,12 @@ use std::net::ToSocketAddrs; use futures::future::FutureResult; use futures::stream::repeat; use futures::stream::Stream; -use lab_ebml::chunk::{Chunk, WebmStream, ChunkingError}; -use lab_ebml::fixers::ChunkStream; -use lab_ebml::stream_parser::StreamEbml; use hyper::{Get, StatusCode}; use hyper::header::ContentType; use hyper::server::{Http, Request, Response, Service}; +use webmetro::chunk::{Chunk, WebmStream, ChunkingError}; +use webmetro::fixers::ChunkStream; +use webmetro::stream_parser::StreamEbml; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); diff --git a/src/bin/relay_server.rs b/src/bin/relay_server.rs index 6f80d01..8addd87 100644 --- a/src/bin/relay_server.rs +++ b/src/bin/relay_server.rs @@ -1,6 +1,6 @@ extern crate futures; extern crate hyper; -extern crate lab_ebml; +extern crate webmetro; use std::env::args; use std::io::ErrorKind; @@ -20,16 +20,6 @@ use futures::{ }, stream::empty }; -use lab_ebml::{ - channel::{ - Channel, - Listener, - Transmitter - }, - chunk::{Chunk, WebmStream, ChunkingError}, - fixers::ChunkStream, - stream_parser::StreamEbml -}; use hyper::{ Error as HyperError, Get, @@ -39,6 +29,16 @@ use hyper::{ header::ContentType, server::{Http, Request, Response, Service} }; +use webmetro::{ + channel::{ + Channel, + Listener, + Transmitter + }, + chunk::{Chunk, WebmStream, ChunkingError}, + fixers::ChunkStream, + stream_parser::StreamEbml +}; type BodyStream = Box>; diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs index 1289585..be77627 100644 --- a/src/bin/resynth.rs +++ b/src/bin/resynth.rs @@ -1,8 +1,9 @@ -extern crate lab_ebml; +extern crate webmetro; use std::io::{Cursor, stdout, Write}; -use lab_ebml::webm::*; -use lab_ebml::webm::WebmElement::*; + +use webmetro::webm::*; +use webmetro::webm::WebmElement::*; const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); diff --git a/src/bin/stub.rs b/src/bin/stub.rs index b3e0951..0ecd41d 100644 --- a/src/bin/stub.rs +++ b/src/bin/stub.rs @@ -1,7 +1,7 @@ -extern crate lab_ebml; +extern crate webmetro; use std::io::{Cursor, stdout, Write}; -use lab_ebml::webm::*; +use webmetro::webm::*; pub fn main() { let mut cursor = Cursor::new(Vec::new()); From f3aa76243f3d2dbf1c51e1d17051adc2eee07fec Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 00:55:41 -0400 Subject: [PATCH 132/164] Create stub main binary, using clap argument parser --- .gitignore | 1 - Cargo.lock | 705 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 17 ++ 4 files changed, 723 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore index a9d37c5..eb5a316 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ target -Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..708b836 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,705 @@ +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "httparse" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazycell" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memoffset" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_cpus" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "odds" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rawpointer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rawslice 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unchecked-index 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rawpointer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rawslice" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rawpointer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "relay" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scoped-tls" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scopeguard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slab" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strsim" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "take" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-core" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-executor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-io" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-udp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicase" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "version_check" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "webmetro" +version = "0.1.0" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.24 (registry+https://github.com/rust-lang/crates.io-index)", + "odds 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" +"checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" +"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" +"checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" +"checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" +"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" +"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c1bdc73742c36f7f35ebcda81dbb33a7e0d33757d03a06d9ddca762712ec5ea2" +"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81" +"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" +"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5a3176836efa0b37f0e321b86672dfada1564aeb516fbed67b7c24050a0263" +"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +"checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" +"checksum hyper 0.11.24 (registry+https://github.com/rust-lang/crates.io-index)" = "df4dd5dae401458087396b6db7fabc4d6760aa456a5fa8e92bda549f39cae661" +"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" +"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" +"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" +"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e00e17be181010a91dbfefb01660b17311059dc8c7f48b9017677721e732bd" +"checksum mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "6d771e3ef92d58a8da8df7d6976bfca9371ed1de6619d9d5a5ce5b1f29b85bfe" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum odds 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a9a18d7081eb052145753e982d7b8de495f15f74636d0d963f09116581eab665" +"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum rawpointer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebac11a9d2e11f2af219b8b8d833b76b1ea0e054aa0e8d8e9e4cbde353bdf019" +"checksum rawslice 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "22b23b9f57ea250c6db4b21e2897b43ff08209217ca8260469fae6c0f9ad7e25" +"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" +"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8674d439c964889e2476f474a3bf198cc9e199e77499960893bac5de7e9218a4" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" +"checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" +"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" +"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" +"checksum tokio 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "65bd27f59c223e7c9e406bcdadb453e143bcf1242e162ae6c0f0eb6d14487306" +"checksum tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "799492ccba3d8ed5e41f2520a7cfd504cb65bbfe5fbbbd0012e335ae5f188051" +"checksum tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3aca092a94dc6e736819347a990a86ed734a6543a9d6f817929fa4dc8c4334e2" +"checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" +"checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +"checksum tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3cedc8e5af5131dc3423ffa4f877cce78ad25259a9a62de0613735a13ebc64b" +"checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +"checksum tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec9b094851aadd2caf83ba3ad8e8c4ce65a42104f7b94d9e6550023f0407853f" +"checksum tokio-threadpool 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2057ff8a75d33639f9ea1b4b85cb113c7bbf4e06d132f148521d12cb6daa1a22" +"checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" +"checksum unchecked-index 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" +"checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" +"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" +"checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" +"checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/Cargo.toml b/Cargo.toml index a79b6c4..280f28f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Tangent 128 "] [dependencies] bytes = "0.4" +clap = "2.31.2" futures = "0.1.20" hyper = "0.11.24" odds = { version = "0.3.1", features = ["std-vec"] } diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..1f44870 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,17 @@ +#[macro_use] +extern crate clap; + +use clap::{App, AppSettings}; + +fn main() { + let args = App::new("webmetro") + .version(crate_version!()) + .about("Utilities for broadcasting & relaying live WebM video/audio streams") + .setting(AppSettings::SubcommandRequired) + .setting(AppSettings::VersionlessSubcommands) + .get_matches(); + + match args.subcommand() { + _ => {} + } +} From 98f7f446f9299b45b0d17b10ce8aba3126ee18aa Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 01:39:28 -0400 Subject: [PATCH 133/164] Move relay_server into a subcommand --- src/commands/mod.rs | 1 + .../relay_server.rs => commands/relay.rs} | 21 ++++++++++++------- src/main.rs | 11 ++++++++-- 3 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 src/commands/mod.rs rename src/{bin/relay_server.rs => commands/relay.rs} (85%) diff --git a/src/commands/mod.rs b/src/commands/mod.rs new file mode 100644 index 0000000..6193dd9 --- /dev/null +++ b/src/commands/mod.rs @@ -0,0 +1 @@ +pub mod relay; diff --git a/src/bin/relay_server.rs b/src/commands/relay.rs similarity index 85% rename from src/bin/relay_server.rs rename to src/commands/relay.rs index 8addd87..b944c89 100644 --- a/src/bin/relay_server.rs +++ b/src/commands/relay.rs @@ -1,8 +1,3 @@ -extern crate futures; -extern crate hyper; -extern crate webmetro; - -use std::env::args; use std::io::ErrorKind; use std::net::ToSocketAddrs; use std::sync::{ @@ -10,6 +5,7 @@ use std::sync::{ Mutex }; +use clap::{App, Arg, ArgMatches, SubCommand}; use futures::{ Future, Stream, @@ -112,8 +108,19 @@ impl Service for RelayServer { } } -pub fn main() { +pub fn args() -> App<'static, 'static> { + SubCommand::with_name("relay") + .about("Hosts an HTTP-based relay server") + .arg(Arg::with_name("listen") + .help("The address:port to listen to") + .required(true)) +} + +pub fn run(args: &ArgMatches) { let single_channel = Channel::new(); - let addr = args().nth(1).expect("Need binding address+port").to_socket_addrs().unwrap().next().unwrap(); + + let addr_str = args.value_of("listen").unwrap(); + let addr = addr_str.to_socket_addrs().expect("parsing bind address").next().expect("resolving bind address"); + Http::new().bind(&addr, move || Ok(RelayServer(single_channel.clone()))).unwrap().run().unwrap(); } diff --git a/src/main.rs b/src/main.rs index 1f44870..bfd104a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,12 @@ -#[macro_use] -extern crate clap; +#[macro_use] extern crate clap; +extern crate futures; +extern crate hyper; +extern crate webmetro; + +mod commands; use clap::{App, AppSettings}; +use commands::{relay}; fn main() { let args = App::new("webmetro") @@ -9,9 +14,11 @@ fn main() { .about("Utilities for broadcasting & relaying live WebM video/audio streams") .setting(AppSettings::SubcommandRequired) .setting(AppSettings::VersionlessSubcommands) + .subcommand(relay::args()) .get_matches(); match args.subcommand() { + ("relay", Some(sub_args)) => relay::run(sub_args), _ => {} } } From 7563c3ef46fd437c169372cd8fb6bea48135f456 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 01:43:40 -0400 Subject: [PATCH 134/164] Support PUT streams too --- src/commands/relay.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index b944c89..93dd108 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -21,6 +21,7 @@ use hyper::{ Get, Head, Post, + Put, StatusCode, header::ContentType, server::{Http, Request, Response, Service} @@ -96,7 +97,7 @@ impl Service for RelayServer { .with_header(ContentType("video/webm".parse().unwrap())) .with_body(self.get_stream()) }, - (Post, "/live") => { + (Post, "/live") | (Put, "/live") => { Response::new() .with_body(self.post_stream(request_body)) }, From 97359801c2550d1f11448769f873aa65b0d8ce34 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 01:50:18 -0400 Subject: [PATCH 135/164] Print help by default --- src/commands/relay.rs | 2 +- src/main.rs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 93dd108..282f024 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -109,7 +109,7 @@ impl Service for RelayServer { } } -pub fn args() -> App<'static, 'static> { +pub fn options() -> App<'static, 'static> { SubCommand::with_name("relay") .about("Hosts an HTTP-based relay server") .arg(Arg::with_name("listen") diff --git a/src/main.rs b/src/main.rs index bfd104a..9975024 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,17 +8,22 @@ mod commands; use clap::{App, AppSettings}; use commands::{relay}; -fn main() { - let args = App::new("webmetro") +fn options() -> App<'static, 'static> { + App::new("webmetro") .version(crate_version!()) .about("Utilities for broadcasting & relaying live WebM video/audio streams") - .setting(AppSettings::SubcommandRequired) .setting(AppSettings::VersionlessSubcommands) - .subcommand(relay::args()) - .get_matches(); + .subcommand(relay::options()) +} + +fn main() { + let args = options().get_matches(); match args.subcommand() { ("relay", Some(sub_args)) => relay::run(sub_args), - _ => {} + _ => { + options().print_help().unwrap(); + println!("") + } } } From 9dbd59c31308cbae88931974f3d6439d801db4e5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 19:45:02 -0400 Subject: [PATCH 136/164] Use ? error handling in subcommands --- src/commands/relay.rs | 10 ++++++---- src/main.rs | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 282f024..fad253b 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -1,3 +1,4 @@ +use std::error::Error; use std::io::ErrorKind; use std::net::ToSocketAddrs; use std::sync::{ @@ -117,11 +118,12 @@ pub fn options() -> App<'static, 'static> { .required(true)) } -pub fn run(args: &ArgMatches) { +pub fn run(args: &ArgMatches) -> Result<(), Box> { let single_channel = Channel::new(); - let addr_str = args.value_of("listen").unwrap(); - let addr = addr_str.to_socket_addrs().expect("parsing bind address").next().expect("resolving bind address"); + let addr_str = args.value_of("listen").ok_or("Listen address wasn't provided")?; + let addr = addr_str.to_socket_addrs()?.next().ok_or("Listen address didn't resolve")?; - Http::new().bind(&addr, move || Ok(RelayServer(single_channel.clone()))).unwrap().run().unwrap(); + Http::new().bind(&addr, move || Ok(RelayServer(single_channel.clone())))?.run()?; + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 9975024..788cc7f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,8 @@ fn main() { ("relay", Some(sub_args)) => relay::run(sub_args), _ => { options().print_help().unwrap(); - println!("") + println!(""); + Ok(()) } - } + }.unwrap_or_else(|err| println!("Error: {}", err)); } From da54623006b600e7150a30c62365b6618d32efaf Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 21:54:02 -0400 Subject: [PATCH 137/164] Move bin/dump into a debug subcommand, read stdin instead of a file --- src/bin/dump.rs | 29 ----------------------- src/commands/dump.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++ src/commands/mod.rs | 1 + src/main.rs | 7 +++++- 4 files changed, 62 insertions(+), 30 deletions(-) delete mode 100644 src/bin/dump.rs create mode 100644 src/commands/dump.rs diff --git a/src/bin/dump.rs b/src/bin/dump.rs deleted file mode 100644 index 4512dc4..0000000 --- a/src/bin/dump.rs +++ /dev/null @@ -1,29 +0,0 @@ -extern crate webmetro; - -use std::env::args; -use std::fs::File; -use std::io::Read; -use std::path::Path; -use webmetro::webm::{ parse_webm, SimpleBlock }; -use webmetro::webm::WebmElement::*; - -pub fn main() { - let mut args = args(); - let _ = args.next(); - let filename = args.next().expect("Reading filename"); - - let mut buffer = Vec::new(); - let mut file = File::open(Path::new(&filename)).expect("Opening file"); - - file.read_to_end(&mut buffer).expect("Reading file contents"); - - for element in parse_webm(buffer.as_slice()) { - match element { - // suppress printing byte arrays - Tracks(slice) => println!("Tracks[{}]", slice.len()), - SimpleBlock(SimpleBlock {timecode, ..}) => println!("SimpleBlock@{}", timecode), - other => println!("{:?}", other) - } - } - -} diff --git a/src/commands/dump.rs b/src/commands/dump.rs new file mode 100644 index 0000000..af94cec --- /dev/null +++ b/src/commands/dump.rs @@ -0,0 +1,55 @@ +use std::{ + error::Error, + io::{self, prelude::*} +}; + +use clap::{App, AppSettings, ArgMatches, SubCommand}; +use futures::{ + Async, + stream::poll_fn +}; + +use webmetro::{ + stream_parser::StreamEbml, + webm::{ + SimpleBlock, + WebmElement::* + } +}; + +pub fn options() -> App<'static, 'static> { + SubCommand::with_name("dump") + .setting(AppSettings::Hidden) + .about("Dumps WebM parsing events from parsing stdin") +} + +pub fn run(_args: &ArgMatches) -> Result<(), Box> { + + let stdin = io::stdin(); + let mut buf_reader = stdin.lock(); + let mut read_bytes = 0; + + let mut events = poll_fn(|| { + buf_reader.consume(read_bytes); + buf_reader.fill_buf().map(|slice| { + read_bytes = slice.len(); + if read_bytes > 0 { + Async::Ready(Some(Into::>::into(slice))) + } else { + Async::Ready(None) + } + }) + }).parse_ebml(); + + // stdin is sync so Async::NotReady will never happen + while let Ok(Async::Ready(Some(element))) = events.poll_event() { + match element { + // suppress printing byte arrays + Tracks(slice) => println!("Tracks[{}]", slice.len()), + SimpleBlock(SimpleBlock {timecode, ..}) => println!("SimpleBlock@{}", timecode), + other => println!("{:?}", other) + } + } + + Ok(()) +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 6193dd9..98a684e 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1 +1,2 @@ +pub mod dump; pub mod relay; diff --git a/src/main.rs b/src/main.rs index 788cc7f..d91be28 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,10 @@ extern crate webmetro; mod commands; use clap::{App, AppSettings}; -use commands::{relay}; +use commands::{ + relay, + dump +}; fn options() -> App<'static, 'static> { App::new("webmetro") @@ -14,6 +17,7 @@ fn options() -> App<'static, 'static> { .about("Utilities for broadcasting & relaying live WebM video/audio streams") .setting(AppSettings::VersionlessSubcommands) .subcommand(relay::options()) + .subcommand(dump::options()) } fn main() { @@ -21,6 +25,7 @@ fn main() { match args.subcommand() { ("relay", Some(sub_args)) => relay::run(sub_args), + ("dump", Some(sub_args)) => dump::run(sub_args), _ => { options().print_help().unwrap(); println!(""); From f890437c178ddfee2b98578711a2a68970acd8a1 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 22:45:53 -0400 Subject: [PATCH 138/164] convert bin/stub into a sanity test --- src/bin/stub.rs | 28 ---------------------------- src/data/encode_webm_test.webm | Bin 0 -> 73 bytes src/lib.rs | 1 + src/webm.rs | 31 ++++++++++++++++++++++++++++++- 4 files changed, 31 insertions(+), 29 deletions(-) delete mode 100644 src/bin/stub.rs create mode 100644 src/data/encode_webm_test.webm diff --git a/src/bin/stub.rs b/src/bin/stub.rs deleted file mode 100644 index 0ecd41d..0000000 --- a/src/bin/stub.rs +++ /dev/null @@ -1,28 +0,0 @@ -extern crate webmetro; - -use std::io::{Cursor, stdout, Write}; -use webmetro::webm::*; - -pub fn main() { - let mut cursor = Cursor::new(Vec::new()); - - encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); - - encode_webm_element(WebmElement::Tracks(&[]), &mut cursor).unwrap(); - - encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); - - encode_webm_element(WebmElement::SimpleBlock(SimpleBlock { - track: 3, - flags: 0x0, - timecode: 123, - data: "Hello, World".as_bytes() - }), &mut cursor).unwrap(); - - encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); - encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); - - stdout().write_all(&cursor.get_ref()).unwrap(); -} diff --git a/src/data/encode_webm_test.webm b/src/data/encode_webm_test.webm new file mode 100644 index 0000000000000000000000000000000000000000..badfef2a3cd95bc6f299a7ac94e535cf309b1d6c GIT binary patch literal 73 zcmb1gy}ww1fq~trsiizMDOV!6A^pEt$hzzXdFO4V|DSg-K*8b(%?#BH9;rDw`8o>W N`9(P?C<>Tg005?S7@7b8 literal 0 HcmV?d00001 diff --git a/src/lib.rs b/src/lib.rs index b7cde9f..3e80f2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ mod tests { use futures::future::{ok, Future}; pub const TEST_FILE: &'static [u8] = include_bytes!("data/test1.webm"); + pub const ENCODE_WEBM_TEST_FILE: &'static [u8] = include_bytes!("data/encode_webm_test.webm"); #[test] fn hello_futures() { diff --git a/src/webm.rs b/src/webm.rs index 5386b4a..6a48b60 100644 --- a/src/webm.rs +++ b/src/webm.rs @@ -131,7 +131,10 @@ pub fn encode_webm_element(element: WebmElement, output: &mut T #[cfg(test)] mod tests { - use tests::TEST_FILE; + use tests::{ + TEST_FILE, + ENCODE_WEBM_TEST_FILE + }; use webm::*; #[test] @@ -188,4 +191,30 @@ mod tests { assert_eq!(iter.next(), Some(WebmElement::Cues)); assert_eq!(iter.next(), None); } + + #[test] + fn encode_webm_test() { + let mut cursor = Cursor::new(Vec::new()); + + encode_webm_element(WebmElement::EbmlHead, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Segment, &mut cursor).unwrap(); + + encode_webm_element(WebmElement::Tracks(&[]), &mut cursor).unwrap(); + + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(0), &mut cursor).unwrap(); + + encode_webm_element(WebmElement::SimpleBlock(SimpleBlock { + track: 3, + flags: 0x0, + timecode: 123, + data: "Hello, World".as_bytes() + }), &mut cursor).unwrap(); + + encode_webm_element(WebmElement::Cluster, &mut cursor).unwrap(); + encode_webm_element(WebmElement::Timecode(1000), &mut cursor).unwrap(); + + assert_eq!(cursor.get_ref(), &ENCODE_WEBM_TEST_FILE); + } + } From 9b1e61ff80f338183b88386bd2149e3436825f98 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Wed, 11 Apr 2018 22:47:03 -0400 Subject: [PATCH 139/164] delete bin/ toys that aren't especially useful anymore --- src/bin/loop_server.rs | 57 ------------------------- src/bin/resynth.rs | 95 ------------------------------------------ 2 files changed, 152 deletions(-) delete mode 100644 src/bin/loop_server.rs delete mode 100644 src/bin/resynth.rs diff --git a/src/bin/loop_server.rs b/src/bin/loop_server.rs deleted file mode 100644 index c290944..0000000 --- a/src/bin/loop_server.rs +++ /dev/null @@ -1,57 +0,0 @@ -extern crate futures; -extern crate hyper; -extern crate webmetro; - -use std::env::args; -use std::net::ToSocketAddrs; - -use futures::future::FutureResult; -use futures::stream::repeat; -use futures::stream::Stream; -use hyper::{Get, StatusCode}; -use hyper::header::ContentType; -use hyper::server::{Http, Request, Response, Service}; -use webmetro::chunk::{Chunk, WebmStream, ChunkingError}; -use webmetro::fixers::ChunkStream; -use webmetro::stream_parser::StreamEbml; - -const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); - -struct LoopServer; - -type BodyStream = Box>; - -impl Service for LoopServer { - type Request = Request; - type Response = Response; - type Error = hyper::Error; - type Future = FutureResult; - - fn call(&self, req: Request) -> Self::Future { - let response = match (req.method(), req.path()) { - (&Get, "/loop") => { - let stream: BodyStream = Box::new( - repeat::<&[u8], ()>(SRC_FILE).take(5) - .parse_ebml().chunk_webm().fix_timecodes().find_starting_point() - .map_err(|err| match err { - ChunkingError::IoError(io_err) => hyper::Error::Io(io_err), - ChunkingError::OtherError(_) => hyper::Error::Incomplete - }) - ); - Response::new() - .with_header(ContentType("video/webm".parse().unwrap())) - .with_body(stream) - }, - _ => { - Response::new() - .with_status(StatusCode::NotFound) - } - }; - futures::future::ok(response) - } -} - -pub fn main() { - let addr = args().nth(1).expect("Need binding address+port").to_socket_addrs().unwrap().next().unwrap(); - Http::new().bind(&addr, move || Ok(LoopServer)).unwrap().run().unwrap(); -} diff --git a/src/bin/resynth.rs b/src/bin/resynth.rs deleted file mode 100644 index be77627..0000000 --- a/src/bin/resynth.rs +++ /dev/null @@ -1,95 +0,0 @@ -extern crate webmetro; - -use std::io::{Cursor, stdout, Write}; - -use webmetro::webm::*; -use webmetro::webm::WebmElement::*; - -const SRC_FILE: &'static [u8] = include_bytes!("../data/test1.webm"); - -pub fn main() { - - let mut head = Vec::new(); - let mut body = Vec::new(); - - let mut reading_head = true; - - for element in parse_webm(SRC_FILE) { - match element { - Cluster => reading_head = false, - // TODO: skip elements not required for streaming - Info => continue, - Void => continue, - Unknown(_) => continue, - _ => (), - } - - if reading_head { - head.push(element); - } else { - body.push(element); - } - } - - let mut cursor = Cursor::new(Vec::new()); - - let mut fixer = TimecodeFixer::new(); - - for element in &head { - encode_webm_element(fixer.process(element), &mut cursor).unwrap(); - } - - for element in &body { - encode_webm_element(fixer.process(element), &mut cursor).unwrap(); - } - - for element in &body { - encode_webm_element(fixer.process(element), &mut cursor).unwrap(); - } - - let mut output = cursor.into_inner(); - stdout().write_all(&output).unwrap(); - output.clear(); - -} - -pub struct TimecodeFixer { - pub current_offset: u64, - pub last_cluster_base: u64, - pub last_observed_timecode: u64, - pub assumed_duration: u64 -} - -impl TimecodeFixer { - pub fn new() -> TimecodeFixer { - TimecodeFixer { - current_offset: 0, - last_cluster_base: 0, - last_observed_timecode: 0, - assumed_duration: 33 - } - } - - pub fn process<'b>(&mut self, element: &WebmElement<'b>) -> WebmElement<'b> { - match element { - &WebmElement::Timecode(timecode) => { - // detect a jump backwards in the source, meaning we need to recalculate our offset - if timecode < self.last_cluster_base { - let next_timecode = self.last_observed_timecode + self.assumed_duration; - self.current_offset = next_timecode - timecode; - } - - // remember the source timecode to detect future jumps - self.last_cluster_base = timecode; - - // return adjusted timecode - WebmElement::Timecode(timecode + self.current_offset) - }, - &WebmElement::SimpleBlock(block) => { - self.last_observed_timecode = self.last_cluster_base + (block.timecode as u64); - *element - }, - _ => *element - } - } -} From e61244bce360d32109dca5c2fa00b14d3b6fb4e3 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Apr 2018 00:14:16 -0400 Subject: [PATCH 140/164] Factor StdinStream out of dump applet --- src/commands/dump.rs | 23 ++++------------------ src/commands/mod.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/commands/dump.rs b/src/commands/dump.rs index af94cec..fa2fb40 100644 --- a/src/commands/dump.rs +++ b/src/commands/dump.rs @@ -1,14 +1,12 @@ use std::{ error::Error, - io::{self, prelude::*} + io }; use clap::{App, AppSettings, ArgMatches, SubCommand}; -use futures::{ - Async, - stream::poll_fn -}; +use futures::Async; +use super::StdinStream; use webmetro::{ stream_parser::StreamEbml, webm::{ @@ -26,20 +24,7 @@ pub fn options() -> App<'static, 'static> { pub fn run(_args: &ArgMatches) -> Result<(), Box> { let stdin = io::stdin(); - let mut buf_reader = stdin.lock(); - let mut read_bytes = 0; - - let mut events = poll_fn(|| { - buf_reader.consume(read_bytes); - buf_reader.fill_buf().map(|slice| { - read_bytes = slice.len(); - if read_bytes > 0 { - Async::Ready(Some(Into::>::into(slice))) - } else { - Async::Ready(None) - } - }) - }).parse_ebml(); + let mut events = StdinStream::new(stdin.lock()).parse_ebml(); // stdin is sync so Async::NotReady will never happen while let Ok(Async::Ready(Some(element))) = events.poll_event() { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 98a684e..3ff2c18 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,2 +1,48 @@ +use std::io::{ + Error as IoError, + StdinLock, + prelude::* +}; + +use futures::{ + Async, + stream::Stream +}; + pub mod dump; pub mod relay; + +/// A hackish adapter that makes chunks of bytes from stdin available as a Stream; +/// is NOT actually async, and just uses blocking read. Buffers aren't optimized either +/// and copy more than necessary. +pub struct StdinStream<'a> { + buf_reader: StdinLock<'a>, + read_bytes: usize +} + +impl<'a> StdinStream<'a> { + pub fn new(lock: StdinLock<'a>) -> Self { + StdinStream { + buf_reader: lock, + read_bytes: 0 + } + } +} + +impl<'a> Stream for StdinStream<'a> { + type Item = Vec; + type Error = IoError; + + fn poll(&mut self) -> Result>, Self::Error> { + self.buf_reader.consume(self.read_bytes); + let read_bytes = &mut self.read_bytes; + self.buf_reader.fill_buf().map(|slice| { + *read_bytes = slice.len(); + if *read_bytes > 0 { + Async::Ready(Some(Into::>::into(slice))) + } else { + Async::Ready(None) + } + }) + } +} From bac34e94c511ac1b2b24898a86e0e6b1d571b7c0 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Apr 2018 01:59:28 -0400 Subject: [PATCH 141/164] impl Error for error types --- src/chunk.rs | 24 +++++++++++++++++++++--- src/ebml.rs | 15 +++++++++++++++ src/stream_parser.rs | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 7a220b1..0b78b5e 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,7 +1,11 @@ use futures::{Async, Stream}; -use std::io::Cursor; -use std::mem; -use std::sync::Arc; +use std::{ + error::Error, + fmt::{Display, Formatter, Result as FmtResult}, + io::Cursor, + mem, + sync::Arc +}; use ebml::EbmlEventSource; use webm::*; @@ -92,6 +96,20 @@ pub enum ChunkingError { OtherError(E) } +impl Display for ChunkingError { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + write!(f, "Chunking error: {}", self.description()) + } +} +impl Error for ChunkingError { + fn description(&self) -> &str { + match self { + &ChunkingError::IoError(ref err) => err.description(), + &ChunkingError::OtherError(ref err) => err.description() + } + } +} + pub struct WebmChunker { source: S, state: ChunkerState diff --git a/src/ebml.rs b/src/ebml.rs index a9fc150..cfb9876 100644 --- a/src/ebml.rs +++ b/src/ebml.rs @@ -15,6 +15,21 @@ pub enum EbmlError { UnknownElementLength, CorruptPayload, } +impl Display for EbmlError { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + write!(f, "{}", self.description()) + } +} +impl ErrorTrait for EbmlError { + fn description(&self) -> &str { + match self { + &EbmlError::CorruptVarint => "EBML Varint could not be parsed", + &EbmlError::UnknownElementId => "EBML element ID was \"unknown\"", + &EbmlError::UnknownElementLength => "EBML element length was \"unknown\" for an element not allowing that", + &EbmlError::CorruptPayload => "EBML element payload could not be parsed", + } + } +} #[derive(Debug, PartialEq)] pub enum WriteError { diff --git a/src/stream_parser.rs b/src/stream_parser.rs index a2c99a5..645ccf3 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -1,3 +1,8 @@ +use std::{ + error::Error, + fmt::{Display, Formatter, Result as FmtResult} +}; + use bytes::BytesMut; use bytes::BufMut; use futures::Async; @@ -12,6 +17,19 @@ pub enum ParsingError { EbmlError(EbmlError), OtherError(E) } +impl Display for ParsingError { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + write!(f, "Parsing error: {}", self.description()) + } +} +impl Error for ParsingError { + fn description(&self) -> &str { + match self { + &ParsingError::EbmlError(ref err) => err.description(), + &ParsingError::OtherError(ref err) => err.description() + } + } +} pub struct EbmlStreamingParser { stream: S, From 413375102e89ab503595f6419372d969c38384a6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Apr 2018 02:03:46 -0400 Subject: [PATCH 142/164] Implement core of filter subcommand --- src/commands/filter.rs | 43 ++++++++++++++++++++++++++++++++++++++++++ src/commands/mod.rs | 1 + src/main.rs | 3 +++ 3 files changed, 47 insertions(+) create mode 100644 src/commands/filter.rs diff --git a/src/commands/filter.rs b/src/commands/filter.rs new file mode 100644 index 0000000..2e85c53 --- /dev/null +++ b/src/commands/filter.rs @@ -0,0 +1,43 @@ +use std::{ + error::Error, + io, + io::prelude::* +}; + +use clap::{App, ArgMatches, SubCommand}; +use futures::Stream; + +use super::StdinStream; +use webmetro::{ + chunk::{ + Chunk, + WebmStream + }, + fixers::ChunkStream, + stream_parser::StreamEbml +}; + +pub fn options() -> App<'static, 'static> { + SubCommand::with_name("filter") + .about("Copies WebM from stdin to stdout, applying the same cleanup & stripping the relay server does.") +} + +pub fn run(_args: &ArgMatches) -> Result<(), Box> { + + let stdin = io::stdin(); + let chunk_stream: Box>> = Box::new( + StdinStream::new(stdin.lock()) + .parse_ebml() + .chunk_webm() + .map_err(|err| Box::new(err) as Box) + .fix_timecodes() + ); + + let stdout = io::stdout(); + let mut stdout_writer = stdout.lock(); + for chunk in chunk_stream.wait() { + stdout_writer.write_all(chunk?.as_ref())?; + } + + Ok(()) +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 3ff2c18..a4ca1a1 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -10,6 +10,7 @@ use futures::{ }; pub mod dump; +pub mod filter; pub mod relay; /// A hackish adapter that makes chunks of bytes from stdin available as a Stream; diff --git a/src/main.rs b/src/main.rs index d91be28..d2f70d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod commands; use clap::{App, AppSettings}; use commands::{ relay, + filter, dump }; @@ -17,6 +18,7 @@ fn options() -> App<'static, 'static> { .about("Utilities for broadcasting & relaying live WebM video/audio streams") .setting(AppSettings::VersionlessSubcommands) .subcommand(relay::options()) + .subcommand(filter::options()) .subcommand(dump::options()) } @@ -24,6 +26,7 @@ fn main() { let args = options().get_matches(); match args.subcommand() { + ("filter", Some(sub_args)) => filter::run(sub_args), ("relay", Some(sub_args)) => relay::run(sub_args), ("dump", Some(sub_args)) => dump::run(sub_args), _ => { From 6434db0f82bacd0e466315831a6f8b3a2914555f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Apr 2018 23:29:12 -0400 Subject: [PATCH 143/164] Stub out throttle filter --- src/commands/filter.rs | 13 ++++++++++--- src/fixers.rs | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/commands/filter.rs b/src/commands/filter.rs index 2e85c53..c3a4839 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -4,7 +4,7 @@ use std::{ io::prelude::* }; -use clap::{App, ArgMatches, SubCommand}; +use clap::{App, Arg, ArgMatches, SubCommand}; use futures::Stream; use super::StdinStream; @@ -20,12 +20,15 @@ use webmetro::{ pub fn options() -> App<'static, 'static> { SubCommand::with_name("filter") .about("Copies WebM from stdin to stdout, applying the same cleanup & stripping the relay server does.") + .arg(Arg::with_name("throttle") + .long("throttle") + .help("Slow down output to \"realtime\" speed as determined by the timestamps (useful for streaming)")) } -pub fn run(_args: &ArgMatches) -> Result<(), Box> { +pub fn run(args: &ArgMatches) -> Result<(), Box> { let stdin = io::stdin(); - let chunk_stream: Box>> = Box::new( + let mut chunk_stream: Box>> = Box::new( StdinStream::new(stdin.lock()) .parse_ebml() .chunk_webm() @@ -33,6 +36,10 @@ pub fn run(_args: &ArgMatches) -> Result<(), Box> { .fix_timecodes() ); + if args.is_present("throttle") { + chunk_stream = Box::new(chunk_stream.throttle()); + } + let stdout = io::stdout(); let mut stdout_writer = stdout.lock(); for chunk in chunk_stream.wait() { diff --git a/src/fixers.rs b/src/fixers.rs index 4957781..5925c26 100644 --- a/src/fixers.rs +++ b/src/fixers.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use futures::Async; use futures::stream::Stream; @@ -82,6 +84,21 @@ impl> Stream for StartingPointFinder } } +pub struct Throttle { + stream: S, + start_time: Instant +} + +impl> Stream for Throttle +{ + type Item = S::Item; + type Error = S::Error; + + fn poll(&mut self) -> Result>, Self::Error> { + self.stream.poll() + } +} + pub trait ChunkStream where Self : Sized + Stream { fn fix_timecodes(self) -> ChunkTimecodeFixer { ChunkTimecodeFixer { @@ -99,6 +116,13 @@ pub trait ChunkStream where Self : Sized + Stream { seen_keyframe: false } } + + fn throttle(self) -> Throttle { + Throttle { + stream: self, + start_time: Instant::now() + } + } } impl> ChunkStream for T {} From 15ebdb6b8c57bf416b228303f8ccfa27328633dc Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Thu, 12 Apr 2018 23:46:12 -0400 Subject: [PATCH 144/164] Update dependencies + officially add Tokio --- Cargo.lock | 94 ++++++++++++++++++++++++++++++------------------------ Cargo.toml | 3 +- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 708b836..e679726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,7 @@ name = "base64" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -40,7 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -48,7 +48,7 @@ name = "bytes" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -125,7 +125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -133,7 +133,7 @@ name = "futures-cpupool" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -144,12 +144,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "hyper" -version = "0.11.24" +version = "0.11.25" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -159,7 +159,7 @@ dependencies = [ "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -351,7 +351,7 @@ name = "relay" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -424,43 +424,44 @@ dependencies = [ [[package]] name = "tokio" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-threadpool 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-core" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-executor" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -469,7 +470,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -478,14 +479,14 @@ name = "tokio-proto" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -495,11 +496,11 @@ name = "tokio-reactor" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -508,7 +509,7 @@ name = "tokio-service" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -517,7 +518,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -526,15 +527,24 @@ dependencies = [ [[package]] name = "tokio-threadpool" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-timer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -543,7 +553,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -584,9 +594,10 @@ version = "0.1.0" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.24 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", "odds 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -633,7 +644,7 @@ dependencies = [ "checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" "checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" -"checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" +"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" "checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" @@ -643,10 +654,10 @@ dependencies = [ "checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" -"checksum futures 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5a3176836efa0b37f0e321b86672dfada1564aeb516fbed67b7c24050a0263" +"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" "checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" -"checksum hyper 0.11.24 (registry+https://github.com/rust-lang/crates.io-index)" = "df4dd5dae401458087396b6db7fabc4d6760aa456a5fa8e92bda549f39cae661" +"checksum hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)" = "549dbb86397490ce69d908425b9beebc85bbaad25157d67479d4995bb56fdf9a" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" @@ -682,15 +693,16 @@ dependencies = [ "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" -"checksum tokio 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "65bd27f59c223e7c9e406bcdadb453e143bcf1242e162ae6c0f0eb6d14487306" -"checksum tokio-core 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "799492ccba3d8ed5e41f2520a7cfd504cb65bbfe5fbbbd0012e335ae5f188051" -"checksum tokio-executor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3aca092a94dc6e736819347a990a86ed734a6543a9d6f817929fa4dc8c4334e2" +"checksum tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "be15ef40f675c9fe66e354d74c73f3ed012ca1aa14d65846a33ee48f1ae8d922" +"checksum tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "aeeffbbb94209023feaef3c196a41cbcdafa06b4a6f893f68779bb5e53796f71" +"checksum tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8cac2a7883ff3567e9d66bb09100d09b33d90311feca0206c7ca034bc0c55113" "checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" "checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" "checksum tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3cedc8e5af5131dc3423ffa4f877cce78ad25259a9a62de0613735a13ebc64b" "checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" "checksum tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec9b094851aadd2caf83ba3ad8e8c4ce65a42104f7b94d9e6550023f0407853f" -"checksum tokio-threadpool 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2057ff8a75d33639f9ea1b4b85cb113c7bbf4e06d132f148521d12cb6daa1a22" +"checksum tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3d05cdd6a78005e535d2b27c21521bdf91fbb321027a62d8e178929d18966d" +"checksum tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "29a89e4ad0c8f1e4c9860e605c38c69bfdad3cccd4ea446e58ff588c1c07a397" "checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" "checksum unchecked-index 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" "checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" diff --git a/Cargo.toml b/Cargo.toml index 280f28f..f2d5d16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,6 @@ authors = ["Tangent 128 "] bytes = "0.4" clap = "2.31.2" futures = "0.1.20" -hyper = "0.11.24" +hyper = "0.11.25" odds = { version = "0.3.1", features = ["std-vec"] } +tokio = "0.1.5" From a9932ee6a19b3352091be176e8657df8ecc450db Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 13 Apr 2018 19:01:40 -0400 Subject: [PATCH 145/164] Create unified error type --- src/error.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 56 insertions(+) create mode 100644 src/error.rs diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..2073d53 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,55 @@ +use std::{ + error::Error, + fmt::{ + Display, + Formatter, + Result as FmtResult + }, + io::Error as IoError +}; + +use ebml::EbmlError; + +#[derive(Debug)] +pub enum WebmetroError { + EbmlError(EbmlError), + IoError(IoError), + Unknown(Box) +} + +impl Display for WebmetroError { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + &WebmetroError::EbmlError(ref err) => err.fmt(f), + &WebmetroError::IoError(ref err) => err.fmt(f), + &WebmetroError::Unknown(ref err) => err.fmt(f), + } + } +} +impl Error for WebmetroError { + fn description(&self) -> &str { + match self { + &WebmetroError::EbmlError(ref err) => err.description(), + &WebmetroError::IoError(ref err) => err.description(), + &WebmetroError::Unknown(ref err) => err.description(), + } + } +} + +impl From for WebmetroError { + fn from(err: EbmlError) -> WebmetroError { + WebmetroError::EbmlError(err) + } +} + +impl From for WebmetroError { + fn from(err: IoError) -> WebmetroError { + WebmetroError::IoError(err) + } +} + +impl From> for WebmetroError { + fn from(err: Box) -> WebmetroError { + WebmetroError::Unknown(err) + } +} diff --git a/src/lib.rs b/src/lib.rs index 3e80f2b..1f064cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ extern crate futures; extern crate odds; pub mod ebml; +pub mod error; pub mod iterator; pub mod slice; pub mod stream_parser; From bf56789810020f410afd7db78a633238d01f273f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 03:34:59 -0400 Subject: [PATCH 146/164] Make EbmlStreamingParser use WebmetroErrors specifically --- src/commands/mod.rs | 6 +++--- src/commands/relay.rs | 8 ++++++-- src/stream_parser.rs | 44 ++++++++++--------------------------------- 3 files changed, 19 insertions(+), 39 deletions(-) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index a4ca1a1..46f739b 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,5 +1,4 @@ use std::io::{ - Error as IoError, StdinLock, prelude::* }; @@ -8,6 +7,7 @@ use futures::{ Async, stream::Stream }; +use webmetro::error::WebmetroError; pub mod dump; pub mod filter; @@ -32,7 +32,7 @@ impl<'a> StdinStream<'a> { impl<'a> Stream for StdinStream<'a> { type Item = Vec; - type Error = IoError; + type Error = WebmetroError; fn poll(&mut self) -> Result>, Self::Error> { self.buf_reader.consume(self.read_bytes); @@ -44,6 +44,6 @@ impl<'a> Stream for StdinStream<'a> { } else { Async::Ready(None) } - }) + }).map_err(WebmetroError::IoError) } } diff --git a/src/commands/relay.rs b/src/commands/relay.rs index fad253b..6fe841c 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -34,6 +34,7 @@ use webmetro::{ Transmitter }, chunk::{Chunk, WebmStream, ChunkingError}, + error::WebmetroError, fixers::ChunkStream, stream_parser::StreamEbml }; @@ -56,8 +57,11 @@ impl RelayServer { ) } - fn post_stream, S: Stream + 'static>(&self, stream: S) -> BodyStream { - let source = stream.parse_ebml().chunk_webm(); + fn post_stream, S: Stream + 'static>(&self, stream: S) -> BodyStream + where S::Error: Error { + let source = stream + .map_err(|err| WebmetroError::Unknown(err.into())) + .parse_ebml().chunk_webm(); let sink = Transmitter::new(self.get_channel()); Box::new( diff --git a/src/stream_parser.rs b/src/stream_parser.rs index 645ccf3..c1e4665 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -1,35 +1,11 @@ -use std::{ - error::Error, - fmt::{Display, Formatter, Result as FmtResult} -}; - use bytes::BytesMut; use bytes::BufMut; use futures::Async; use futures::stream::Stream; -use ebml::EbmlError; use ebml::EbmlEventSource; use ebml::FromEbml; - -#[derive(Debug)] -pub enum ParsingError { - EbmlError(EbmlError), - OtherError(E) -} -impl Display for ParsingError { - fn fmt(&self, f: &mut Formatter) -> FmtResult { - write!(f, "Parsing error: {}", self.description()) - } -} -impl Error for ParsingError { - fn description(&self) -> &str { - match self { - &ParsingError::EbmlError(ref err) => err.description(), - &ParsingError::OtherError(ref err) => err.description() - } - } -} +use error::WebmetroError; pub struct EbmlStreamingParser { stream: S, @@ -47,10 +23,10 @@ pub trait StreamEbml where Self: Sized + Stream, Self::Item: AsRef<[u8]> { } } -impl, S: Stream> StreamEbml for S {} +impl, S: Stream> StreamEbml for S {} -impl, S: Stream> EbmlStreamingParser { - pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, ParsingError> { +impl, S: Stream> EbmlStreamingParser { + pub fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, WebmetroError> { // release buffer from previous event self.buffer.advance(self.last_read); self.last_read = 0; @@ -60,9 +36,9 @@ impl, S: Stream> EbmlStreamingParser { Ok(None) => { // need to refill buffer, below }, - other => return other.map_err(ParsingError::EbmlError).and_then(move |_| { + other => return other.map_err(WebmetroError::EbmlError).and_then(move |_| { match T::decode_element(&self.buffer) { - Err(err) => Err(ParsingError::EbmlError(err)), + Err(err) => Err(WebmetroError::EbmlError(err)), Ok(None) => panic!("Buffer was supposed to have enough data to parse element, somehow did not."), Ok(Some((element, element_size))) => { self.last_read = element_size; @@ -72,7 +48,7 @@ impl, S: Stream> EbmlStreamingParser { }) } - match self.stream.poll().map_err(ParsingError::OtherError) { + match self.stream.poll() { Ok(Async::Ready(Some(chunk))) => { self.buffer.reserve(chunk.as_ref().len()); self.buffer.put_slice(chunk.as_ref()); @@ -84,10 +60,10 @@ impl, S: Stream> EbmlStreamingParser { } } -impl, S: Stream> EbmlEventSource for EbmlStreamingParser { - type Error = ParsingError; +impl, S: Stream> EbmlEventSource for EbmlStreamingParser { + type Error = WebmetroError; - fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, Self::Error> { + fn poll_event<'a, T: FromEbml<'a>>(&'a mut self) -> Result>, WebmetroError> { return EbmlStreamingParser::poll_event(self); } } From 2170096a2142d404726a83be38560c81fdf2f1c8 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 13 Apr 2018 19:16:34 -0400 Subject: [PATCH 147/164] Make WebmChunker use WebmetroError --- src/chunk.rs | 40 ++++++++++------------------------------ src/commands/relay.rs | 8 ++++---- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 0b78b5e..0c514c6 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -1,12 +1,11 @@ use futures::{Async, Stream}; use std::{ - error::Error, - fmt::{Display, Formatter, Result as FmtResult}, io::Cursor, mem, sync::Arc }; use ebml::EbmlEventSource; +use error::WebmetroError; use webm::*; #[derive(Clone, Debug)] @@ -90,37 +89,18 @@ enum ChunkerState { End } -#[derive(Debug)] -pub enum ChunkingError { - IoError(::std::io::Error), - OtherError(E) -} - -impl Display for ChunkingError { - fn fmt(&self, f: &mut Formatter) -> FmtResult { - write!(f, "Chunking error: {}", self.description()) - } -} -impl Error for ChunkingError { - fn description(&self) -> &str { - match self { - &ChunkingError::IoError(ref err) => err.description(), - &ChunkingError::OtherError(ref err) => err.description() - } - } -} - pub struct WebmChunker { source: S, state: ChunkerState } impl Stream for WebmChunker +where S::Error: Into { type Item = Chunk; - type Error = ChunkingError; + type Error = WebmetroError; - fn poll(&mut self) -> Result>, Self::Error> { + fn poll(&mut self) -> Result>, WebmetroError> { loop { let mut return_value = None; let mut new_state = None; @@ -128,7 +108,7 @@ impl Stream for WebmChunker match self.state { ChunkerState::BuildingHeader(ref mut buffer) => { match self.source.poll_event() { - Err(passthru) => return Err(ChunkingError::OtherError(passthru)), + Err(passthru) => return Err(passthru.into()), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(None)) => return Ok(Async::Ready(None)), Ok(Async::Ready(Some(WebmElement::Cluster))) => { @@ -148,7 +128,7 @@ impl Stream for WebmChunker match encode_webm_element(element, buffer) { Ok(_) => {}, Err(err) => { - return_value = Some(Err(ChunkingError::IoError(err))); + return_value = Some(Err(err.into())); new_state = Some(ChunkerState::End); } } @@ -157,7 +137,7 @@ impl Stream for WebmChunker }, ChunkerState::BuildingCluster(ref mut cluster_head, ref mut buffer) => { match self.source.poll_event() { - Err(passthru) => return Err(ChunkingError::OtherError(passthru)), + Err(passthru) => return Err(passthru.into()), Ok(Async::NotReady) => return Ok(Async::NotReady), Ok(Async::Ready(Some(element @ WebmElement::EbmlHead))) | Ok(Async::Ready(Some(element @ WebmElement::Segment))) => { @@ -174,7 +154,7 @@ impl Stream for WebmChunker }); }, Err(err) => { - return_value = Some(Err(ChunkingError::IoError(err))); + return_value = Some(Err(err.into())); new_state = Some(ChunkerState::End); } } @@ -198,7 +178,7 @@ impl Stream for WebmChunker match encode_webm_element(WebmElement::SimpleBlock(*block), buffer) { Ok(_) => {}, Err(err) => { - return_value = Some(Err(ChunkingError::IoError(err))); + return_value = Some(Err(err.into())); new_state = Some(ChunkerState::End); } } @@ -210,7 +190,7 @@ impl Stream for WebmChunker match encode_webm_element(element, buffer) { Ok(_) => {}, Err(err) => { - return_value = Some(Err(ChunkingError::IoError(err))); + return_value = Some(Err(err.into())); new_state = Some(ChunkerState::End); } } diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 6fe841c..c382293 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -33,7 +33,7 @@ use webmetro::{ Listener, Transmitter }, - chunk::{Chunk, WebmStream, ChunkingError}, + chunk::{Chunk, WebmStream}, error::WebmetroError, fixers::ChunkStream, stream_parser::StreamEbml @@ -65,13 +65,13 @@ impl RelayServer { let sink = Transmitter::new(self.get_channel()); Box::new( - source.forward(sink.sink_map_err(|err| match err {})) + source.forward(sink.sink_map_err(|err| -> WebmetroError {match err {}})) .into_stream() .map(|_| empty()) .map_err(|err| { let io_err = match err { - ChunkingError::IoError(io_err) => io_err, - ChunkingError::OtherError(_) => ErrorKind::InvalidData.into() + WebmetroError::IoError(io_err) => io_err, + _ => ErrorKind::InvalidData.into() }; println!("Post failed: {}", &io_err); io_err From 8b0467c1d7a28098a6a50efe2f267d603b06f863 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Fri, 13 Apr 2018 18:53:34 -0400 Subject: [PATCH 148/164] Add Send bound to WebmetroError::Unknown --- src/commands/relay.rs | 4 ++-- src/error.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index c382293..6c5b29b 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -58,9 +58,9 @@ impl RelayServer { } fn post_stream, S: Stream + 'static>(&self, stream: S) -> BodyStream - where S::Error: Error { + where S::Error: Error + Send { let source = stream - .map_err(|err| WebmetroError::Unknown(err.into())) + .map_err(|err| WebmetroError::Unknown(Box::new(err))) .parse_ebml().chunk_webm(); let sink = Transmitter::new(self.get_channel()); diff --git a/src/error.rs b/src/error.rs index 2073d53..050f147 100644 --- a/src/error.rs +++ b/src/error.rs @@ -14,7 +14,7 @@ use ebml::EbmlError; pub enum WebmetroError { EbmlError(EbmlError), IoError(IoError), - Unknown(Box) + Unknown(Box) } impl Display for WebmetroError { @@ -48,8 +48,8 @@ impl From for WebmetroError { } } -impl From> for WebmetroError { - fn from(err: Box) -> WebmetroError { +impl From> for WebmetroError { + fn from(err: Box) -> WebmetroError { WebmetroError::Unknown(err) } } From 9123d63343b1ece4926782232bcc15db9554fc90 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 04:53:18 -0400 Subject: [PATCH 149/164] Express filter as a future --- src/commands/filter.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/commands/filter.rs b/src/commands/filter.rs index c3a4839..663a2bb 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -5,7 +5,7 @@ use std::{ }; use clap::{App, Arg, ArgMatches, SubCommand}; -use futures::Stream; +use futures::prelude::*; use super::StdinStream; use webmetro::{ @@ -40,11 +40,8 @@ pub fn run(args: &ArgMatches) -> Result<(), Box> { chunk_stream = Box::new(chunk_stream.throttle()); } - let stdout = io::stdout(); - let mut stdout_writer = stdout.lock(); - for chunk in chunk_stream.wait() { - stdout_writer.write_all(chunk?.as_ref())?; - } - - Ok(()) + let result = chunk_stream.fold((), |_, chunk| { + io::stdout().write_all(chunk.as_ref()) + }).wait(); + result } From e77a3d0e985e79b0a83dd2b004fb7584fdd56033 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 04:45:35 -0400 Subject: [PATCH 150/164] Implement Throttle filter (fails because not executed on runtime) --- src/commands/filter.rs | 11 ++++++----- src/fixers.rs | 34 +++++++++++++++++++++++++--------- src/lib.rs | 1 + src/main.rs | 1 + 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/commands/filter.rs b/src/commands/filter.rs index 663a2bb..4e6d7bd 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -13,6 +13,7 @@ use webmetro::{ Chunk, WebmStream }, + error::WebmetroError, fixers::ChunkStream, stream_parser::StreamEbml }; @@ -28,11 +29,10 @@ pub fn options() -> App<'static, 'static> { pub fn run(args: &ArgMatches) -> Result<(), Box> { let stdin = io::stdin(); - let mut chunk_stream: Box>> = Box::new( + let mut chunk_stream: Box> = Box::new( StdinStream::new(stdin.lock()) .parse_ebml() .chunk_webm() - .map_err(|err| Box::new(err) as Box) .fix_timecodes() ); @@ -40,8 +40,9 @@ pub fn run(args: &ArgMatches) -> Result<(), Box> { chunk_stream = Box::new(chunk_stream.throttle()); } - let result = chunk_stream.fold((), |_, chunk| { + chunk_stream.fold((), |_, chunk| { io::stdout().write_all(chunk.as_ref()) - }).wait(); - result + }).wait()?; + + Ok(()) } diff --git a/src/fixers.rs b/src/fixers.rs index 5925c26..dc6fd24 100644 --- a/src/fixers.rs +++ b/src/fixers.rs @@ -1,9 +1,10 @@ -use std::time::Instant; +use std::time::{Duration, Instant}; -use futures::Async; -use futures::stream::Stream; +use futures::prelude::*; +use tokio::timer::Delay; use chunk::Chunk; +use error::WebmetroError; pub struct ChunkTimecodeFixer { stream: S, @@ -86,16 +87,29 @@ impl> Stream for StartingPointFinder pub struct Throttle { stream: S, - start_time: Instant + start_time: Instant, + sleep: Delay } -impl> Stream for Throttle +impl> Stream for Throttle { type Item = S::Item; - type Error = S::Error; + type Error = WebmetroError; - fn poll(&mut self) -> Result>, Self::Error> { - self.stream.poll() + fn poll(&mut self) -> Result>, WebmetroError> { + match self.sleep.poll() { + Err(err) => return Err(WebmetroError::Unknown(Box::new(err))), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(())) => { /* can continue */ } + } + + let next_chunk = self.stream.poll(); + if let Ok(Async::Ready(Some(Chunk::ClusterHead(ref cluster_head)))) = next_chunk { + // snooze until real time has "caught up" to the stream + let offset = Duration::from_millis(cluster_head.end); + self.sleep.reset(self.start_time + offset); + } + next_chunk } } @@ -118,9 +132,11 @@ pub trait ChunkStream where Self : Sized + Stream { } fn throttle(self) -> Throttle { + let now = Instant::now(); Throttle { stream: self, - start_time: Instant::now() + start_time: now, + sleep: Delay::new(now) } } } diff --git a/src/lib.rs b/src/lib.rs index 1f064cb..b15d184 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ extern crate bytes; extern crate futures; extern crate odds; +extern crate tokio; pub mod ebml; pub mod error; diff --git a/src/main.rs b/src/main.rs index d2f70d8..eb6d5ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ #[macro_use] extern crate clap; extern crate futures; extern crate hyper; +extern crate tokio; extern crate webmetro; mod commands; From a847d62b34d4eb714e873dcf17992a48293135b0 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 18:18:50 -0400 Subject: [PATCH 151/164] Have utilities read stdin via tokio_io wrapper --- Cargo.lock | 1 + Cargo.toml | 1 + src/commands/dump.rs | 10 +++----- src/commands/filter.rs | 5 ++-- src/commands/mod.rs | 55 ++++++++++++++---------------------------- src/main.rs | 1 + 6 files changed, 26 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e679726..559258a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,6 +598,7 @@ dependencies = [ "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", "odds 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f2d5d16..99d0239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,4 @@ futures = "0.1.20" hyper = "0.11.25" odds = { version = "0.3.1", features = ["std-vec"] } tokio = "0.1.5" +tokio-io = "0.1.6" diff --git a/src/commands/dump.rs b/src/commands/dump.rs index fa2fb40..7a83a69 100644 --- a/src/commands/dump.rs +++ b/src/commands/dump.rs @@ -1,12 +1,9 @@ -use std::{ - error::Error, - io -}; +use std::error::Error; use clap::{App, AppSettings, ArgMatches, SubCommand}; use futures::Async; -use super::StdinStream; +use super::stdin_stream; use webmetro::{ stream_parser::StreamEbml, webm::{ @@ -23,8 +20,7 @@ pub fn options() -> App<'static, 'static> { pub fn run(_args: &ArgMatches) -> Result<(), Box> { - let stdin = io::stdin(); - let mut events = StdinStream::new(stdin.lock()).parse_ebml(); + let mut events = stdin_stream().parse_ebml(); // stdin is sync so Async::NotReady will never happen while let Ok(Async::Ready(Some(element))) = events.poll_event() { diff --git a/src/commands/filter.rs b/src/commands/filter.rs index 4e6d7bd..5025afc 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -7,7 +7,7 @@ use std::{ use clap::{App, Arg, ArgMatches, SubCommand}; use futures::prelude::*; -use super::StdinStream; +use super::stdin_stream; use webmetro::{ chunk::{ Chunk, @@ -28,9 +28,8 @@ pub fn options() -> App<'static, 'static> { pub fn run(args: &ArgMatches) -> Result<(), Box> { - let stdin = io::stdin(); let mut chunk_stream: Box> = Box::new( - StdinStream::new(stdin.lock()) + stdin_stream() .parse_ebml() .chunk_webm() .fix_timecodes() diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 46f739b..a7654ce 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,11 +1,19 @@ use std::io::{ - StdinLock, - prelude::* + Error as IoError, + stdin, + Stdin }; use futures::{ - Async, - stream::Stream + prelude::*, + stream::MapErr +}; +use tokio_io::{ + io::AllowStdIo, + codec::{ + BytesCodec, + FramedRead + } }; use webmetro::error::WebmetroError; @@ -13,37 +21,10 @@ pub mod dump; pub mod filter; pub mod relay; -/// A hackish adapter that makes chunks of bytes from stdin available as a Stream; -/// is NOT actually async, and just uses blocking read. Buffers aren't optimized either -/// and copy more than necessary. -pub struct StdinStream<'a> { - buf_reader: StdinLock<'a>, - read_bytes: usize -} - -impl<'a> StdinStream<'a> { - pub fn new(lock: StdinLock<'a>) -> Self { - StdinStream { - buf_reader: lock, - read_bytes: 0 - } - } -} - -impl<'a> Stream for StdinStream<'a> { - type Item = Vec; - type Error = WebmetroError; - - fn poll(&mut self) -> Result>, Self::Error> { - self.buf_reader.consume(self.read_bytes); - let read_bytes = &mut self.read_bytes; - self.buf_reader.fill_buf().map(|slice| { - *read_bytes = slice.len(); - if *read_bytes > 0 { - Async::Ready(Some(Into::>::into(slice))) - } else { - Async::Ready(None) - } - }).map_err(WebmetroError::IoError) - } +/// An adapter that makes chunks of bytes from stdin available as a Stream; +/// is NOT actually async, and just uses blocking read. Don't use more than +/// one at once, who knows who gets which bytes. +pub fn stdin_stream() -> MapErr, BytesCodec>, fn(IoError) -> WebmetroError> { + FramedRead::new(AllowStdIo::new(stdin()), BytesCodec::new()) + .map_err(WebmetroError::IoError) } diff --git a/src/main.rs b/src/main.rs index eb6d5ab..3e76ad8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ extern crate futures; extern crate hyper; extern crate tokio; +extern crate tokio_io; extern crate webmetro; mod commands; From ab55d951a0262a680e1e0b1e7d23f28303044e15 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 18:44:09 -0400 Subject: [PATCH 152/164] Use nonzero exit code on error. --- src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 3e76ad8..9b5cd7d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,5 +36,8 @@ fn main() { println!(""); Ok(()) } - }.unwrap_or_else(|err| println!("Error: {}", err)); + }.unwrap_or_else(|err| { + println!("Error: {}", err); + ::std::process::exit(1); + }); } From 0075e52f7b548a949f9450be73b9b9253e1663fc Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 18:44:53 -0400 Subject: [PATCH 153/164] Run filter under Tokio runtime so that Delay works --- src/commands/filter.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/commands/filter.rs b/src/commands/filter.rs index 5025afc..2062d01 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -6,6 +6,7 @@ use std::{ use clap::{App, Arg, ArgMatches, SubCommand}; use futures::prelude::*; +use tokio; use super::stdin_stream; use webmetro::{ @@ -28,7 +29,7 @@ pub fn options() -> App<'static, 'static> { pub fn run(args: &ArgMatches) -> Result<(), Box> { - let mut chunk_stream: Box> = Box::new( + let mut chunk_stream: Box + Send> = Box::new( stdin_stream() .parse_ebml() .chunk_webm() @@ -39,9 +40,12 @@ pub fn run(args: &ArgMatches) -> Result<(), Box> { chunk_stream = Box::new(chunk_stream.throttle()); } - chunk_stream.fold((), |_, chunk| { - io::stdout().write_all(chunk.as_ref()) - }).wait()?; + tokio::run(chunk_stream.for_each(|chunk| { + io::stdout().write_all(chunk.as_ref()).map_err(WebmetroError::IoError) + }).map_err(|err| { + println!("Error: {}", err); + ::std::process::exit(1); + })); Ok(()) } From 49541347a48f2ad4a237fe56d9faac2cca244fe6 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sat, 14 Apr 2018 22:34:46 -0400 Subject: [PATCH 154/164] Give all subcommands option of returning a future to run in a Tokio context --- src/commands/dump.rs | 7 +++---- src/commands/filter.rs | 14 +++----------- src/commands/relay.rs | 11 +++++++++-- src/error.rs | 7 +++++++ src/main.rs | 30 +++++++++++++++++++++--------- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/commands/dump.rs b/src/commands/dump.rs index 7a83a69..8b691e6 100644 --- a/src/commands/dump.rs +++ b/src/commands/dump.rs @@ -1,10 +1,9 @@ -use std::error::Error; - use clap::{App, AppSettings, ArgMatches, SubCommand}; -use futures::Async; +use futures::prelude::*; use super::stdin_stream; use webmetro::{ + error::WebmetroError, stream_parser::StreamEbml, webm::{ SimpleBlock, @@ -18,7 +17,7 @@ pub fn options() -> App<'static, 'static> { .about("Dumps WebM parsing events from parsing stdin") } -pub fn run(_args: &ArgMatches) -> Result<(), Box> { +pub fn run(_args: &ArgMatches) -> Result<(), WebmetroError> { let mut events = stdin_stream().parse_ebml(); diff --git a/src/commands/filter.rs b/src/commands/filter.rs index 2062d01..a276d88 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -1,12 +1,10 @@ use std::{ - error::Error, io, io::prelude::* }; use clap::{App, Arg, ArgMatches, SubCommand}; use futures::prelude::*; -use tokio; use super::stdin_stream; use webmetro::{ @@ -27,8 +25,7 @@ pub fn options() -> App<'static, 'static> { .help("Slow down output to \"realtime\" speed as determined by the timestamps (useful for streaming)")) } -pub fn run(args: &ArgMatches) -> Result<(), Box> { - +pub fn run(args: &ArgMatches) -> Box + Send> { let mut chunk_stream: Box + Send> = Box::new( stdin_stream() .parse_ebml() @@ -40,12 +37,7 @@ pub fn run(args: &ArgMatches) -> Result<(), Box> { chunk_stream = Box::new(chunk_stream.throttle()); } - tokio::run(chunk_stream.for_each(|chunk| { + Box::new(chunk_stream.for_each(|chunk| { io::stdout().write_all(chunk.as_ref()).map_err(WebmetroError::IoError) - }).map_err(|err| { - println!("Error: {}", err); - ::std::process::exit(1); - })); - - Ok(()) + })) } diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 6c5b29b..88e116d 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -122,12 +122,19 @@ pub fn options() -> App<'static, 'static> { .required(true)) } -pub fn run(args: &ArgMatches) -> Result<(), Box> { +pub fn run(args: &ArgMatches) -> Result<(), WebmetroError> { let single_channel = Channel::new(); let addr_str = args.value_of("listen").ok_or("Listen address wasn't provided")?; let addr = addr_str.to_socket_addrs()?.next().ok_or("Listen address didn't resolve")?; - Http::new().bind(&addr, move || Ok(RelayServer(single_channel.clone())))?.run()?; + Http::new() + .bind(&addr, move || { + Ok(RelayServer(single_channel.clone())) + }) + .map_err(|err| WebmetroError::Unknown(Box::new(err)))? + .run() + .map_err(|err| WebmetroError::Unknown(Box::new(err)))?; + Ok(()) } diff --git a/src/error.rs b/src/error.rs index 050f147..d0c754d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -53,3 +53,10 @@ impl From> for WebmetroError { WebmetroError::Unknown(err) } } + +impl<'a> From<&'a str> for WebmetroError { + fn from(err: &'a str) -> WebmetroError { + let error: Box = err.into(); + WebmetroError::Unknown(error) + } +} diff --git a/src/main.rs b/src/main.rs index 9b5cd7d..5278bc9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,9 @@ extern crate webmetro; mod commands; use clap::{App, AppSettings}; +use futures::prelude::*; +use webmetro::error::WebmetroError; + use commands::{ relay, filter, @@ -27,17 +30,26 @@ fn options() -> App<'static, 'static> { fn main() { let args = options().get_matches(); - match args.subcommand() { - ("filter", Some(sub_args)) => filter::run(sub_args), - ("relay", Some(sub_args)) => relay::run(sub_args), - ("dump", Some(sub_args)) => dump::run(sub_args), - _ => { + tokio_run(match args.subcommand() { + ("filter", Some(sub_args)) => box_up(filter::run(sub_args)), + ("relay", Some(sub_args)) => box_up(relay::run(sub_args)), + ("dump", Some(sub_args)) => box_up(dump::run(sub_args)), + _ => box_up(futures::lazy(|| { options().print_help().unwrap(); println!(""); Ok(()) - } - }.unwrap_or_else(|err| { - println!("Error: {}", err); - ::std::process::exit(1); + })) }); } + +fn tokio_run(task: Box + Send + 'static>) { + tokio::run(task.into_future().map_err(|err| { + eprintln!("Error: {}", err); + ::std::process::exit(1); + })); +} + +fn box_up>(task: F) -> Box + Send + 'static> +where F::Future: Send + 'static { + Box::new(task.into_future()) +} From ee82846590068150190e1abb62a3bed1e378b62f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 15 Apr 2018 01:21:00 -0400 Subject: [PATCH 155/164] Use tokio_core instead of runtime as top loop --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 559258a..2d36767 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,6 +598,7 @@ dependencies = [ "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", "odds 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 99d0239..7a72521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,5 @@ futures = "0.1.20" hyper = "0.11.25" odds = { version = "0.3.1", features = ["std-vec"] } tokio = "0.1.5" +tokio-core = "0.1.17" tokio-io = "0.1.6" diff --git a/src/main.rs b/src/main.rs index 5278bc9..71abbe0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ extern crate futures; extern crate hyper; extern crate tokio; +extern crate tokio_core; extern crate tokio_io; extern crate webmetro; @@ -9,6 +10,7 @@ mod commands; use clap::{App, AppSettings}; use futures::prelude::*; +use tokio_core::reactor::Core; use webmetro::error::WebmetroError; use commands::{ @@ -30,7 +32,10 @@ fn options() -> App<'static, 'static> { fn main() { let args = options().get_matches(); - tokio_run(match args.subcommand() { + let core = Core::new().unwrap(); + let handle = core.handle(); + + tokio_run(core, match args.subcommand() { ("filter", Some(sub_args)) => box_up(filter::run(sub_args)), ("relay", Some(sub_args)) => box_up(relay::run(sub_args)), ("dump", Some(sub_args)) => box_up(dump::run(sub_args)), @@ -42,14 +47,15 @@ fn main() { }); } -fn tokio_run(task: Box + Send + 'static>) { - tokio::run(task.into_future().map_err(|err| { +fn tokio_run(mut core: Core, task: Box>) { + core.run(task.into_future()).unwrap_or_else(|err| { eprintln!("Error: {}", err); ::std::process::exit(1); - })); + }); } -fn box_up>(task: F) -> Box + Send + 'static> -where F::Future: Send + 'static { +fn box_up>(task: F) -> Box> +where F::Future: 'static +{ Box::new(task.into_future()) } From 885681f00950ac0f0f2282f9c5df27f890efbd12 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 15 Apr 2018 01:42:47 -0400 Subject: [PATCH 156/164] Add some reusable error conversions --- src/commands/mod.rs | 10 ++++++++++ src/error.rs | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index a7654ce..1daf601 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,5 +1,7 @@ +use std::error::Error; use std::io::{ Error as IoError, + ErrorKind, stdin, Stdin }; @@ -8,6 +10,7 @@ use futures::{ prelude::*, stream::MapErr }; +use hyper::Error as HyperError; use tokio_io::{ io::AllowStdIo, codec::{ @@ -28,3 +31,10 @@ pub fn stdin_stream() -> MapErr, BytesCodec>, fn(Io FramedRead::new(AllowStdIo::new(stdin()), BytesCodec::new()) .map_err(WebmetroError::IoError) } + +pub fn to_hyper_error(err: WebmetroError) -> HyperError { + match err { + WebmetroError::IoError(io_err) => io_err.into(), + err => IoError::new(ErrorKind::InvalidData, err.description()).into() + } +} diff --git a/src/error.rs b/src/error.rs index d0c754d..ad11148 100644 --- a/src/error.rs +++ b/src/error.rs @@ -17,6 +17,16 @@ pub enum WebmetroError { Unknown(Box) } +impl WebmetroError { + pub fn from_str(string: &str) -> WebmetroError { + string.into() + } + + pub fn from_err(err: E) -> WebmetroError { + WebmetroError::Unknown(Box::new(err)) + } +} + impl Display for WebmetroError { fn fmt(&self, f: &mut Formatter) -> FmtResult { match self { From 739862bc3597db7be99b338ca1691248efac94d5 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 15 Apr 2018 01:43:23 -0400 Subject: [PATCH 157/164] Add send subcommand for uploading WebM --- src/commands/filter.rs | 2 +- src/commands/mod.rs | 1 + src/commands/send.rs | 79 ++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 4 +++ 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/commands/send.rs diff --git a/src/commands/filter.rs b/src/commands/filter.rs index a276d88..1cdecde 100644 --- a/src/commands/filter.rs +++ b/src/commands/filter.rs @@ -22,7 +22,7 @@ pub fn options() -> App<'static, 'static> { .about("Copies WebM from stdin to stdout, applying the same cleanup & stripping the relay server does.") .arg(Arg::with_name("throttle") .long("throttle") - .help("Slow down output to \"realtime\" speed as determined by the timestamps (useful for streaming)")) + .help("Slow down output to \"real time\" speed as determined by the timestamps (useful for streaming static files)")) } pub fn run(args: &ArgMatches) -> Box + Send> { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 1daf601..9ebb8b4 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -23,6 +23,7 @@ use webmetro::error::WebmetroError; pub mod dump; pub mod filter; pub mod relay; +pub mod send; /// An adapter that makes chunks of bytes from stdin available as a Stream; /// is NOT actually async, and just uses blocking read. Don't use more than diff --git a/src/commands/send.rs b/src/commands/send.rs new file mode 100644 index 0000000..c070444 --- /dev/null +++ b/src/commands/send.rs @@ -0,0 +1,79 @@ +use clap::{App, Arg, ArgMatches, SubCommand}; +use futures::{ + future, + prelude::* +}; +use hyper::{ + Error as HyperError, + Method, + client::{ + Config, + Request + } +}; +use tokio_core::reactor::{ + Handle +}; + +use super::{ + stdin_stream, + to_hyper_error +}; +use webmetro::{ + chunk::{ + Chunk, + WebmStream + }, + error::WebmetroError, + fixers::ChunkStream, + stream_parser::StreamEbml +}; + +pub fn options() -> App<'static, 'static> { + SubCommand::with_name("send") + .about("PUTs WebM from stdin to a relay server.") + .arg(Arg::with_name("url") + .help("The location to upload to") + .required(true)) + .arg(Arg::with_name("throttle") + .long("throttle") + .help("Slow down upload to \"real time\" speed as determined by the timestamps (useful for streaming static files)")) +} + +type BoxedChunkStream = Box>; +type BoxedHyperStream = Box>; + +pub fn run(handle: Handle, args: &ArgMatches) -> Box> { + let mut chunk_stream: BoxedChunkStream = Box::new( + stdin_stream() + .parse_ebml() + .chunk_webm() + .fix_timecodes() + ); + + let url_str = match args.value_of("url") { + Some(url) => String::from(url), + _ => return Box::new(Err(WebmetroError::from_str("Listen address wasn't provided")).into_future()) + }; + + if args.is_present("throttle") { + chunk_stream = Box::new(chunk_stream.throttle()); + } + + let request_body_stream = Box::new(chunk_stream.map_err(to_hyper_error)) as BoxedHyperStream; + + Box::new(future::lazy(move || { + url_str.parse().map_err(WebmetroError::from_err) + }).and_then(move |uri| { + let client = Config::default() + .body::() + .build(&handle); + + let mut request: Request = Request::new(Method::Put, uri); + request.set_body(request_body_stream); + + client.request(request) + .map(|_response| ()) + .map_err(WebmetroError::from_err) + })) +} diff --git a/src/main.rs b/src/main.rs index 71abbe0..a1232ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use webmetro::error::WebmetroError; use commands::{ relay, filter, + send, dump }; @@ -23,9 +24,11 @@ fn options() -> App<'static, 'static> { App::new("webmetro") .version(crate_version!()) .about("Utilities for broadcasting & relaying live WebM video/audio streams") + .setting(AppSettings::DisableHelpSubcommand) .setting(AppSettings::VersionlessSubcommands) .subcommand(relay::options()) .subcommand(filter::options()) + .subcommand(send::options()) .subcommand(dump::options()) } @@ -38,6 +41,7 @@ fn main() { tokio_run(core, match args.subcommand() { ("filter", Some(sub_args)) => box_up(filter::run(sub_args)), ("relay", Some(sub_args)) => box_up(relay::run(sub_args)), + ("send", Some(sub_args)) => box_up(send::run(handle, sub_args)), ("dump", Some(sub_args)) => box_up(dump::run(sub_args)), _ => box_up(futures::lazy(|| { options().print_help().unwrap(); From 8d76bd5c574148bc0fe16e1cc05604dbc5953f9d Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Sun, 15 Apr 2018 22:06:42 -0400 Subject: [PATCH 158/164] Cleanup error handling --- src/commands/relay.rs | 13 +++++-------- src/commands/send.rs | 5 ++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 88e116d..0c2dfbc 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -1,5 +1,4 @@ use std::error::Error; -use std::io::ErrorKind; use std::net::ToSocketAddrs; use std::sync::{ Arc, @@ -39,6 +38,8 @@ use webmetro::{ stream_parser::StreamEbml }; +use super::to_hyper_error; + type BodyStream = Box>; struct RelayServer(Arc>); @@ -60,7 +61,7 @@ impl RelayServer { fn post_stream, S: Stream + 'static>(&self, stream: S) -> BodyStream where S::Error: Error + Send { let source = stream - .map_err(|err| WebmetroError::Unknown(Box::new(err))) + .map_err(WebmetroError::from_err) .parse_ebml().chunk_webm(); let sink = Transmitter::new(self.get_channel()); @@ -69,12 +70,8 @@ impl RelayServer { .into_stream() .map(|_| empty()) .map_err(|err| { - let io_err = match err { - WebmetroError::IoError(io_err) => io_err, - _ => ErrorKind::InvalidData.into() - }; - println!("Post failed: {}", &io_err); - io_err + println!("{}", err); + to_hyper_error(err) }) .flatten() ) diff --git a/src/commands/send.rs b/src/commands/send.rs index c070444..d7932f6 100644 --- a/src/commands/send.rs +++ b/src/commands/send.rs @@ -60,7 +60,10 @@ pub fn run(handle: Handle, args: &ArgMatches) -> Box Date: Sun, 15 Apr 2018 22:07:21 -0400 Subject: [PATCH 159/164] Fix upload tool, keep request active instead of dropping the Response --- src/commands/send.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/commands/send.rs b/src/commands/send.rs index d7932f6..6987f01 100644 --- a/src/commands/send.rs +++ b/src/commands/send.rs @@ -76,7 +76,11 @@ pub fn run(handle: Handle, args: &ArgMatches) -> Box Date: Sun, 15 Apr 2018 23:43:15 -0400 Subject: [PATCH 160/164] Remove debug prints from relay command --- src/commands/relay.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 0c2dfbc..37d97c6 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -70,7 +70,7 @@ impl RelayServer { .into_stream() .map(|_| empty()) .map_err(|err| { - println!("{}", err); + //TODO: log something somewhere to_hyper_error(err) }) .flatten() @@ -87,7 +87,7 @@ impl Service for RelayServer { fn call(&self, request: Request) -> Self::Future { let (method, uri, _http_version, _headers, request_body) = request.deconstruct(); - eprintln!("New {} Request: {}", method, uri.path()); + //TODO: log equiv to: eprintln!("New {} Request: {}", method, uri.path()); ok(match (method, uri.path()) { (Head, "/live") => { From 982c5c2dcb8b645e674131a3f9fb2b87a7aa312f Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Apr 2018 00:59:53 -0400 Subject: [PATCH 161/164] Limit buffer size for stream parser --- src/commands/relay.rs | 4 +++- src/error.rs | 3 +++ src/stream_parser.rs | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/commands/relay.rs b/src/commands/relay.rs index 37d97c6..be7e50d 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -40,6 +40,8 @@ use webmetro::{ use super::to_hyper_error; +const BUFFER_LIMIT: usize = 2 * 1024 * 1024; + type BodyStream = Box>; struct RelayServer(Arc>); @@ -62,7 +64,7 @@ impl RelayServer { where S::Error: Error + Send { let source = stream .map_err(WebmetroError::from_err) - .parse_ebml().chunk_webm(); + .parse_ebml().with_buffer_limit(BUFFER_LIMIT).chunk_webm(); let sink = Transmitter::new(self.get_channel()); Box::new( diff --git a/src/error.rs b/src/error.rs index ad11148..40f2007 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,6 +12,7 @@ use ebml::EbmlError; #[derive(Debug)] pub enum WebmetroError { + ResourcesExceeded, EbmlError(EbmlError), IoError(IoError), Unknown(Box) @@ -30,6 +31,7 @@ impl WebmetroError { impl Display for WebmetroError { fn fmt(&self, f: &mut Formatter) -> FmtResult { match self { + &WebmetroError::ResourcesExceeded => write!(f, "resources exceeded"), &WebmetroError::EbmlError(ref err) => err.fmt(f), &WebmetroError::IoError(ref err) => err.fmt(f), &WebmetroError::Unknown(ref err) => err.fmt(f), @@ -39,6 +41,7 @@ impl Display for WebmetroError { impl Error for WebmetroError { fn description(&self) -> &str { match self { + &WebmetroError::ResourcesExceeded => "resources exceeded", &WebmetroError::EbmlError(ref err) => err.description(), &WebmetroError::IoError(ref err) => err.description(), &WebmetroError::Unknown(ref err) => err.description(), diff --git a/src/stream_parser.rs b/src/stream_parser.rs index c1e4665..58d215b 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -10,14 +10,23 @@ use error::WebmetroError; pub struct EbmlStreamingParser { stream: S, buffer: BytesMut, + buffer_size_limit: Option, last_read: usize } +impl EbmlStreamingParser { + pub fn with_buffer_limit(mut self, limit: usize) -> Self { + self.buffer_size_limit = Some(limit); + self + } +} + pub trait StreamEbml where Self: Sized + Stream, Self::Item: AsRef<[u8]> { fn parse_ebml(self) -> EbmlStreamingParser { EbmlStreamingParser { stream: self, buffer: BytesMut::new(), + buffer_size_limit: None, last_read: 0 } } @@ -48,6 +57,12 @@ impl, S: Stream> EbmlStreamingPa }) } + if let Some(limit) = self.buffer_size_limit { + if limit <= self.buffer.len() { + return Err(WebmetroError::ResourcesExceeded); + } + } + match self.stream.poll() { Ok(Async::Ready(Some(chunk))) => { self.buffer.reserve(chunk.as_ref().len()); From 32cf6dd2efe57ee1199bcb4142410eb58e5d0d18 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Apr 2018 01:16:03 -0400 Subject: [PATCH 162/164] Add buffer size limit to chunker --- src/chunk.rs | 56 +++++++++++++++++++++++++------------------ src/commands/relay.rs | 3 ++- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 0c514c6..3f6df32 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -91,9 +91,27 @@ enum ChunkerState { pub struct WebmChunker { source: S, + buffer_size_limit: Option, state: ChunkerState } +impl WebmChunker { + pub fn with_buffer_limit(mut self, limit: usize) -> Self { + self.buffer_size_limit = Some(limit); + self + } +} + +fn encode(element: WebmElement, buffer: &mut Cursor>, limit: Option) -> Result<(), WebmetroError> { + if let Some(limit) = limit { + if limit <= buffer.get_ref().len() { + return Err(WebmetroError::ResourcesExceeded); + } + } + + encode_webm_element(element, buffer).map_err(|err| err.into()) +} + impl Stream for WebmChunker where S::Error: Into { @@ -125,13 +143,10 @@ where S::Error: Into Ok(Async::Ready(Some(WebmElement::Void))) => {}, Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {}, Ok(Async::Ready(Some(element))) => { - match encode_webm_element(element, buffer) { - Ok(_) => {}, - Err(err) => { - return_value = Some(Err(err.into())); - new_state = Some(ChunkerState::End); - } - } + encode(element, buffer, self.buffer_size_limit).unwrap_or_else(|err| { + return_value = Some(Err(err)); + new_state = Some(ChunkerState::End); + }); } } }, @@ -145,7 +160,7 @@ where S::Error: Into let liberated_buffer = mem::replace(buffer, Cursor::new(Vec::new())); let mut new_header_cursor = Cursor::new(Vec::new()); - match encode_webm_element(element, &mut new_header_cursor) { + match encode(element, &mut new_header_cursor, self.buffer_size_limit) { Ok(_) => { return_value = Some(Ok(Async::Ready(Some(Chunk::ClusterHead(liberated_cluster_head))))); new_state = Some(ChunkerState::EmittingClusterBodyBeforeNewHeader{ @@ -154,7 +169,7 @@ where S::Error: Into }); }, Err(err) => { - return_value = Some(Err(err.into())); + return_value = Some(Err(err)); new_state = Some(ChunkerState::End); } } @@ -175,25 +190,19 @@ where S::Error: Into cluster_head.keyframe = true; } cluster_head.observe_simpleblock_timecode(block.timecode); - match encode_webm_element(WebmElement::SimpleBlock(*block), buffer) { - Ok(_) => {}, - Err(err) => { - return_value = Some(Err(err.into())); - new_state = Some(ChunkerState::End); - } - } + encode(WebmElement::SimpleBlock(*block), buffer, self.buffer_size_limit).unwrap_or_else(|err| { + return_value = Some(Err(err)); + new_state = Some(ChunkerState::End); + }); }, Ok(Async::Ready(Some(WebmElement::Info))) => {}, Ok(Async::Ready(Some(WebmElement::Void))) => {}, Ok(Async::Ready(Some(WebmElement::Unknown(_)))) => {}, Ok(Async::Ready(Some(element))) => { - match encode_webm_element(element, buffer) { - Ok(_) => {}, - Err(err) => { - return_value = Some(Err(err.into())); - new_state = Some(ChunkerState::End); - } - } + encode(element, buffer, self.buffer_size_limit).unwrap_or_else(|err| { + return_value = Some(Err(err)); + new_state = Some(ChunkerState::End); + }); }, Ok(Async::Ready(None)) => { // flush final Cluster on end of stream @@ -245,6 +254,7 @@ pub trait WebmStream where Self: Sized + EbmlEventSource { fn chunk_webm(self) -> WebmChunker { WebmChunker { source: self, + buffer_size_limit: None, state: ChunkerState::BuildingHeader(Cursor::new(Vec::new())) } } diff --git a/src/commands/relay.rs b/src/commands/relay.rs index be7e50d..d213601 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -64,7 +64,8 @@ impl RelayServer { where S::Error: Error + Send { let source = stream .map_err(WebmetroError::from_err) - .parse_ebml().with_buffer_limit(BUFFER_LIMIT).chunk_webm(); + .parse_ebml().with_buffer_limit(BUFFER_LIMIT) + .chunk_webm().with_buffer_limit(BUFFER_LIMIT); let sink = Transmitter::new(self.get_channel()); Box::new( From 0e370556a2a796d837d0aa242d26feb93485423b Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Apr 2018 01:58:28 -0400 Subject: [PATCH 163/164] add a little documentation re: soft buffer limits --- src/chunk.rs | 5 ++++- src/commands/relay.rs | 4 ++-- src/stream_parser.rs | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/chunk.rs b/src/chunk.rs index 3f6df32..b81d16f 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -96,7 +96,10 @@ pub struct WebmChunker { } impl WebmChunker { - pub fn with_buffer_limit(mut self, limit: usize) -> Self { + /// add a "soft" buffer size limit; if a chunk buffer exceeds this size, + /// error the stream instead of resuming. It's still possible for a buffer + /// to exceed this size *after* a write, so ensure input sizes are reasonable. + pub fn with_soft_limit(mut self, limit: usize) -> Self { self.buffer_size_limit = Some(limit); self } diff --git a/src/commands/relay.rs b/src/commands/relay.rs index d213601..17bc018 100644 --- a/src/commands/relay.rs +++ b/src/commands/relay.rs @@ -64,8 +64,8 @@ impl RelayServer { where S::Error: Error + Send { let source = stream .map_err(WebmetroError::from_err) - .parse_ebml().with_buffer_limit(BUFFER_LIMIT) - .chunk_webm().with_buffer_limit(BUFFER_LIMIT); + .parse_ebml().with_soft_limit(BUFFER_LIMIT) + .chunk_webm().with_soft_limit(BUFFER_LIMIT); let sink = Transmitter::new(self.get_channel()); Box::new( diff --git a/src/stream_parser.rs b/src/stream_parser.rs index 58d215b..0d3c9cb 100644 --- a/src/stream_parser.rs +++ b/src/stream_parser.rs @@ -15,7 +15,10 @@ pub struct EbmlStreamingParser { } impl EbmlStreamingParser { - pub fn with_buffer_limit(mut self, limit: usize) -> Self { + /// add a "soft" buffer size limit; if the input buffer exceeds this size, + /// error the stream instead of resuming. It's still possible for the buffer + /// to exceed this size *after* a fill, so ensure input sizes are reasonable. + pub fn with_soft_limit(mut self, limit: usize) -> Self { self.buffer_size_limit = Some(limit); self } From 9506c91f5ab84eefe739937aa0bd8e73e5d40522 Mon Sep 17 00:00:00 2001 From: Tangent 128 Date: Mon, 16 Apr 2018 03:29:43 -0400 Subject: [PATCH 164/164] Write README.md --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ notes.md | 2 -- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 README.md delete mode 100644 notes.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5ca841 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# webmetro + +`webmetro` is a simple relay server for broadcasting a WebM stream from one uploader to many downloaders, via HTTP. + +The initialization segment is remembered, so that viewers can join mid-stream. + +Cluster timestamps are rewritten to be monotonic, so multiple (compatibly-encoded) webm files can be chained together without clients needing to reconnect. + +## Usage + +Launch a relay server with the `relay` subcommand: + +`webmetro relay localhost:8080` + +At this point you can open http://localhost:8080/live in a web browser. + +Next, a source client will need to `POST` or `PUT` a stream to that URL; a static file can be uploaded with the `send` subcommand: + +`webmetro send --throttle http://localhost:8080/live < file.webm` + +You can even glue together multiple files, provided they share the same codecs and track order: + +`cat 1.webm 2.webm 3.webm | webmetro send --throttle http://localhost:8080/live` + +You can use ffmpeg to transcode a non-WebM file or access a media device: + +`ffmpeg -i file.mp4 -deadline realtime -threads 4 -vb 700k -vcodec libvpx -f webm -live 1 - | webmetro send --throttle http://localhost:8080/live` + +(if the source is itself a live stream, you can leave off the `--throttle` flag) + +## Limitations + +* HTTPS is not supported yet. It really should be. +* There aren't any access controls on either the source or viewer roles yet. +* Currently the server only recognizes a single stream, at `/live`. +* The server tries to start a viewer at a cluster containing a keyframe; it is not yet smart enough to ensure that the keyframe belongs to the *video* stream. +* The server doesn't parse any metadata, such as tags; the Info segment is stripped out, everything else is blindly passed along. +* The server drops any source that it feels uses too much buffer space. This is not yet configurable, though sane files probably won't hit the limit. (Essentially, clusters & the initialization segment can't individually be more than 2M) + +## See Also + +* the [Icecast](http://www.icecast.org/) streaming server likewise relays media streams over HTTP, and supports additional non-WebM formats such as Ogg. It does not support clients connecting to a stream before the source, however. + +## License + +`webmetro` is licensed under the MIT license; see the LICENSE file. diff --git a/notes.md b/notes.md deleted file mode 100644 index ccac46b..0000000 --- a/notes.md +++ /dev/null @@ -1,2 +0,0 @@ -* support memory limit as toplevel limit on buffer size when Ok(None) (so only needed in futures/Read version) -* rustfmt modules