tsarevvv/rust-ipfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: merge and adapt the libipld dependency

Browse Source 
Signed-off-by: ljedrz <ljedrz@gmail.com>
This commit is contained in:
ljedrz 2020-08-07 10:12:57 +02:00
parent 16ae43011b
commit b1bc672559
17 changed files with 1865 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -1285,6 +1285,7 @@ dependencies = [
"async-std",
"async-stream",
"async-trait",
"base64 0.12.3",
"bitswap",
"byteorder 1.3.4",
"bytes 0.5.6",
@ -1295,7 +1296,6 @@ dependencies = [
"futures 0.3.5",
"hex-literal",
"ipfs-unixfs",
"libipld",
"libp2p",
"multibase",
"multihash",

6
Cargo.toml
View File

@ -14,8 +14,9 @@ anyhow = { default-features = false, version = "1.0" }
async-std = { default-features = false, features = ["attributes", "std"], version = "1.6" }
async-stream = { default-features = false, version = "0.3" }
async-trait = { default-features = false, version = "0.1" }
base64 = { default-features = false, features = ["alloc"], version = "0.12" }
bitswap = { path = "bitswap" }
byteorder = { default-features = false, version = "1.3" }
byteorder = { default-features = false, version = "1.3" }
bytes = { default-features = false, version = "0.5" }
cid = { default-features = false, version = "0.5" }
dirs = { default-features = false, version = "3.0" }
@ -23,14 +24,13 @@ domain = { default-features = false, version = "0.5" }
domain-resolv = { default-features = false, version = "0.5" }
futures = { default-features = false, features = ["compat", "io-compat"], version = "0.3.5" }
ipfs-unixfs = { path = "unixfs" }
libipld = { branch = "update_cid", default-features = false, features = ["dag-pb", "dag-json"], git = "https://github.com/ljedrz/rust-ipld" }
libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-async-std", "mdns", "mplex", "ping", "secio", "yamux"], version = "0.22" }
multibase = { default-features = false, version = "0.8" }
multihash = { default-features = false, version = "0.11" }
prost = { default-features = false, version = "0.6" }
rand = { default-features = false, features = ["getrandom"], version = "0.7" }
serde = { default-features = false, features = ["derive"], version = "1.0" }
serde_json = { default-features = false, version = "1.0" }
serde_json = { default-features = false, features = ["std"], version = "1.0" }
thiserror = { default-features = false, version = "1.0" }
tracing = { default-features = false, features = ["log"], version = "0.1" }
tracing-futures = { default-features = false, features = ["std", "futures-03"], version = "0.2" }

9
README.md
View File

@ -79,8 +79,7 @@ _Note: binaries available via `cargo install` is coming soon._
```rust,no_run
use async_std::task;
use futures::join;
use ipfs::{Ipfs, IpfsOptions, IpfsPath, Ipld, Types, UninitializedIpfs};
use libipld::ipld;
use ipfs::{make_ipld, Ipfs, IpfsPath, Ipld, Types, UninitializedIpfs};
fn main() {
tracing_subscriber::fmt::init();
@ -91,10 +90,10 @@ fn main() {
task::spawn(fut);
// Create a DAG
let f1 = ipfs.put_dag(ipld!("block1"));
let f2 = ipfs.put_dag(ipld!("block2"));
let f1 = ipfs.put_dag(make_ipld!("block1"));
let f2 = ipfs.put_dag(make_ipld!("block2"));
let (res1, res2) = join!(f1, f2);
let root = ipld!([res1.unwrap(), res2.unwrap()]);
let root = make_ipld!([res1.unwrap(), res2.unwrap()]);
let cid = ipfs.put_dag(root).await.unwrap();
let path = IpfsPath::from(cid);

6
build.rs
View File

@ -1,3 +1,7 @@
fn main() {
prost_build::compile_protos(&["src/ipns/ipns_pb.proto"], &["src"]).unwrap();
prost_build::compile_protos(
&["src/ipns/ipns_pb.proto", "src/ipld/dag_pb.proto"],
&["src"],
)
.unwrap();
}

9
examples/client1.rs
View File

@ -1,7 +1,6 @@
use async_std::task;
use futures::join;
use ipfs::{Ipfs, TestTypes, UninitializedIpfs};
use libipld::ipld;
use ipfs::{make_ipld, Ipfs, TestTypes, UninitializedIpfs};
fn main() {
tracing_subscriber::fmt::init();
@ -11,11 +10,11 @@ fn main() {
UninitializedIpfs::default().await.start().await.unwrap();
task::spawn(fut);
let f1 = ipfs.put_dag(ipld!("block1"));
let f2 = ipfs.put_dag(ipld!("block2"));
let f1 = ipfs.put_dag(make_ipld!("block1"));
let f2 = ipfs.put_dag(make_ipld!("block2"));
let (res1, res2) = join!(f1, f2);
let root = ipld!([res1.unwrap(), res2.unwrap()]);
let root = make_ipld!([res1.unwrap(), res2.unwrap()]);
ipfs.put_dag(root).await.unwrap();
ipfs.exit_daemon().await;

10
http/src/v0/refs.rs
View File

@ -2,9 +2,9 @@ use crate::v0::support::{with_ipfs, MaybeTimeoutExt, StringError};
use cid::{self, Cid};
use futures::stream;
use futures::stream::Stream;
use ipfs::ipld::{decode_ipld, Ipld};
use ipfs::{Block, Error};
use ipfs::{Ipfs, IpfsTypes};
use libipld::{block::decode_ipld, Ipld};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::VecDeque;
@ -161,7 +161,7 @@ pub struct WalkError {
#[derive(Debug)]
pub enum WalkFailed {
Loading(Error),
Parsing(libipld::error::BlockError),
Parsing(ipfs::ipld::BlockError),
DagPb(ipfs::unixfs::ll::ResolveError),
IpldWalking(path::WalkFailed),
}
@ -204,8 +204,8 @@ impl From<Error> for WalkFailed {
}
}
impl From<libipld::error::BlockError> for WalkFailed {
fn from(e: libipld::error::BlockError) -> Self {
impl From<ipfs::ipld::BlockError> for WalkFailed {
fn from(e: ipfs::ipld::BlockError) -> Self {
WalkFailed::Parsing(e)
}
}
@ -621,8 +621,8 @@ mod tests {
use super::{ipld_links, local, refs_paths, Edge, IpfsPath};
use cid::{self, Cid};
use futures::stream::TryStreamExt;
use ipfs::ipld::{decode_ipld, validate};
use ipfs::{Block, Node};
use libipld::block::{decode_ipld, validate};
use std::collections::HashSet;
use std::convert::TryFrom;

10
http/src/v0/refs/path.rs
View File

@ -6,7 +6,7 @@
//! Does not allow the root to be anything else than `/ipfs/` or missing at the moment.
use cid::{self, Cid};
use libipld::Ipld;
use ipfs::ipld::Ipld;
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::fmt;
@ -303,7 +303,7 @@ impl ExactSizeIterator for IpfsPath {
mod tests {
use super::{IpfsPath, WalkSuccess};
use cid::Cid;
use libipld::{ipld, Ipld};
use ipfs::{ipld::Ipld, make_ipld};
use std::convert::TryFrom;
// good_paths, good_but_unsupported, bad_paths from https://github.com/ipfs/go-path/blob/master/path_test.go
@ -389,7 +389,7 @@ mod tests {
fn example_doc_and_a_cid() -> (Ipld, Cid) {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!({
let doc = make_ipld!({
"nested": {
"even": [
{
@ -453,7 +453,7 @@ mod tests {
#[test]
fn walk_link_with_dot() {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!(cid.clone());
let doc = make_ipld!(cid.clone());
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/./foobar";
let mut p = IpfsPath::try_from(path).unwrap();
@ -468,7 +468,7 @@ mod tests {
#[test]
fn walk_link_without_dot_is_unsupported() {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!(cid);
let doc = make_ipld!(cid);
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/foobar";
let mut p = IpfsPath::try_from(path).unwrap();

26
src/dag.rs
View File

@ -1,11 +1,10 @@
use crate::error::Error;
use crate::ipld::{decode_ipld, encode_ipld, Ipld};
use crate::path::{IpfsPath, IpfsPathError, SubPath};
use crate::repo::RepoTypes;
use crate::Ipfs;
use bitswap::Block;
use cid::{Cid, Codec, Version};
use libipld::block::{decode_ipld, encode_ipld};
use libipld::ipld::Ipld;
#[derive(Clone, Debug)]
pub struct IpldDag<Types: RepoTypes> {
@ -94,14 +93,13 @@ fn resolve(ipld: Ipld, sub_path: &SubPath) -> Ipld {
#[cfg(test)]
mod tests {
use super::*;
use crate::Node;
use libipld::ipld;
use crate::{make_ipld, Node};
#[async_std::test]
async fn test_resolve_root_cid() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, 2, 3]);
let data = make_ipld!([1, 2, 3]);
let cid = dag.put(data.clone(), Codec::DagCBOR).await.unwrap();
let res = dag.get(IpfsPath::from(cid)).await.unwrap();
assert_eq!(res, data);
@ -111,33 +109,33 @@ mod tests {
async fn test_resolve_array_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, 2, 3]);
let data = make_ipld!([1, 2, 3]);
let cid = dag.put(data.clone(), Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid).sub_path("1").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(2));
assert_eq!(res, make_ipld!(2));
}
#[async_std::test]
async fn test_resolve_nested_array_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, [2], 3,]);
let data = make_ipld!([1, [2], 3,]);
let cid = dag.put(data, Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid).sub_path("1/0").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(2));
assert_eq!(res, make_ipld!(2));
}
#[async_std::test]
async fn test_resolve_object_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!({
let data = make_ipld!({
"key": false,
});
let cid = dag.put(data, Codec::DagCBOR).await.unwrap();
@ -145,21 +143,21 @@ mod tests {
.get(IpfsPath::from(cid).sub_path("key").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(false));
assert_eq!(res, make_ipld!(false));
}
#[async_std::test]
async fn test_resolve_cid_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data1 = ipld!([1]);
let data1 = make_ipld!([1]);
let cid1 = dag.put(data1, Codec::DagCBOR).await.unwrap();
let data2 = ipld!([cid1]);
let data2 = make_ipld!([cid1]);
let cid2 = dag.put(data2, Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid2).sub_path("0/0").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(1));
assert_eq!(res, make_ipld!(1));
}
}

857
src/ipld/dag_cbor.rs Normal file
View File

@ -0,0 +1,857 @@
//! DAG-CBOR codec.
use crate::ipld::{BlockError, Ipld, IpldError};
use byteorder::{BigEndian, ByteOrder};
use cid::Cid;
use std::{
collections::BTreeMap,
convert::TryFrom,
io::{Read, Write},
};
use thiserror::Error;
/// CBOR codec.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct DagCborCodec;
impl DagCborCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, CborError> {
let mut bytes = Vec::new();
ipld.write_cbor(&mut bytes)?;
Ok(bytes.into_boxed_slice())
}
pub fn decode(mut data: &[u8]) -> Result<Ipld, CborError> {
Ipld::read_cbor(&mut data)
}
}
/// CBOR error.
#[derive(Debug, Error)]
pub enum CborError {
/// Number larger than u64.
#[error("Number larger than u64.")]
NumberOutOfRange,
/// Length larger than usize or too small, for example zero length cid field.
#[error("Length out of range.")]
LengthOutOfRange,
/// Unexpected cbor code.
#[error("Unexpected cbor code.")]
UnexpectedCode,
/// Unknown cbor tag.
#[error("Unkown cbor tag.")]
UnknownTag,
/// Unexpected key.
#[error("Wrong key.")]
UnexpectedKey,
/// Unexpected eof.
#[error("Unexpected end of file.")]
UnexpectedEof,
/// Io error.
#[error("{0}")]
Io(#[from] std::io::Error),
/// Utf8 error.
#[error("{0}")]
Utf8(#[from] std::str::Utf8Error),
/// The byte before Cid was not multibase identity prefix.
#[error("Invalid Cid prefix: {0}")]
InvalidCidPrefix(u8),
/// Cid error.
#[error("{0}")]
Cid(#[from] cid::Error),
/// Ipld error.
#[error("{0}")]
Ipld(#[from] IpldError),
}
impl From<CborError> for BlockError {
fn from(err: CborError) -> Self {
Self::CodecError(err.into())
}
}
/// CBOR result.
pub type CborResult<T> = Result<T, CborError>;
#[inline]
pub fn write_null<W: Write>(w: &mut W) -> CborResult<()> {
w.write_all(&[0xf6])?;
Ok(())
}
#[inline]
pub fn write_u8<W: Write>(w: &mut W, major: u8, value: u8) -> CborResult<()> {
if value <= 0x17 {
let buf = [major << 5 | value];
w.write_all(&buf)?;
} else {
let buf = [major << 5 | 24, value];
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u16<W: Write>(w: &mut W, major: u8, value: u16) -> CborResult<()> {
if value <= u16::from(u8::max_value()) {
write_u8(w, major, value as u8)?;
} else {
let mut buf = [major << 5 | 25, 0, 0];
BigEndian::write_u16(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u32<W: Write>(w: &mut W, major: u8, value: u32) -> CborResult<()> {
if value <= u32::from(u16::max_value()) {
write_u16(w, major, value as u16)?;
} else {
let mut buf = [major << 5 | 26, 0, 0, 0, 0];
BigEndian::write_u32(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u64<W: Write>(w: &mut W, major: u8, value: u64) -> CborResult<()> {
if value <= u64::from(u32::max_value()) {
write_u32(w, major, value as u32)?;
} else {
let mut buf = [major << 5 | 27, 0, 0, 0, 0, 0, 0, 0, 0];
BigEndian::write_u64(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_tag<W: Write>(w: &mut W, tag: u64) -> CborResult<()> {
write_u64(w, 6, tag)
}
pub trait WriteCbor {
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()>;
}
impl WriteCbor for bool {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
let buf = if *self { [0xf5] } else { [0xf4] };
w.write_all(&buf)?;
Ok(())
}
}
impl WriteCbor for u8 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u8(w, 0, *self)
}
}
impl WriteCbor for u16 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u16(w, 0, *self)
}
}
impl WriteCbor for u32 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u32(w, 0, *self)
}
}
impl WriteCbor for u64 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 0, *self)
}
}
impl WriteCbor for i8 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u8(w, 1, -(*self + 1) as u8)
}
}
impl WriteCbor for i16 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u16(w, 1, -(*self + 1) as u16)
}
}
impl WriteCbor for i32 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u32(w, 1, -(*self + 1) as u32)
}
}
impl WriteCbor for i64 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 1, -(*self + 1) as u64)
}
}
impl WriteCbor for f32 {
#[inline]
#[allow(clippy::float_cmp)]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if self.is_infinite() {
if self.is_sign_positive() {
w.write_all(&[0xf9, 0x7c, 0x00])?;
} else {
w.write_all(&[0xf9, 0xfc, 0x00])?;
}
} else if self.is_nan() {
w.write_all(&[0xf9, 0x7e, 0x00])?;
} else {
let mut buf = [0xfa, 0, 0, 0, 0];
BigEndian::write_f32(&mut buf[1..], *self);
w.write_all(&buf)?;
}
Ok(())
}
}
impl WriteCbor for f64 {
#[inline]
#[allow(clippy::float_cmp)]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if !self.is_finite() || f64::from(*self as f32) == *self {
let value = *self as f32;
value.write_cbor(w)?;
} else {
let mut buf = [0xfb, 0, 0, 0, 0, 0, 0, 0, 0];
BigEndian::write_f64(&mut buf[1..], *self);
w.write_all(&buf)?;
}
Ok(())
}
}
impl WriteCbor for [u8] {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 2, self.len() as u64)?;
w.write_all(self)?;
Ok(())
}
}
impl WriteCbor for str {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 3, self.len() as u64)?;
w.write_all(self.as_bytes())?;
Ok(())
}
}
impl WriteCbor for String {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
self.as_str().write_cbor(w)
}
}
impl WriteCbor for i128 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if *self < 0 {
if -(*self + 1) > u64::max_value() as i128 {
return Err(CborError::NumberOutOfRange);
}
write_u64(w, 1, -(*self + 1) as u64)?;
} else {
if *self > u64::max_value() as i128 {
return Err(CborError::NumberOutOfRange);
}
write_u64(w, 0, *self as u64)?;
}
Ok(())
}
}
impl WriteCbor for Cid {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_tag(w, 42)?;
// insert zero byte per https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links
let bytes = self.to_bytes();
write_u64(w, 2, (bytes.len() + 1) as u64)?;
w.write_all(&[0])?;
w.write_all(&bytes)?;
Ok(())
}
}
impl<T: WriteCbor> WriteCbor for Option<T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if let Some(value) = self {
value.write_cbor(w)?;
} else {
write_null(w)?;
}
Ok(())
}
}
impl<T: WriteCbor> WriteCbor for Vec<T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 4, self.len() as u64)?;
for value in self {
value.write_cbor(w)?;
}
Ok(())
}
}
impl<T: WriteCbor + 'static> WriteCbor for BTreeMap<String, T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 5, self.len() as u64)?;
for (k, v) in self {
k.write_cbor(w)?;
v.write_cbor(w)?;
}
Ok(())
}
}
impl WriteCbor for Ipld {
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
match self {
Ipld::Null => write_null(w),
Ipld::Bool(b) => b.write_cbor(w),
Ipld::Integer(i) => i.write_cbor(w),
Ipld::Float(f) => f.write_cbor(w),
Ipld::Bytes(b) => b.as_slice().write_cbor(w),
Ipld::String(s) => s.as_str().write_cbor(w),
Ipld::List(l) => l.write_cbor(w),
Ipld::Map(m) => m.write_cbor(w),
Ipld::Link(c) => c.write_cbor(w),
}
}
}
#[inline]
pub fn read_u8<R: Read>(r: &mut R) -> CborResult<u8> {
let mut buf = [0; 1];
r.read_exact(&mut buf)?;
Ok(buf[0])
}
#[inline]
pub fn read_u16<R: Read>(r: &mut R) -> CborResult<u16> {
let mut buf = [0; 2];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u16(&buf))
}
#[inline]
pub fn read_u32<R: Read>(r: &mut R) -> CborResult<u32> {
let mut buf = [0; 4];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u32(&buf))
}
#[inline]
pub fn read_u64<R: Read>(r: &mut R) -> CborResult<u64> {
let mut buf = [0; 8];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u64(&buf))
}
#[inline]
pub fn read_f32<R: Read>(r: &mut R) -> CborResult<f32> {
let mut buf = [0; 4];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_f32(&buf))
}
#[inline]
pub fn read_f64<R: Read>(r: &mut R) -> CborResult<f64> {
let mut buf = [0; 8];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_f64(&buf))
}
#[inline]
pub fn read_bytes<R: Read>(r: &mut R, len: usize) -> CborResult<Vec<u8>> {
let mut buf = vec![0; len];
r.read_exact(&mut buf)?;
Ok(buf)
}
#[inline]
pub fn read_str<R: Read>(r: &mut R, len: usize) -> CborResult<String> {
let bytes = read_bytes(r, len)?;
let string = std::str::from_utf8(&bytes)?;
Ok(string.to_string())
}
#[inline]
pub fn read_list<R: Read, T: ReadCbor>(r: &mut R, len: usize) -> CborResult<Vec<T>> {
let mut list: Vec<T> = Vec::with_capacity(len);
for _ in 0..len {
list.push(T::read_cbor(r)?);
}
Ok(list)
}
#[inline]
pub fn read_map<R: Read, T: ReadCbor>(r: &mut R, len: usize) -> CborResult<BTreeMap<String, T>> {
let mut map: BTreeMap<String, T> = BTreeMap::new();
for _ in 0..len {
let key = String::read_cbor(r)?;
let value = T::read_cbor(r)?;
map.insert(key, value);
}
Ok(map)
}
#[inline]
pub fn read_link<R: Read>(r: &mut R) -> CborResult<Cid> {
let tag = read_u8(r)?;
if tag != 42 {
return Err(CborError::UnknownTag);
}
let ty = read_u8(r)?;
if ty != 0x58 {
return Err(CborError::UnknownTag);
}
let len = read_u8(r)?;
if len == 0 {
return Err(CborError::LengthOutOfRange);
}
let bytes = read_bytes(r, len as usize)?;
if bytes[0] != 0 {
return Err(CborError::InvalidCidPrefix(bytes[0]));
}
// skip the first byte per
// https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links
Ok(Cid::try_from(&bytes[1..])?)
}
pub trait ReadCbor: Sized {
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>>;
#[inline]
fn read_cbor<R: Read>(r: &mut R) -> CborResult<Self> {
let major = read_u8(r)?;
if let Some(res) = Self::try_read_cbor(r, major)? {
Ok(res)
} else {
Err(CborError::UnexpectedCode)
}
}
}
impl ReadCbor for bool {
#[inline]
fn try_read_cbor<R: Read>(_: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xf4 => Ok(Some(false)),
0xf5 => Ok(Some(true)),
_ => Ok(None),
}
}
}
impl ReadCbor for u8 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major)),
0x18 => Ok(Some(read_u8(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u16 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u16)),
0x18 => Ok(Some(read_u8(r)? as u16)),
0x19 => Ok(Some(read_u16(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u32)),
0x18 => Ok(Some(read_u8(r)? as u32)),
0x19 => Ok(Some(read_u16(r)? as u32)),
0x1a => Ok(Some(read_u32(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u64)),
0x18 => Ok(Some(read_u8(r)? as u64)),
0x19 => Ok(Some(read_u16(r)? as u64)),
0x1a => Ok(Some(read_u32(r)? as u64)),
0x1b => Ok(Some(read_u64(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for i8 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i8)),
0x38 => Ok(Some(-1 - read_u8(r)? as i8)),
_ => Ok(None),
}
}
}
impl ReadCbor for i16 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i16)),
0x38 => Ok(Some(-1 - read_u8(r)? as i16)),
0x39 => Ok(Some(-1 - read_u16(r)? as i16)),
_ => Ok(None),
}
}
}
impl ReadCbor for i32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i32)),
0x38 => Ok(Some(-1 - read_u8(r)? as i32)),
0x39 => Ok(Some(-1 - read_u16(r)? as i32)),
0x3a => Ok(Some(-1 - read_u32(r)? as i32)),
_ => Ok(None),
}
}
}
impl ReadCbor for i64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i64)),
0x38 => Ok(Some(-1 - read_u8(r)? as i64)),
0x39 => Ok(Some(-1 - read_u16(r)? as i64)),
0x3a => Ok(Some(-1 - read_u32(r)? as i64)),
0x3b => Ok(Some(-1 - read_u64(r)? as i64)),
_ => Ok(None),
}
}
}
impl ReadCbor for f32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xfa => Ok(Some(read_f32(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for f64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xfa => Ok(Some(read_f32(r)? as f64)),
0xfb => Ok(Some(read_f64(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for String {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x60..=0x77 => major as usize - 0x60,
0x78 => read_u8(r)? as usize,
0x79 => read_u16(r)? as usize,
0x7a => read_u32(r)? as usize,
0x7b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_str(r, len)?))
}
}
impl ReadCbor for Cid {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xd8 => Ok(Some(read_link(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for Box<[u8]> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x40..=0x57 => major as usize - 0x40,
0x58 => read_u8(r)? as usize,
0x59 => read_u16(r)? as usize,
0x5a => read_u32(r)? as usize,
0x5b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_bytes(r, len)?.into_boxed_slice()))
}
}
impl<T: ReadCbor> ReadCbor for Option<T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xf6 => Ok(Some(None)),
0xf7 => Ok(Some(None)),
_ => {
if let Some(res) = T::try_read_cbor(r, major)? {
Ok(Some(Some(res)))
} else {
Ok(None)
}
}
}
}
}
impl<T: ReadCbor> ReadCbor for Vec<T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x80..=0x97 => major as usize - 0x80,
0x98 => read_u8(r)? as usize,
0x99 => read_u16(r)? as usize,
0x9a => read_u32(r)? as usize,
0x9b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_list(r, len)?))
}
}
impl<T: ReadCbor> ReadCbor for BTreeMap<String, T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0xa0..=0xb7 => major as usize - 0xa0,
0xb8 => read_u8(r)? as usize,
0xb9 => read_u16(r)? as usize,
0xba => read_u32(r)? as usize,
0xbb => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_map(r, len)?))
}
}
impl ReadCbor for Ipld {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let ipld = match major {
// Major type 0: an unsigned integer
0x00..=0x17 => Ipld::Integer(major as i128),
0x18 => Ipld::Integer(read_u8(r)? as i128),
0x19 => Ipld::Integer(read_u16(r)? as i128),
0x1a => Ipld::Integer(read_u32(r)? as i128),
0x1b => Ipld::Integer(read_u64(r)? as i128),
// Major type 1: a negative integer
0x20..=0x37 => Ipld::Integer(-1 - (major - 0x20) as i128),
0x38 => Ipld::Integer(-1 - read_u8(r)? as i128),
0x39 => Ipld::Integer(-1 - read_u16(r)? as i128),
0x3a => Ipld::Integer(-1 - read_u32(r)? as i128),
0x3b => Ipld::Integer(-1 - read_u64(r)? as i128),
// Major type 2: a byte string
0x40..=0x57 => {
let len = major - 0x40;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x58 => {
let len = read_u8(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x59 => {
let len = read_u16(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x5a => {
let len = read_u32(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x5b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
// Major type 3: a text string
0x60..=0x77 => {
let len = major - 0x60;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x78 => {
let len = read_u8(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x79 => {
let len = read_u16(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x7a => {
let len = read_u32(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x7b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
// Major type 4: an array of data items
0x80..=0x97 => {
let len = major - 0x80;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x98 => {
let len = read_u8(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x99 => {
let len = read_u16(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x9a => {
let len = read_u32(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x9b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
// Major type 5: a map of pairs of data items
0xa0..=0xb7 => {
let len = major - 0xa0;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xb8 => {
let len = read_u8(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xb9 => {
let len = read_u16(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xba => {
let len = read_u32(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xbb => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
// Major type 6: optional semantic tagging of other major types
0xd8 => Ipld::Link(read_link(r)?),
// Major type 7: floating-point numbers and other simple data types that need no content
0xf4 => Ipld::Bool(false),
0xf5 => Ipld::Bool(true),
0xf6 => Ipld::Null,
0xf7 => Ipld::Null,
0xfa => Ipld::Float(read_f32(r)? as f64),
0xfb => Ipld::Float(read_f64(r)?),
_ => return Ok(None),
};
Ok(Some(ipld))
}
}

247
src/ipld/dag_json.rs Normal file
View File

@ -0,0 +1,247 @@
//! DAG-JSON codec.
use crate::ipld::{BlockError, Ipld};
use cid::Cid;
use core::convert::TryFrom;
use serde::de::Error as SerdeError;
use serde::{de, ser, Deserialize, Serialize};
use serde_json::ser::Serializer;
use serde_json::Error;
use std::collections::BTreeMap;
use std::fmt;
use std::iter::FromIterator;
/// Json codec.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct DagJsonCodec;
impl DagJsonCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, BlockError> {
json_encode(ipld).map_err(|e| BlockError::CodecError(e.into()))
}
pub fn decode(data: &[u8]) -> Result<Ipld, BlockError> {
json_decode(data).map_err(|e| BlockError::CodecError(e.into()))
}
}
const LINK_KEY: &str = "/";
pub fn json_encode(ipld: &Ipld) -> Result<Box<[u8]>, Error> {
let mut writer = Vec::with_capacity(128);
let mut ser = Serializer::new(&mut writer);
serialize(&ipld, &mut ser)?;
Ok(writer.into_boxed_slice())
}
pub fn json_decode(data: &[u8]) -> Result<Ipld, Error> {
let mut de = serde_json::Deserializer::from_slice(&data);
Ok(deserialize(&mut de)?)
}
fn serialize<S>(ipld: &Ipld, ser: S) -> Result<S::Ok, S::Error>
where
S: ser::Serializer,
{
match &ipld {
Ipld::Null => ser.serialize_none(),
Ipld::Bool(bool) => ser.serialize_bool(*bool),
Ipld::Integer(i128) => ser.serialize_i128(*i128),
Ipld::Float(f64) => ser.serialize_f64(*f64),
Ipld::String(string) => ser.serialize_str(&string),
Ipld::Bytes(bytes) => ser.serialize_bytes(&bytes),
Ipld::List(list) => {
let wrapped = list.iter().map(|ipld| Wrapper(ipld));
ser.collect_seq(wrapped)
}
Ipld::Map(map) => {
let wrapped = map.iter().map(|(key, ipld)| (key, Wrapper(ipld)));
ser.collect_map(wrapped)
}
Ipld::Link(link) => {
let value = base64::encode(&link.to_bytes());
let mut map = BTreeMap::new();
map.insert("/", value);
ser.collect_map(map)
}
}
}
fn deserialize<'de, D>(deserializer: D) -> Result<Ipld, D::Error>
where
D: de::Deserializer<'de>,
{
deserializer.deserialize_any(JSONVisitor)
}
// Needed for `collect_seq` and `collect_map` in Seserializer
struct Wrapper<'a>(&'a Ipld);
impl<'a> Serialize for Wrapper<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: ser::Serializer,
{
serialize(&self.0, serializer)
}
}
// serde deserializer visitor that is used by Deseraliazer to decode
// json into IPLD.
struct JSONVisitor;
impl<'de> de::Visitor<'de> for JSONVisitor {
type Value = Ipld;
fn expecting(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.write_str("any valid JSON value")
}
#[inline]
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_string(String::from(value))
}
#[inline]
fn visit_string<E>(self, value: String) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::String(value))
}
#[inline]
fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_byte_buf(v.to_owned())
}
#[inline]
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Bytes(v))
}
#[inline]
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v.into()))
}
#[inline]
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v.into()))
}
#[inline]
fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v))
}
#[inline]
fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Bool(v))
}
#[inline]
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_unit()
}
#[inline]
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Null)
}
#[inline]
fn visit_seq<V>(self, mut visitor: V) -> Result<Self::Value, V::Error>
where
V: de::SeqAccess<'de>,
{
let mut vec: Vec<WrapperOwned> = Vec::new();
while let Some(elem) = visitor.next_element()? {
vec.push(elem);
}
let unwrapped = vec.into_iter().map(|WrapperOwned(ipld)| ipld).collect();
Ok(Ipld::List(unwrapped))
}
#[inline]
fn visit_map<V>(self, mut visitor: V) -> Result<Self::Value, V::Error>
where
V: de::MapAccess<'de>,
{
let mut values: Vec<(String, WrapperOwned)> = Vec::new();
while let Some((key, value)) = visitor.next_entry()? {
values.push((key, value));
}
// JSON Object represents IPLD Link if it is `{ "/": "...." }` therefor
// we valiadet if that is the case here.
if let Some((key, WrapperOwned(Ipld::String(value)))) = values.first() {
if key == LINK_KEY && values.len() == 1 {
let link = base64::decode(value).map_err(SerdeError::custom)?;
let cid = Cid::try_from(link).map_err(SerdeError::custom)?;
return Ok(Ipld::Link(cid));
}
}
let unwrapped = values
.into_iter()
.map(|(key, WrapperOwned(value))| (key, value));
Ok(Ipld::Map(BTreeMap::from_iter(unwrapped)))
}
#[inline]
fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Float(v))
}
}
// Needed for `visit_seq` and `visit_map` in Deserializer
/// We cannot directly implement `serde::Deserializer` for `Ipld` as it is a remote type.
/// Instead wrap it into a newtype struct and implement `serde::Deserialize` for that one.
/// All the deserializer does is calling the `deserialize()` function we defined which returns
/// an unwrapped `Ipld` instance. Wrap that `Ipld` instance in `Wrapper` and return it.
/// Users of this wrapper will then unwrap it again so that they can return the expected `Ipld`
/// instance.
struct WrapperOwned(Ipld);
impl<'de> Deserialize<'de> for WrapperOwned {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: de::Deserializer<'de>,
{
let deserialized = deserialize(deserializer);
// Better version of Ok(Wrapper(deserialized.unwrap()))
deserialized.map(Self)
}
}

22
src/ipld/dag_pb.proto Normal file
View File

@ -0,0 +1,22 @@
// https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-pb.md
syntax = "proto3";
package dag_pb;
// An IPFS MerkleDAG Link
message PBLink {
// binary CID (with no multibase prefix) of the target object
bytes Hash = 1;
// UTF-8 string name
string Name = 2;
// cumulative size of target object
uint64 Tsize = 3;
}
// An IPFS MerkleDAG Node
message PBNode {
// refs to other objects
repeated PBLink Links = 2;
// opaque user data
bytes Data = 1;
}

163
src/ipld/dag_pb.rs Normal file
View File

@ -0,0 +1,163 @@
//! DAG-Protobuf codec.
use crate::ipld::{BlockError, Ipld, IpldError};
use cid::Cid;
use std::{
collections::BTreeMap,
convert::{TryFrom, TryInto},
};
use thiserror::Error;
/// Protobuf codec.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct DagPbCodec;
impl DagPbCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, ProtobufError> {
let pb_node: PbNode = ipld.try_into()?;
Ok(pb_node.into_bytes())
}
pub fn decode(data: &[u8]) -> Result<Ipld, ProtobufError> {
Ok(PbNode::from_bytes(data)?.into())
}
}
/// Protobuf error.
#[derive(Debug, Error)]
pub enum ProtobufError {
#[error("{0}")]
Prost(#[from] prost::DecodeError),
#[error("{0}")]
Cid(#[from] cid::Error),
#[error("{0}")]
Ipld(#[from] IpldError),
}
impl From<ProtobufError> for BlockError {
fn from(error: ProtobufError) -> Self {
Self::CodecError(error.into())
}
}
mod pb {
include!(concat!(env!("OUT_DIR"), "/dag_pb.rs"));
}
pub struct PbLink {
pub cid: Cid,
pub name: String,
pub size: u64,
}
pub struct PbNode {
pub links: Vec<PbLink>,
pub data: Vec<u8>,
}
use prost::Message;
impl PbNode {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, ProtobufError> {
let proto: pb::PbNode = pb::PbNode::decode(bytes)?;
let data = proto.data;
let mut links = Vec::new();
for link in proto.links {
let cid = Cid::try_from(link.hash)?;
let name = link.name;
let size = link.tsize;
links.push(PbLink { cid, name, size });
}
Ok(PbNode { links, data })
}
pub fn into_bytes(self) -> Box<[u8]> {
let links = self
.links
.into_iter()
.map(|link| pb::PbLink {
hash: link.cid.to_bytes(),
name: link.name,
tsize: link.size,
})
.collect::<Vec<_>>();
let proto = pb::PbNode {
data: self.data,
links,
};
let mut res = Vec::with_capacity(proto.encoded_len());
proto
.encode(&mut res)
.expect("there is no situation in which the protobuf message can be invalid");
res.into_boxed_slice()
}
}
impl Into<Ipld> for PbNode {
fn into(self) -> Ipld {
let mut map = BTreeMap::<String, Ipld>::new();
let links = self
.links
.into_iter()
.map(|link| link.into())
.collect::<Vec<Ipld>>();
map.insert("Links".to_string(), links.into());
map.insert("Data".to_string(), self.data.into());
map.into()
}
}
impl Into<Ipld> for PbLink {
fn into(self) -> Ipld {
let mut map = BTreeMap::<String, Ipld>::new();
map.insert("Hash".to_string(), self.cid.into());
map.insert("Name".to_string(), self.name.into());
map.insert("Tsize".to_string(), self.size.into());
map.into()
}
}
impl TryFrom<&Ipld> for PbNode {
type Error = IpldError;
fn try_from(ipld: &Ipld) -> Result<PbNode, Self::Error> {
let links = if let Ipld::List(links) = ipld.get("Links").ok_or(IpldError::KeyNotFound)? {
links
.iter()
.map(|link| link.try_into())
.collect::<Result<_, _>>()?
} else {
return Err(IpldError::NotList);
};
let data = if let Ipld::Bytes(data) = ipld.get("Data").ok_or(IpldError::KeyNotFound)? {
data.clone()
} else {
return Err(IpldError::NotBytes);
};
Ok(PbNode { links, data })
}
}
impl TryFrom<&Ipld> for PbLink {
type Error = IpldError;
fn try_from(ipld: &Ipld) -> Result<PbLink, Self::Error> {
let cid = if let Ipld::Link(cid) = ipld.get("Hash").ok_or(IpldError::KeyNotFound)? {
cid.clone()
} else {
return Err(IpldError::NotLink);
};
let name = if let Ipld::String(name) = ipld.get("Name").ok_or(IpldError::KeyNotFound)? {
name.clone()
} else {
return Err(IpldError::NotString);
};
let size = if let Ipld::Integer(size) = ipld.get("Tsize").ok_or(IpldError::KeyNotFound)? {
*size as u64
} else {
return Err(IpldError::NotInteger);
};
Ok(PbLink { cid, name, size })
}
}

236
src/ipld/ipld_macro.rs Normal file
View File

@ -0,0 +1,236 @@
#[macro_export(local_inner_macros)]
macro_rules! make_ipld {
// Hide distracting implementation details from the generated rustdoc.
($($ipld:tt)+) => {
ipld_internal!($($ipld)+)
};
}
#[macro_export(local_inner_macros)]
#[doc(hidden)]
macro_rules! ipld_internal {
//////////////////////////////////////////////////////////////////////////
// TT muncher for parsing the inside of an array [...]. Produces a vec![...]
// of the elements.
//
// Must be invoked as: ipld_internal!(@array [] $($tt)*)
//////////////////////////////////////////////////////////////////////////
// Done with trailing comma.
(@array [$($elems:expr,)*]) => {
ipld_internal_vec![$($elems,)*]
};
// Done without trailing comma.
(@array [$($elems:expr),*]) => {
ipld_internal_vec![$($elems),*]
};
// Next element is `null`.
(@array [$($elems:expr,)*] null $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(null)] $($rest)*)
};
// Next element is `true`.
(@array [$($elems:expr,)*] true $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(true)] $($rest)*)
};
// Next element is `false`.
(@array [$($elems:expr,)*] false $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(false)] $($rest)*)
};
// Next element is an array.
(@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!([$($array)*])] $($rest)*)
};
// Next element is a map.
(@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!({$($map)*})] $($rest)*)
};
// Next element is an expression followed by comma.
(@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!($next),] $($rest)*)
};
// Last element is an expression with no trailing comma.
(@array [$($elems:expr,)*] $last:expr) => {
ipld_internal!(@array [$($elems,)* ipld_internal!($last)])
};
// Comma after the most recent element.
(@array [$($elems:expr),*] , $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)*] $($rest)*)
};
// Unexpected token after most recent element.
(@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => {
ipld_unexpected!($unexpected)
};
//////////////////////////////////////////////////////////////////////////
// TT muncher for parsing the inside of an object {...}. Each entry is
// inserted into the given map variable.
//
// Must be invoked as: json_internal!(@object $map () ($($tt)*) ($($tt)*))
//
// We require two copies of the input tokens so that we can match on one
// copy and trigger errors on the other copy.
//////////////////////////////////////////////////////////////////////////
// Done.
(@object $object:ident () () ()) => {};
// Insert the current entry followed by trailing comma.
(@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
let _ = $object.insert(($($key)+).into(), $value);
ipld_internal!(@object $object () ($($rest)*) ($($rest)*));
};
// Current entry followed by unexpected token.
(@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => {
ipld_unexpected!($unexpected);
};
// Insert the last entry without trailing comma.
(@object $object:ident [$($key:tt)+] ($value:expr)) => {
let _ = $object.insert(($($key)+).into(), $value);
};
// Next value is `null`.
(@object $object:ident ($($key:tt)+) (: null $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(null)) $($rest)*);
};
// Next value is `true`.
(@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(true)) $($rest)*);
};
// Next value is `false`.
(@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(false)) $($rest)*);
};
// Next value is an array.
(@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!([$($array)*])) $($rest)*);
};
// Next value is a map.
(@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!({$($map)*})) $($rest)*);
};
// Next value is an expression followed by comma.
(@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!($value)) , $($rest)*);
};
// Last value is an expression with no trailing comma.
(@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!($value)));
};
// Missing value for last entry. Trigger a reasonable error message.
(@object $object:ident ($($key:tt)+) (:) $copy:tt) => {
// "unexpected end of macro invocation"
ipld_internal!();
};
// Missing colon and value for last entry. Trigger a reasonable error
// message.
(@object $object:ident ($($key:tt)+) () $copy:tt) => {
// "unexpected end of macro invocation"
ipld_internal!();
};
// Misplaced colon. Trigger a reasonable error message.
(@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => {
// Takes no arguments so "no rules expected the token `:`".
ipld_unexpected!($colon);
};
// Found a comma inside a key. Trigger a reasonable error message.
(@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => {
// Takes no arguments so "no rules expected the token `,`".
ipld_unexpected!($comma);
};
// Key is fully parenthesized. This avoids clippy double_parens false
// positives because the parenthesization may be necessary here.
(@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*));
};
// Munch a token into the current key.
(@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*));
};
//////////////////////////////////////////////////////////////////////////
// The main implementation.
//
// Must be invoked as: json_internal!($($json)+)
//////////////////////////////////////////////////////////////////////////
(null) => {
$crate::Ipld::Null
};
(true) => {
$crate::Ipld::Bool(true)
};
(false) => {
$crate::Ipld::Bool(false)
};
([]) => {
$crate::Ipld::List(ipld_internal_vec![])
};
([ $($tt:tt)+ ]) => {
$crate::Ipld::List(ipld_internal!(@array [] $($tt)+))
};
({}) => {
$crate::Ipld::Map(std::collections::BTreeMap::new())
};
({ $($tt:tt)+ }) => {
$crate::Ipld::Map({
let mut object = std::collections::BTreeMap::new();
ipld_internal!(@object object () ($($tt)+) ($($tt)+));
object
})
};
// Any Serialize type: numbers, strings, struct literals, variables etc.
// Must be below every other rule.
($other:expr) => {
{
$crate::Ipld::from($other)
}
};
}
// The json_internal macro above cannot invoke vec directly because it uses
// local_inner_macros. A vec invocation there would resolve to $crate::vec.
// Instead invoke vec here outside of local_inner_macros.
#[macro_export]
#[doc(hidden)]
macro_rules! ipld_internal_vec {
($($content:tt)*) => {
vec![$($content)*]
};
}
#[macro_export]
#[doc(hidden)]
macro_rules! ipld_unexpected {
() => {};
}

293
src/ipld/mod.rs Normal file
View File

@ -0,0 +1,293 @@
// This code was adapted from https://github.com/ipfs-rust/rust-ipld, and most of
// its code is the same as at revision b2286c53c13f3eeec2a3766387f2926838e8e4c9;
// it used to be a direct dependency, but recent updates to cid and multihash crates
// made it incompatible with them.
pub mod dag_cbor;
pub mod dag_json;
pub mod dag_pb;
#[macro_use]
pub mod ipld_macro;
use cid::{Cid, Codec};
use dag_cbor::DagCborCodec;
use dag_json::DagJsonCodec;
use dag_pb::DagPbCodec;
use multihash::Multihash;
use std::collections::BTreeMap;
use thiserror::Error;
/// Ipld
#[derive(Clone, Debug, PartialEq)]
pub enum Ipld {
/// Represents the absence of a value or the value undefined.
Null,
/// Represents a boolean value.
Bool(bool),
/// Represents an integer.
Integer(i128),
/// Represents a floating point value.
Float(f64),
/// Represents an UTF-8 string.
String(String),
/// Represents a sequence of bytes.
Bytes(Vec<u8>),
/// Represents a list.
List(Vec<Ipld>),
/// Represents a map.
Map(BTreeMap<String, Ipld>),
/// Represents a link to an Ipld node
Link(Cid),
}
macro_rules! derive_to_ipld_prim {
($enum:ident, $ty:ty, $fn:ident) => {
impl From<$ty> for Ipld {
fn from(t: $ty) -> Self {
Ipld::$enum(t.$fn() as _)
}
}
};
}
macro_rules! derive_to_ipld {
($enum:ident, $ty:ty, $fn:ident) => {
impl From<$ty> for Ipld {
fn from(t: $ty) -> Self {
Ipld::$enum(t.$fn())
}
}
};
}
derive_to_ipld!(Bool, bool, clone);
derive_to_ipld_prim!(Integer, i8, clone);
derive_to_ipld_prim!(Integer, i16, clone);
derive_to_ipld_prim!(Integer, i32, clone);
derive_to_ipld_prim!(Integer, i64, clone);
derive_to_ipld_prim!(Integer, i128, clone);
derive_to_ipld_prim!(Integer, isize, clone);
derive_to_ipld_prim!(Integer, u8, clone);
derive_to_ipld_prim!(Integer, u16, clone);
derive_to_ipld_prim!(Integer, u32, clone);
derive_to_ipld_prim!(Integer, u64, clone);
derive_to_ipld_prim!(Integer, usize, clone);
derive_to_ipld_prim!(Float, f32, clone);
derive_to_ipld_prim!(Float, f64, clone);
derive_to_ipld!(String, String, into);
derive_to_ipld!(String, &str, to_string);
derive_to_ipld!(Bytes, Vec<u8>, into);
derive_to_ipld!(Bytes, &[u8], to_vec);
derive_to_ipld!(List, Vec<Ipld>, into);
derive_to_ipld!(Map, BTreeMap<String, Ipld>, to_owned);
derive_to_ipld!(Link, Cid, clone);
derive_to_ipld!(Link, &Cid, to_owned);
/// An index into ipld
pub enum IpldIndex<'a> {
/// An index into an ipld list.
List(usize),
/// An owned index into an ipld map.
Map(String),
/// An index into an ipld map.
MapRef(&'a str),
}
impl<'a> From<usize> for IpldIndex<'a> {
fn from(index: usize) -> Self {
Self::List(index)
}
}
impl<'a> From<String> for IpldIndex<'a> {
fn from(key: String) -> Self {
Self::Map(key)
}
}
impl<'a> From<&'a str> for IpldIndex<'a> {
fn from(key: &'a str) -> Self {
Self::MapRef(key)
}
}
impl Ipld {
/// Indexes into a ipld list or map.
pub fn get<'a, T: Into<IpldIndex<'a>>>(&self, index: T) -> Option<&Ipld> {
match self {
Ipld::List(l) => match index.into() {
IpldIndex::List(i) => l.get(i),
_ => None,
},
Ipld::Map(m) => match index.into() {
IpldIndex::Map(ref key) => m.get(key),
IpldIndex::MapRef(key) => m.get(key),
_ => None,
},
_ => None,
}
}
/// Returns an iterator.
pub fn iter(&self) -> IpldIter<'_> {
IpldIter {
stack: vec![Box::new(vec![self].into_iter())],
}
}
}
/// Ipld iterator.
pub struct IpldIter<'a> {
stack: Vec<Box<dyn Iterator<Item = &'a Ipld> + 'a>>,
}
impl<'a> Iterator for IpldIter<'a> {
type Item = &'a Ipld;
fn next(&mut self) -> Option<Self::Item> {
loop {
if let Some(iter) = self.stack.last_mut() {
if let Some(ipld) = iter.next() {
match ipld {
Ipld::List(list) => {
self.stack.push(Box::new(list.iter()));
}
Ipld::Map(map) => {
self.stack.push(Box::new(map.values()));
}
_ => {}
}
return Some(ipld);
} else {
self.stack.pop();
}
} else {
return None;
}
}
}
}
/// Ipld type error.
#[derive(Debug, Error)]
pub enum IpldError {
/// Expected a boolean.
#[error("Expected a boolean.")]
NotBool,
/// Expected an integer.
#[error("Expected an integer.")]
NotInteger,
/// Expected a float.
#[error("Expected a float.")]
NotFloat,
/// Expected a string.
#[error("Expected a string.")]
NotString,
/// Expected bytes.
#[error("Expected bytes.")]
NotBytes,
/// Expected a list.
#[error("Expected a list.")]
NotList,
/// Expected a map.
#[error("Expected a map.")]
NotMap,
/// Expected a cid.
#[error("Expected a cid.")]
NotLink,
/// Expected a key.
#[error("Expected a key.")]
NotKey,
/// Index not found.
#[error("Index not found.")]
IndexNotFound,
/// Key not found.
#[error("Key not found.")]
KeyNotFound,
}
/// Block error.
#[derive(Debug, Error)]
pub enum BlockError {
/// Block exceeds MAX_BLOCK_SIZE.
#[error("Block size {0} exceeds MAX_BLOCK_SIZE.")]
BlockTooLarge(usize),
/// Hash does not match the CID.
#[error("Hash does not match the CID.")]
InvalidHash(Multihash),
/// The codec is unsupported.
#[error("Unsupported codec {0:?}.")]
UnsupportedCodec(cid::Codec),
/// The multihash is unsupported.
#[error("Unsupported multihash {0:?}.")]
UnsupportedMultihash(multihash::Code),
/// The codec returned an error.
#[error("Codec error: {0}")]
CodecError(Box<dyn std::error::Error + Send + Sync>),
/// Io error.
#[error("{0}")]
Io(std::io::Error),
/// Cid error.
#[error("{0}")]
Cid(cid::Error),
/// Link error.
#[error("Invalid link.")]
InvalidLink,
}
impl From<std::io::Error> for BlockError {
fn from(err: std::io::Error) -> Self {
Self::Io(err)
}
}
impl From<cid::Error> for BlockError {
fn from(err: cid::Error) -> Self {
Self::Cid(err)
}
}
/// The maximum block size is 1MiB.
pub const MAX_BLOCK_SIZE: usize = 1_048_576;
/// Validate a block.
pub fn validate(cid: &Cid, data: &[u8]) -> Result<(), BlockError> {
if data.len() > MAX_BLOCK_SIZE {
return Err(BlockError::BlockTooLarge(data.len()));
}
let hash = cid.hash().algorithm().digest(&data);
if hash.as_ref() != cid.hash() {
return Err(BlockError::InvalidHash(hash));
}
Ok(())
}
/// Encode ipld to bytes.
pub fn encode_ipld(ipld: &Ipld, codec: Codec) -> Result<Box<[u8]>, BlockError> {
let bytes = match codec {
Codec::DagCBOR => DagCborCodec::encode(ipld)?,
Codec::DagProtobuf => DagPbCodec::encode(ipld)?,
Codec::DagJSON => DagJsonCodec::encode(ipld)?,
Codec::Raw => {
if let Ipld::Bytes(bytes) = ipld {
bytes.to_vec().into_boxed_slice()
} else {
return Err(BlockError::CodecError(IpldError::NotBytes.into()));
}
}
_ => return Err(BlockError::UnsupportedCodec(codec)),
};
Ok(bytes)
}
/// Decode block to ipld.
pub fn decode_ipld(cid: &Cid, data: &[u8]) -> Result<Ipld, BlockError> {
let ipld = match cid.codec() {
Codec::DagCBOR => DagCborCodec::decode(data)?,
Codec::DagProtobuf => DagPbCodec::decode(data)?,
Codec::DagJSON => DagJsonCodec::decode(data)?,
Codec::Raw => Ipld::Bytes(data.to_vec()),
_ => return Err(BlockError::UnsupportedCodec(cid.codec())),
};
Ok(ipld)
}

10
src/lib.rs
View File

@ -6,6 +6,7 @@
#[macro_use]
extern crate tracing;
pub use crate::ipld::Ipld;
use anyhow::{anyhow, format_err};
use async_std::path::PathBuf;
pub use bitswap::{BitswapEvent, Block, Stats};
@ -15,7 +16,6 @@ use futures::channel::mpsc::{channel, Receiver, Sender};
use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender};
use futures::sink::SinkExt;
use futures::stream::{Fuse, Stream};
pub use libipld::ipld::Ipld;
pub use libp2p::core::{connection::ListenerId, ConnectedPoint, Multiaddr, PeerId, PublicKey};
pub use libp2p::identity::Keypair;
use tracing::Span;
@ -33,6 +33,8 @@ use std::task::{Context, Poll};
mod config;
mod dag;
pub mod error;
#[macro_use]
pub mod ipld;
pub mod ipns;
pub mod p2p;
pub mod path;
@ -1203,7 +1205,7 @@ fn could_be_bound_from_ephemeral(
#[cfg(test)]
mod tests {
use super::*;
use libipld::ipld;
use crate::make_ipld;
use libp2p::build_multiaddr;
use multihash::Sha2_256;
@ -1289,7 +1291,7 @@ mod tests {
async fn test_put_and_get_dag() {
let ipfs = Node::new("test_node").await;
let data = ipld!([-1, -2, -3]);
let data = make_ipld!([-1, -2, -3]);
let cid = ipfs.put_dag(data.clone()).await.unwrap();
let new_data = ipfs.get_dag(cid.into()).await.unwrap();
assert_eq!(data, new_data);
@ -1299,7 +1301,7 @@ mod tests {
async fn test_pin_and_unpin() {
let ipfs = Node::new("test_node").await;
let data = ipld!([-1, -2, -3]);
let data = make_ipld!([-1, -2, -3]);
let cid = ipfs.put_dag(data.clone()).await.unwrap();
ipfs.pin_block(&cid).await.unwrap();

2
src/path.rs
View File

@ -1,7 +1,7 @@
use crate::error::{Error, TryError};
use crate::ipld::Ipld;
use cid::Cid;
use core::convert::{TryFrom, TryInto};
use libipld::ipld::Ipld;
use libp2p::PeerId;
use std::fmt;
use std::str::FromStr;

3
src/unixfs/mod.rs
View File

@ -1,13 +1,12 @@
use crate::dag::IpldDag;
use crate::error::Error;
use crate::ipld::{dag_pb::PbNode, Ipld};
use crate::path::IpfsPath;
use crate::repo::RepoTypes;
use async_std::fs;
use async_std::io::ReadExt;
use async_std::path::PathBuf;
use cid::{Cid, Codec};
use libipld::ipld::Ipld;
use libipld::pb::PbNode;
use std::collections::BTreeMap;
use std::convert::TryInto;