tsarevvv/rust-ipfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge #295

Browse Source 
295: Merge and adapt the libipld dependency r=koivunej a=ljedrz

Our [libipld](https://crates.io/crates/libipld) dependency has been out of date with _its_ dependencies for a while now and since we don't need to maintain a shadow fork of the entire repo in order to be able to keep up, we might as well merge the bits we need until the upstream repo(s) are updated and we can depend on them directly again.

All the copied (with minor adjustments) code resides in the `ipld` module where the source repo is credited.

Co-authored-by: ljedrz <ljedrz@gmail.com>
This commit is contained in:
bors[bot] 2020-08-10 12:31:35 +00:00 committed by GitHub
commit 1b1aab339b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1865 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -1285,6 +1285,7 @@ dependencies = [
"async-std",
"async-stream",
"async-trait",
"base64 0.12.3",
"bitswap",
"byteorder 1.3.4",
"bytes 0.5.6",
@ -1295,7 +1296,6 @@ dependencies = [
"futures 0.3.5",
"hex-literal",
"ipfs-unixfs",
"libipld",
"libp2p",
"multibase",
"multihash",

6
Cargo.toml
View File

@ -14,8 +14,9 @@ anyhow = { default-features = false, version = "1.0" }
async-std = { default-features = false, features = ["attributes", "std"], version = "1.6" }
async-stream = { default-features = false, version = "0.3" }
async-trait = { default-features = false, version = "0.1" }
base64 = { default-features = false, features = ["alloc"], version = "0.12" }
bitswap = { path = "bitswap" }
byteorder = { default-features = false, version = "1.3" }
byteorder = { default-features = false, version = "1.3" }
bytes = { default-features = false, version = "0.5" }
cid = { default-features = false, version = "0.5" }
dirs = { default-features = false, version = "3.0" }
@ -23,14 +24,13 @@ domain = { default-features = false, version = "0.5" }
domain-resolv = { default-features = false, version = "0.5" }
futures = { default-features = false, features = ["compat", "io-compat"], version = "0.3.5" }
ipfs-unixfs = { path = "unixfs" }
libipld = { branch = "update_cid", default-features = false, features = ["dag-pb", "dag-json"], git = "https://github.com/ljedrz/rust-ipld" }
libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-async-std", "mdns", "mplex", "ping", "secio", "yamux"], version = "0.22" }
multibase = { default-features = false, version = "0.8" }
multihash = { default-features = false, version = "0.11" }
prost = { default-features = false, version = "0.6" }
rand = { default-features = false, features = ["getrandom"], version = "0.7" }
serde = { default-features = false, features = ["derive"], version = "1.0" }
serde_json = { default-features = false, version = "1.0" }
serde_json = { default-features = false, features = ["std"], version = "1.0" }
thiserror = { default-features = false, version = "1.0" }
tracing = { default-features = false, features = ["log"], version = "0.1" }
tracing-futures = { default-features = false, features = ["std", "futures-03"], version = "0.2" }

9
README.md
View File

@ -79,8 +79,7 @@ _Note: binaries available via `cargo install` is coming soon._
```rust,no_run
use async_std::task;
use futures::join;
use ipfs::{Ipfs, IpfsOptions, IpfsPath, Ipld, Types, UninitializedIpfs};
use libipld::ipld;
use ipfs::{make_ipld, Ipfs, IpfsPath, Ipld, Types, UninitializedIpfs};
fn main() {
tracing_subscriber::fmt::init();
@ -91,10 +90,10 @@ fn main() {
task::spawn(fut);
// Create a DAG
let f1 = ipfs.put_dag(ipld!("block1"));
let f2 = ipfs.put_dag(ipld!("block2"));
let f1 = ipfs.put_dag(make_ipld!("block1"));
let f2 = ipfs.put_dag(make_ipld!("block2"));
let (res1, res2) = join!(f1, f2);
let root = ipld!([res1.unwrap(), res2.unwrap()]);
let root = make_ipld!([res1.unwrap(), res2.unwrap()]);
let cid = ipfs.put_dag(root).await.unwrap();
let path = IpfsPath::from(cid);

6
build.rs
View File

@ -1,3 +1,7 @@
fn main() {
prost_build::compile_protos(&["src/ipns/ipns_pb.proto"], &["src"]).unwrap();
prost_build::compile_protos(
&["src/ipns/ipns_pb.proto", "src/ipld/dag_pb.proto"],
&["src"],
)
.unwrap();
}

9
examples/client1.rs
View File

@ -1,7 +1,6 @@
use async_std::task;
use futures::join;
use ipfs::{Ipfs, TestTypes, UninitializedIpfs};
use libipld::ipld;
use ipfs::{make_ipld, Ipfs, TestTypes, UninitializedIpfs};
fn main() {
tracing_subscriber::fmt::init();
@ -11,11 +10,11 @@ fn main() {
UninitializedIpfs::default().await.start().await.unwrap();
task::spawn(fut);
let f1 = ipfs.put_dag(ipld!("block1"));
let f2 = ipfs.put_dag(ipld!("block2"));
let f1 = ipfs.put_dag(make_ipld!("block1"));
let f2 = ipfs.put_dag(make_ipld!("block2"));
let (res1, res2) = join!(f1, f2);
let root = ipld!([res1.unwrap(), res2.unwrap()]);
let root = make_ipld!([res1.unwrap(), res2.unwrap()]);
ipfs.put_dag(root).await.unwrap();
ipfs.exit_daemon().await;

10
http/src/v0/refs.rs
View File

@ -2,9 +2,9 @@ use crate::v0::support::{with_ipfs, MaybeTimeoutExt, StringError};
use cid::{self, Cid};
use futures::stream;
use futures::stream::Stream;
use ipfs::ipld::{decode_ipld, Ipld};
use ipfs::{Block, Error};
use ipfs::{Ipfs, IpfsTypes};
use libipld::{block::decode_ipld, Ipld};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::VecDeque;
@ -161,7 +161,7 @@ pub struct WalkError {
#[derive(Debug)]
pub enum WalkFailed {
Loading(Error),
Parsing(libipld::error::BlockError),
Parsing(ipfs::ipld::BlockError),
DagPb(ipfs::unixfs::ll::ResolveError),
IpldWalking(path::WalkFailed),
}
@ -204,8 +204,8 @@ impl From<Error> for WalkFailed {
}
}
impl From<libipld::error::BlockError> for WalkFailed {
fn from(e: libipld::error::BlockError) -> Self {
impl From<ipfs::ipld::BlockError> for WalkFailed {
fn from(e: ipfs::ipld::BlockError) -> Self {
WalkFailed::Parsing(e)
}
}
@ -621,8 +621,8 @@ mod tests {
use super::{ipld_links, local, refs_paths, Edge, IpfsPath};
use cid::{self, Cid};
use futures::stream::TryStreamExt;
use ipfs::ipld::{decode_ipld, validate};
use ipfs::{Block, Node};
use libipld::block::{decode_ipld, validate};
use std::collections::HashSet;
use std::convert::TryFrom;

10
http/src/v0/refs/path.rs
View File

@ -6,7 +6,7 @@
//! Does not allow the root to be anything else than `/ipfs/` or missing at the moment.
use cid::{self, Cid};
use libipld::Ipld;
use ipfs::ipld::Ipld;
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::fmt;
@ -303,7 +303,7 @@ impl ExactSizeIterator for IpfsPath {
mod tests {
use super::{IpfsPath, WalkSuccess};
use cid::Cid;
use libipld::{ipld, Ipld};
use ipfs::{ipld::Ipld, make_ipld};
use std::convert::TryFrom;
// good_paths, good_but_unsupported, bad_paths from https://github.com/ipfs/go-path/blob/master/path_test.go
@ -389,7 +389,7 @@ mod tests {
fn example_doc_and_a_cid() -> (Ipld, Cid) {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!({
let doc = make_ipld!({
"nested": {
"even": [
{
@ -453,7 +453,7 @@ mod tests {
#[test]
fn walk_link_with_dot() {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!(cid.clone());
let doc = make_ipld!(cid.clone());
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/./foobar";
let mut p = IpfsPath::try_from(path).unwrap();
@ -468,7 +468,7 @@ mod tests {
#[test]
fn walk_link_without_dot_is_unsupported() {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = ipld!(cid);
let doc = make_ipld!(cid);
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/foobar";
let mut p = IpfsPath::try_from(path).unwrap();

26
src/dag.rs
View File

@ -1,11 +1,10 @@
use crate::error::Error;
use crate::ipld::{decode_ipld, encode_ipld, Ipld};
use crate::path::{IpfsPath, IpfsPathError, SubPath};
use crate::repo::RepoTypes;
use crate::Ipfs;
use bitswap::Block;
use cid::{Cid, Codec, Version};
use libipld::block::{decode_ipld, encode_ipld};
use libipld::ipld::Ipld;
#[derive(Clone, Debug)]
pub struct IpldDag<Types: RepoTypes> {
@ -94,14 +93,13 @@ fn resolve(ipld: Ipld, sub_path: &SubPath) -> Ipld {
#[cfg(test)]
mod tests {
use super::*;
use crate::Node;
use libipld::ipld;
use crate::{make_ipld, Node};
#[async_std::test]
async fn test_resolve_root_cid() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, 2, 3]);
let data = make_ipld!([1, 2, 3]);
let cid = dag.put(data.clone(), Codec::DagCBOR).await.unwrap();
let res = dag.get(IpfsPath::from(cid)).await.unwrap();
assert_eq!(res, data);
@ -111,33 +109,33 @@ mod tests {
async fn test_resolve_array_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, 2, 3]);
let data = make_ipld!([1, 2, 3]);
let cid = dag.put(data.clone(), Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid).sub_path("1").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(2));
assert_eq!(res, make_ipld!(2));
}
#[async_std::test]
async fn test_resolve_nested_array_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!([1, [2], 3,]);
let data = make_ipld!([1, [2], 3,]);
let cid = dag.put(data, Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid).sub_path("1/0").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(2));
assert_eq!(res, make_ipld!(2));
}
#[async_std::test]
async fn test_resolve_object_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data = ipld!({
let data = make_ipld!({
"key": false,
});
let cid = dag.put(data, Codec::DagCBOR).await.unwrap();
@ -145,21 +143,21 @@ mod tests {
.get(IpfsPath::from(cid).sub_path("key").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(false));
assert_eq!(res, make_ipld!(false));
}
#[async_std::test]
async fn test_resolve_cid_elem() {
let Node { ipfs, bg_task: _bt } = Node::new("test_node").await;
let dag = IpldDag::new(ipfs);
let data1 = ipld!([1]);
let data1 = make_ipld!([1]);
let cid1 = dag.put(data1, Codec::DagCBOR).await.unwrap();
let data2 = ipld!([cid1]);
let data2 = make_ipld!([cid1]);
let cid2 = dag.put(data2, Codec::DagCBOR).await.unwrap();
let res = dag
.get(IpfsPath::from(cid2).sub_path("0/0").unwrap())
.await
.unwrap();
assert_eq!(res, ipld!(1));
assert_eq!(res, make_ipld!(1));
}
}

857
src/ipld/dag_cbor.rs Normal file
View File

@ -0,0 +1,857 @@
//! DAG-CBOR codec.
use crate::ipld::{BlockError, Ipld, IpldError};
use byteorder::{BigEndian, ByteOrder};
use cid::Cid;
use std::{
collections::BTreeMap,
convert::TryFrom,
io::{Read, Write},
};
use thiserror::Error;
/// CBOR codec.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct DagCborCodec;
impl DagCborCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, CborError> {
let mut bytes = Vec::new();
ipld.write_cbor(&mut bytes)?;
Ok(bytes.into_boxed_slice())
}
pub fn decode(mut data: &[u8]) -> Result<Ipld, CborError> {
Ipld::read_cbor(&mut data)
}
}
/// CBOR error.
#[derive(Debug, Error)]
pub enum CborError {
/// Number larger than u64.
#[error("Number larger than u64.")]
NumberOutOfRange,
/// Length larger than usize or too small, for example zero length cid field.
#[error("Length out of range.")]
LengthOutOfRange,
/// Unexpected cbor code.
#[error("Unexpected cbor code.")]
UnexpectedCode,
/// Unknown cbor tag.
#[error("Unkown cbor tag.")]
UnknownTag,
/// Unexpected key.
#[error("Wrong key.")]
UnexpectedKey,
/// Unexpected eof.
#[error("Unexpected end of file.")]
UnexpectedEof,
/// Io error.
#[error("{0}")]
Io(#[from] std::io::Error),
/// Utf8 error.
#[error("{0}")]
Utf8(#[from] std::str::Utf8Error),
/// The byte before Cid was not multibase identity prefix.
#[error("Invalid Cid prefix: {0}")]
InvalidCidPrefix(u8),
/// Cid error.
#[error("{0}")]
Cid(#[from] cid::Error),
/// Ipld error.
#[error("{0}")]
Ipld(#[from] IpldError),
}
impl From<CborError> for BlockError {
fn from(err: CborError) -> Self {
Self::CodecError(err.into())
}
}
/// CBOR result.
pub type CborResult<T> = Result<T, CborError>;
#[inline]
pub fn write_null<W: Write>(w: &mut W) -> CborResult<()> {
w.write_all(&[0xf6])?;
Ok(())
}
#[inline]
pub fn write_u8<W: Write>(w: &mut W, major: u8, value: u8) -> CborResult<()> {
if value <= 0x17 {
let buf = [major << 5 | value];
w.write_all(&buf)?;
} else {
let buf = [major << 5 | 24, value];
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u16<W: Write>(w: &mut W, major: u8, value: u16) -> CborResult<()> {
if value <= u16::from(u8::max_value()) {
write_u8(w, major, value as u8)?;
} else {
let mut buf = [major << 5 | 25, 0, 0];
BigEndian::write_u16(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u32<W: Write>(w: &mut W, major: u8, value: u32) -> CborResult<()> {
if value <= u32::from(u16::max_value()) {
write_u16(w, major, value as u16)?;
} else {
let mut buf = [major << 5 | 26, 0, 0, 0, 0];
BigEndian::write_u32(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_u64<W: Write>(w: &mut W, major: u8, value: u64) -> CborResult<()> {
if value <= u64::from(u32::max_value()) {
write_u32(w, major, value as u32)?;
} else {
let mut buf = [major << 5 | 27, 0, 0, 0, 0, 0, 0, 0, 0];
BigEndian::write_u64(&mut buf[1..], value);
w.write_all(&buf)?;
}
Ok(())
}
#[inline]
pub fn write_tag<W: Write>(w: &mut W, tag: u64) -> CborResult<()> {
write_u64(w, 6, tag)
}
pub trait WriteCbor {
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()>;
}
impl WriteCbor for bool {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
let buf = if *self { [0xf5] } else { [0xf4] };
w.write_all(&buf)?;
Ok(())
}
}
impl WriteCbor for u8 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u8(w, 0, *self)
}
}
impl WriteCbor for u16 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u16(w, 0, *self)
}
}
impl WriteCbor for u32 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u32(w, 0, *self)
}
}
impl WriteCbor for u64 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 0, *self)
}
}
impl WriteCbor for i8 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u8(w, 1, -(*self + 1) as u8)
}
}
impl WriteCbor for i16 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u16(w, 1, -(*self + 1) as u16)
}
}
impl WriteCbor for i32 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u32(w, 1, -(*self + 1) as u32)
}
}
impl WriteCbor for i64 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 1, -(*self + 1) as u64)
}
}
impl WriteCbor for f32 {
#[inline]
#[allow(clippy::float_cmp)]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if self.is_infinite() {
if self.is_sign_positive() {
w.write_all(&[0xf9, 0x7c, 0x00])?;
} else {
w.write_all(&[0xf9, 0xfc, 0x00])?;
}
} else if self.is_nan() {
w.write_all(&[0xf9, 0x7e, 0x00])?;
} else {
let mut buf = [0xfa, 0, 0, 0, 0];
BigEndian::write_f32(&mut buf[1..], *self);
w.write_all(&buf)?;
}
Ok(())
}
}
impl WriteCbor for f64 {
#[inline]
#[allow(clippy::float_cmp)]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if !self.is_finite() || f64::from(*self as f32) == *self {
let value = *self as f32;
value.write_cbor(w)?;
} else {
let mut buf = [0xfb, 0, 0, 0, 0, 0, 0, 0, 0];
BigEndian::write_f64(&mut buf[1..], *self);
w.write_all(&buf)?;
}
Ok(())
}
}
impl WriteCbor for [u8] {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 2, self.len() as u64)?;
w.write_all(self)?;
Ok(())
}
}
impl WriteCbor for str {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 3, self.len() as u64)?;
w.write_all(self.as_bytes())?;
Ok(())
}
}
impl WriteCbor for String {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
self.as_str().write_cbor(w)
}
}
impl WriteCbor for i128 {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if *self < 0 {
if -(*self + 1) > u64::max_value() as i128 {
return Err(CborError::NumberOutOfRange);
}
write_u64(w, 1, -(*self + 1) as u64)?;
} else {
if *self > u64::max_value() as i128 {
return Err(CborError::NumberOutOfRange);
}
write_u64(w, 0, *self as u64)?;
}
Ok(())
}
}
impl WriteCbor for Cid {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_tag(w, 42)?;
// insert zero byte per https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links
let bytes = self.to_bytes();
write_u64(w, 2, (bytes.len() + 1) as u64)?;
w.write_all(&[0])?;
w.write_all(&bytes)?;
Ok(())
}
}
impl<T: WriteCbor> WriteCbor for Option<T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
if let Some(value) = self {
value.write_cbor(w)?;
} else {
write_null(w)?;
}
Ok(())
}
}
impl<T: WriteCbor> WriteCbor for Vec<T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 4, self.len() as u64)?;
for value in self {
value.write_cbor(w)?;
}
Ok(())
}
}
impl<T: WriteCbor + 'static> WriteCbor for BTreeMap<String, T> {
#[inline]
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
write_u64(w, 5, self.len() as u64)?;
for (k, v) in self {
k.write_cbor(w)?;
v.write_cbor(w)?;
}
Ok(())
}
}
impl WriteCbor for Ipld {
fn write_cbor<W: Write>(&self, w: &mut W) -> CborResult<()> {
match self {
Ipld::Null => write_null(w),
Ipld::Bool(b) => b.write_cbor(w),
Ipld::Integer(i) => i.write_cbor(w),
Ipld::Float(f) => f.write_cbor(w),
Ipld::Bytes(b) => b.as_slice().write_cbor(w),
Ipld::String(s) => s.as_str().write_cbor(w),
Ipld::List(l) => l.write_cbor(w),
Ipld::Map(m) => m.write_cbor(w),
Ipld::Link(c) => c.write_cbor(w),
}
}
}
#[inline]
pub fn read_u8<R: Read>(r: &mut R) -> CborResult<u8> {
let mut buf = [0; 1];
r.read_exact(&mut buf)?;
Ok(buf[0])
}
#[inline]
pub fn read_u16<R: Read>(r: &mut R) -> CborResult<u16> {
let mut buf = [0; 2];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u16(&buf))
}
#[inline]
pub fn read_u32<R: Read>(r: &mut R) -> CborResult<u32> {
let mut buf = [0; 4];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u32(&buf))
}
#[inline]
pub fn read_u64<R: Read>(r: &mut R) -> CborResult<u64> {
let mut buf = [0; 8];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_u64(&buf))
}
#[inline]
pub fn read_f32<R: Read>(r: &mut R) -> CborResult<f32> {
let mut buf = [0; 4];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_f32(&buf))
}
#[inline]
pub fn read_f64<R: Read>(r: &mut R) -> CborResult<f64> {
let mut buf = [0; 8];
r.read_exact(&mut buf)?;
Ok(BigEndian::read_f64(&buf))
}
#[inline]
pub fn read_bytes<R: Read>(r: &mut R, len: usize) -> CborResult<Vec<u8>> {
let mut buf = vec![0; len];
r.read_exact(&mut buf)?;
Ok(buf)
}
#[inline]
pub fn read_str<R: Read>(r: &mut R, len: usize) -> CborResult<String> {
let bytes = read_bytes(r, len)?;
let string = std::str::from_utf8(&bytes)?;
Ok(string.to_string())
}
#[inline]
pub fn read_list<R: Read, T: ReadCbor>(r: &mut R, len: usize) -> CborResult<Vec<T>> {
let mut list: Vec<T> = Vec::with_capacity(len);
for _ in 0..len {
list.push(T::read_cbor(r)?);
}
Ok(list)
}
#[inline]
pub fn read_map<R: Read, T: ReadCbor>(r: &mut R, len: usize) -> CborResult<BTreeMap<String, T>> {
let mut map: BTreeMap<String, T> = BTreeMap::new();
for _ in 0..len {
let key = String::read_cbor(r)?;
let value = T::read_cbor(r)?;
map.insert(key, value);
}
Ok(map)
}
#[inline]
pub fn read_link<R: Read>(r: &mut R) -> CborResult<Cid> {
let tag = read_u8(r)?;
if tag != 42 {
return Err(CborError::UnknownTag);
}
let ty = read_u8(r)?;
if ty != 0x58 {
return Err(CborError::UnknownTag);
}
let len = read_u8(r)?;
if len == 0 {
return Err(CborError::LengthOutOfRange);
}
let bytes = read_bytes(r, len as usize)?;
if bytes[0] != 0 {
return Err(CborError::InvalidCidPrefix(bytes[0]));
}
// skip the first byte per
// https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#links
Ok(Cid::try_from(&bytes[1..])?)
}
pub trait ReadCbor: Sized {
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>>;
#[inline]
fn read_cbor<R: Read>(r: &mut R) -> CborResult<Self> {
let major = read_u8(r)?;
if let Some(res) = Self::try_read_cbor(r, major)? {
Ok(res)
} else {
Err(CborError::UnexpectedCode)
}
}
}
impl ReadCbor for bool {
#[inline]
fn try_read_cbor<R: Read>(_: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xf4 => Ok(Some(false)),
0xf5 => Ok(Some(true)),
_ => Ok(None),
}
}
}
impl ReadCbor for u8 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major)),
0x18 => Ok(Some(read_u8(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u16 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u16)),
0x18 => Ok(Some(read_u8(r)? as u16)),
0x19 => Ok(Some(read_u16(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u32)),
0x18 => Ok(Some(read_u8(r)? as u32)),
0x19 => Ok(Some(read_u16(r)? as u32)),
0x1a => Ok(Some(read_u32(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for u64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x00..=0x17 => Ok(Some(major as u64)),
0x18 => Ok(Some(read_u8(r)? as u64)),
0x19 => Ok(Some(read_u16(r)? as u64)),
0x1a => Ok(Some(read_u32(r)? as u64)),
0x1b => Ok(Some(read_u64(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for i8 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i8)),
0x38 => Ok(Some(-1 - read_u8(r)? as i8)),
_ => Ok(None),
}
}
}
impl ReadCbor for i16 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i16)),
0x38 => Ok(Some(-1 - read_u8(r)? as i16)),
0x39 => Ok(Some(-1 - read_u16(r)? as i16)),
_ => Ok(None),
}
}
}
impl ReadCbor for i32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i32)),
0x38 => Ok(Some(-1 - read_u8(r)? as i32)),
0x39 => Ok(Some(-1 - read_u16(r)? as i32)),
0x3a => Ok(Some(-1 - read_u32(r)? as i32)),
_ => Ok(None),
}
}
}
impl ReadCbor for i64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0x20..=0x37 => Ok(Some(-1 - (major - 0x20) as i64)),
0x38 => Ok(Some(-1 - read_u8(r)? as i64)),
0x39 => Ok(Some(-1 - read_u16(r)? as i64)),
0x3a => Ok(Some(-1 - read_u32(r)? as i64)),
0x3b => Ok(Some(-1 - read_u64(r)? as i64)),
_ => Ok(None),
}
}
}
impl ReadCbor for f32 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xfa => Ok(Some(read_f32(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for f64 {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xfa => Ok(Some(read_f32(r)? as f64)),
0xfb => Ok(Some(read_f64(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for String {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x60..=0x77 => major as usize - 0x60,
0x78 => read_u8(r)? as usize,
0x79 => read_u16(r)? as usize,
0x7a => read_u32(r)? as usize,
0x7b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_str(r, len)?))
}
}
impl ReadCbor for Cid {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xd8 => Ok(Some(read_link(r)?)),
_ => Ok(None),
}
}
}
impl ReadCbor for Box<[u8]> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x40..=0x57 => major as usize - 0x40,
0x58 => read_u8(r)? as usize,
0x59 => read_u16(r)? as usize,
0x5a => read_u32(r)? as usize,
0x5b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_bytes(r, len)?.into_boxed_slice()))
}
}
impl<T: ReadCbor> ReadCbor for Option<T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
match major {
0xf6 => Ok(Some(None)),
0xf7 => Ok(Some(None)),
_ => {
if let Some(res) = T::try_read_cbor(r, major)? {
Ok(Some(Some(res)))
} else {
Ok(None)
}
}
}
}
}
impl<T: ReadCbor> ReadCbor for Vec<T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0x80..=0x97 => major as usize - 0x80,
0x98 => read_u8(r)? as usize,
0x99 => read_u16(r)? as usize,
0x9a => read_u32(r)? as usize,
0x9b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_list(r, len)?))
}
}
impl<T: ReadCbor> ReadCbor for BTreeMap<String, T> {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let len = match major {
0xa0..=0xb7 => major as usize - 0xa0,
0xb8 => read_u8(r)? as usize,
0xb9 => read_u16(r)? as usize,
0xba => read_u32(r)? as usize,
0xbb => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
len as usize
}
_ => return Ok(None),
};
Ok(Some(read_map(r, len)?))
}
}
impl ReadCbor for Ipld {
#[inline]
fn try_read_cbor<R: Read>(r: &mut R, major: u8) -> CborResult<Option<Self>> {
let ipld = match major {
// Major type 0: an unsigned integer
0x00..=0x17 => Ipld::Integer(major as i128),
0x18 => Ipld::Integer(read_u8(r)? as i128),
0x19 => Ipld::Integer(read_u16(r)? as i128),
0x1a => Ipld::Integer(read_u32(r)? as i128),
0x1b => Ipld::Integer(read_u64(r)? as i128),
// Major type 1: a negative integer
0x20..=0x37 => Ipld::Integer(-1 - (major - 0x20) as i128),
0x38 => Ipld::Integer(-1 - read_u8(r)? as i128),
0x39 => Ipld::Integer(-1 - read_u16(r)? as i128),
0x3a => Ipld::Integer(-1 - read_u32(r)? as i128),
0x3b => Ipld::Integer(-1 - read_u64(r)? as i128),
// Major type 2: a byte string
0x40..=0x57 => {
let len = major - 0x40;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x58 => {
let len = read_u8(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x59 => {
let len = read_u16(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x5a => {
let len = read_u32(r)?;
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
0x5b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let bytes = read_bytes(r, len as usize)?;
Ipld::Bytes(bytes)
}
// Major type 3: a text string
0x60..=0x77 => {
let len = major - 0x60;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x78 => {
let len = read_u8(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x79 => {
let len = read_u16(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x7a => {
let len = read_u32(r)?;
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
0x7b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let string = read_str(r, len as usize)?;
Ipld::String(string)
}
// Major type 4: an array of data items
0x80..=0x97 => {
let len = major - 0x80;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x98 => {
let len = read_u8(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x99 => {
let len = read_u16(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x9a => {
let len = read_u32(r)?;
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
0x9b => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let list = read_list(r, len as usize)?;
Ipld::List(list)
}
// Major type 5: a map of pairs of data items
0xa0..=0xb7 => {
let len = major - 0xa0;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xb8 => {
let len = read_u8(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xb9 => {
let len = read_u16(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xba => {
let len = read_u32(r)?;
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
0xbb => {
let len = read_u64(r)?;
if len > usize::max_value() as u64 {
return Err(CborError::LengthOutOfRange);
}
let map = read_map(r, len as usize)?;
Ipld::Map(map)
}
// Major type 6: optional semantic tagging of other major types
0xd8 => Ipld::Link(read_link(r)?),
// Major type 7: floating-point numbers and other simple data types that need no content
0xf4 => Ipld::Bool(false),
0xf5 => Ipld::Bool(true),
0xf6 => Ipld::Null,
0xf7 => Ipld::Null,
0xfa => Ipld::Float(read_f32(r)? as f64),
0xfb => Ipld::Float(read_f64(r)?),
_ => return Ok(None),
};
Ok(Some(ipld))
}
}

247
src/ipld/dag_json.rs Normal file
View File

@ -0,0 +1,247 @@
//! DAG-JSON codec.
use crate::ipld::{BlockError, Ipld};
use cid::Cid;
use core::convert::TryFrom;
use serde::de::Error as SerdeError;
use serde::{de, ser, Deserialize, Serialize};
use serde_json::ser::Serializer;
use serde_json::Error;
use std::collections::BTreeMap;
use std::fmt;
use std::iter::FromIterator;
/// Json codec.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct DagJsonCodec;
impl DagJsonCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, BlockError> {
json_encode(ipld).map_err(|e| BlockError::CodecError(e.into()))
}
pub fn decode(data: &[u8]) -> Result<Ipld, BlockError> {
json_decode(data).map_err(|e| BlockError::CodecError(e.into()))
}
}
const LINK_KEY: &str = "/";
pub fn json_encode(ipld: &Ipld) -> Result<Box<[u8]>, Error> {
let mut writer = Vec::with_capacity(128);
let mut ser = Serializer::new(&mut writer);
serialize(&ipld, &mut ser)?;
Ok(writer.into_boxed_slice())
}
pub fn json_decode(data: &[u8]) -> Result<Ipld, Error> {
let mut de = serde_json::Deserializer::from_slice(&data);
Ok(deserialize(&mut de)?)
}
fn serialize<S>(ipld: &Ipld, ser: S) -> Result<S::Ok, S::Error>
where
S: ser::Serializer,
{
match &ipld {
Ipld::Null => ser.serialize_none(),
Ipld::Bool(bool) => ser.serialize_bool(*bool),
Ipld::Integer(i128) => ser.serialize_i128(*i128),
Ipld::Float(f64) => ser.serialize_f64(*f64),
Ipld::String(string) => ser.serialize_str(&string),
Ipld::Bytes(bytes) => ser.serialize_bytes(&bytes),
Ipld::List(list) => {
let wrapped = list.iter().map(|ipld| Wrapper(ipld));
ser.collect_seq(wrapped)
}
Ipld::Map(map) => {
let wrapped = map.iter().map(|(key, ipld)| (key, Wrapper(ipld)));
ser.collect_map(wrapped)
}
Ipld::Link(link) => {
let value = base64::encode(&link.to_bytes());
let mut map = BTreeMap::new();
map.insert("/", value);
ser.collect_map(map)
}
}
}
fn deserialize<'de, D>(deserializer: D) -> Result<Ipld, D::Error>
where
D: de::Deserializer<'de>,
{
deserializer.deserialize_any(JSONVisitor)
}
// Needed for `collect_seq` and `collect_map` in Seserializer
struct Wrapper<'a>(&'a Ipld);
impl<'a> Serialize for Wrapper<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: ser::Serializer,
{
serialize(&self.0, serializer)
}
}
// serde deserializer visitor that is used by Deseraliazer to decode
// json into IPLD.
struct JSONVisitor;
impl<'de> de::Visitor<'de> for JSONVisitor {
type Value = Ipld;
fn expecting(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.write_str("any valid JSON value")
}
#[inline]
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_string(String::from(value))
}
#[inline]
fn visit_string<E>(self, value: String) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::String(value))
}
#[inline]
fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_byte_buf(v.to_owned())
}
#[inline]
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Bytes(v))
}
#[inline]
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v.into()))
}
#[inline]
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v.into()))
}
#[inline]
fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Integer(v))
}
#[inline]
fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Bool(v))
}
#[inline]
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
self.visit_unit()
}
#[inline]
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Null)
}
#[inline]
fn visit_seq<V>(self, mut visitor: V) -> Result<Self::Value, V::Error>
where
V: de::SeqAccess<'de>,
{
let mut vec: Vec<WrapperOwned> = Vec::new();
while let Some(elem) = visitor.next_element()? {
vec.push(elem);
}
let unwrapped = vec.into_iter().map(|WrapperOwned(ipld)| ipld).collect();
Ok(Ipld::List(unwrapped))
}
#[inline]
fn visit_map<V>(self, mut visitor: V) -> Result<Self::Value, V::Error>
where
V: de::MapAccess<'de>,
{
let mut values: Vec<(String, WrapperOwned)> = Vec::new();
while let Some((key, value)) = visitor.next_entry()? {
values.push((key, value));
}
// JSON Object represents IPLD Link if it is `{ "/": "...." }` therefor
// we valiadet if that is the case here.
if let Some((key, WrapperOwned(Ipld::String(value)))) = values.first() {
if key == LINK_KEY && values.len() == 1 {
let link = base64::decode(value).map_err(SerdeError::custom)?;
let cid = Cid::try_from(link).map_err(SerdeError::custom)?;
return Ok(Ipld::Link(cid));
}
}
let unwrapped = values
.into_iter()
.map(|(key, WrapperOwned(value))| (key, value));
Ok(Ipld::Map(BTreeMap::from_iter(unwrapped)))
}
#[inline]
fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(Ipld::Float(v))
}
}
// Needed for `visit_seq` and `visit_map` in Deserializer
/// We cannot directly implement `serde::Deserializer` for `Ipld` as it is a remote type.
/// Instead wrap it into a newtype struct and implement `serde::Deserialize` for that one.
/// All the deserializer does is calling the `deserialize()` function we defined which returns
/// an unwrapped `Ipld` instance. Wrap that `Ipld` instance in `Wrapper` and return it.
/// Users of this wrapper will then unwrap it again so that they can return the expected `Ipld`
/// instance.
struct WrapperOwned(Ipld);
impl<'de> Deserialize<'de> for WrapperOwned {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: de::Deserializer<'de>,
{
let deserialized = deserialize(deserializer);
// Better version of Ok(Wrapper(deserialized.unwrap()))
deserialized.map(Self)
}
}

22
src/ipld/dag_pb.proto Normal file
View File

@ -0,0 +1,22 @@
// https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-pb.md
syntax = "proto3";
package dag_pb;
// An IPFS MerkleDAG Link
message PBLink {
// binary CID (with no multibase prefix) of the target object
bytes Hash = 1;
// UTF-8 string name
string Name = 2;
// cumulative size of target object
uint64 Tsize = 3;
}
// An IPFS MerkleDAG Node
message PBNode {
// refs to other objects
repeated PBLink Links = 2;
// opaque user data
bytes Data = 1;
}

163
src/ipld/dag_pb.rs Normal file
View File

@ -0,0 +1,163 @@
//! DAG-Protobuf codec.
use crate::ipld::{BlockError, Ipld, IpldError};
use cid::Cid;
use std::{
collections::BTreeMap,
convert::{TryFrom, TryInto},
};
use thiserror::Error;
/// Protobuf codec.
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct DagPbCodec;
impl DagPbCodec {
pub fn encode(ipld: &Ipld) -> Result<Box<[u8]>, ProtobufError> {
let pb_node: PbNode = ipld.try_into()?;
Ok(pb_node.into_bytes())
}
pub fn decode(data: &[u8]) -> Result<Ipld, ProtobufError> {
Ok(PbNode::from_bytes(data)?.into())
}
}
/// Protobuf error.
#[derive(Debug, Error)]
pub enum ProtobufError {
#[error("{0}")]
Prost(#[from] prost::DecodeError),
#[error("{0}")]
Cid(#[from] cid::Error),
#[error("{0}")]
Ipld(#[from] IpldError),
}
impl From<ProtobufError> for BlockError {
fn from(error: ProtobufError) -> Self {
Self::CodecError(error.into())
}
}
mod pb {
include!(concat!(env!("OUT_DIR"), "/dag_pb.rs"));
}
pub struct PbLink {
pub cid: Cid,
pub name: String,
pub size: u64,
}
pub struct PbNode {
pub links: Vec<PbLink>,
pub data: Vec<u8>,
}
use prost::Message;
impl PbNode {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, ProtobufError> {
let proto: pb::PbNode = pb::PbNode::decode(bytes)?;
let data = proto.data;
let mut links = Vec::new();
for link in proto.links {
let cid = Cid::try_from(link.hash)?;
let name = link.name;
let size = link.tsize;
links.push(PbLink { cid, name, size });
}
Ok(PbNode { links, data })
}
pub fn into_bytes(self) -> Box<[u8]> {
let links = self
.links
.into_iter()
.map(|link| pb::PbLink {
hash: link.cid.to_bytes(),
name: link.name,
tsize: link.size,
})
.collect::<Vec<_>>();
let proto = pb::PbNode {
data: self.data,
links,
};
let mut res = Vec::with_capacity(proto.encoded_len());
proto
.encode(&mut res)
.expect("there is no situation in which the protobuf message can be invalid");
res.into_boxed_slice()
}
}
impl Into<Ipld> for PbNode {
fn into(self) -> Ipld {
let mut map = BTreeMap::<String, Ipld>::new();
let links = self
.links
.into_iter()
.map(|link| link.into())
.collect::<Vec<Ipld>>();
map.insert("Links".to_string(), links.into());
map.insert("Data".to_string(), self.data.into());
map.into()
}
}
impl Into<Ipld> for PbLink {
fn into(self) -> Ipld {
let mut map = BTreeMap::<String, Ipld>::new();
map.insert("Hash".to_string(), self.cid.into());
map.insert("Name".to_string(), self.name.into());
map.insert("Tsize".to_string(), self.size.into());
map.into()
}
}
impl TryFrom<&Ipld> for PbNode {
type Error = IpldError;
fn try_from(ipld: &Ipld) -> Result<PbNode, Self::Error> {
let links = if let Ipld::List(links) = ipld.get("Links").ok_or(IpldError::KeyNotFound)? {
links
.iter()
.map(|link| link.try_into())
.collect::<Result<_, _>>()?
} else {
return Err(IpldError::NotList);
};
let data = if let Ipld::Bytes(data) = ipld.get("Data").ok_or(IpldError::KeyNotFound)? {
data.clone()
} else {
return Err(IpldError::NotBytes);
};
Ok(PbNode { links, data })
}
}
impl TryFrom<&Ipld> for PbLink {
type Error = IpldError;
fn try_from(ipld: &Ipld) -> Result<PbLink, Self::Error> {
let cid = if let Ipld::Link(cid) = ipld.get("Hash").ok_or(IpldError::KeyNotFound)? {
cid.clone()
} else {
return Err(IpldError::NotLink);
};
let name = if let Ipld::String(name) = ipld.get("Name").ok_or(IpldError::KeyNotFound)? {
name.clone()
} else {
return Err(IpldError::NotString);
};
let size = if let Ipld::Integer(size) = ipld.get("Tsize").ok_or(IpldError::KeyNotFound)? {
*size as u64
} else {
return Err(IpldError::NotInteger);
};
Ok(PbLink { cid, name, size })
}
}

236
src/ipld/ipld_macro.rs Normal file
View File

@ -0,0 +1,236 @@
#[macro_export(local_inner_macros)]
macro_rules! make_ipld {
// Hide distracting implementation details from the generated rustdoc.
($($ipld:tt)+) => {
ipld_internal!($($ipld)+)
};
}
#[macro_export(local_inner_macros)]
#[doc(hidden)]
macro_rules! ipld_internal {
//////////////////////////////////////////////////////////////////////////
// TT muncher for parsing the inside of an array [...]. Produces a vec![...]
// of the elements.
//
// Must be invoked as: ipld_internal!(@array [] $($tt)*)
//////////////////////////////////////////////////////////////////////////
// Done with trailing comma.
(@array [$($elems:expr,)*]) => {
ipld_internal_vec![$($elems,)*]
};
// Done without trailing comma.
(@array [$($elems:expr),*]) => {
ipld_internal_vec![$($elems),*]
};
// Next element is `null`.
(@array [$($elems:expr,)*] null $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(null)] $($rest)*)
};
// Next element is `true`.
(@array [$($elems:expr,)*] true $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(true)] $($rest)*)
};
// Next element is `false`.
(@array [$($elems:expr,)*] false $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!(false)] $($rest)*)
};
// Next element is an array.
(@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!([$($array)*])] $($rest)*)
};
// Next element is a map.
(@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!({$($map)*})] $($rest)*)
};
// Next element is an expression followed by comma.
(@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)* ipld_internal!($next),] $($rest)*)
};
// Last element is an expression with no trailing comma.
(@array [$($elems:expr,)*] $last:expr) => {
ipld_internal!(@array [$($elems,)* ipld_internal!($last)])
};
// Comma after the most recent element.
(@array [$($elems:expr),*] , $($rest:tt)*) => {
ipld_internal!(@array [$($elems,)*] $($rest)*)
};
// Unexpected token after most recent element.
(@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => {
ipld_unexpected!($unexpected)
};
//////////////////////////////////////////////////////////////////////////
// TT muncher for parsing the inside of an object {...}. Each entry is
// inserted into the given map variable.
//
// Must be invoked as: json_internal!(@object $map () ($($tt)*) ($($tt)*))
//
// We require two copies of the input tokens so that we can match on one
// copy and trigger errors on the other copy.
//////////////////////////////////////////////////////////////////////////
// Done.
(@object $object:ident () () ()) => {};
// Insert the current entry followed by trailing comma.
(@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
let _ = $object.insert(($($key)+).into(), $value);
ipld_internal!(@object $object () ($($rest)*) ($($rest)*));
};
// Current entry followed by unexpected token.
(@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => {
ipld_unexpected!($unexpected);
};
// Insert the last entry without trailing comma.
(@object $object:ident [$($key:tt)+] ($value:expr)) => {
let _ = $object.insert(($($key)+).into(), $value);
};
// Next value is `null`.
(@object $object:ident ($($key:tt)+) (: null $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(null)) $($rest)*);
};
// Next value is `true`.
(@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(true)) $($rest)*);
};
// Next value is `false`.
(@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!(false)) $($rest)*);
};
// Next value is an array.
(@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!([$($array)*])) $($rest)*);
};
// Next value is a map.
(@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!({$($map)*})) $($rest)*);
};
// Next value is an expression followed by comma.
(@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!($value)) , $($rest)*);
};
// Last value is an expression with no trailing comma.
(@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => {
ipld_internal!(@object $object [$($key)+] (ipld_internal!($value)));
};
// Missing value for last entry. Trigger a reasonable error message.
(@object $object:ident ($($key:tt)+) (:) $copy:tt) => {
// "unexpected end of macro invocation"
ipld_internal!();
};
// Missing colon and value for last entry. Trigger a reasonable error
// message.
(@object $object:ident ($($key:tt)+) () $copy:tt) => {
// "unexpected end of macro invocation"
ipld_internal!();
};
// Misplaced colon. Trigger a reasonable error message.
(@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => {
// Takes no arguments so "no rules expected the token `:`".
ipld_unexpected!($colon);
};
// Found a comma inside a key. Trigger a reasonable error message.
(@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => {
// Takes no arguments so "no rules expected the token `,`".
ipld_unexpected!($comma);
};
// Key is fully parenthesized. This avoids clippy double_parens false
// positives because the parenthesization may be necessary here.
(@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*));
};
// Munch a token into the current key.
(@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => {
ipld_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*));
};
//////////////////////////////////////////////////////////////////////////
// The main implementation.
//
// Must be invoked as: json_internal!($($json)+)
//////////////////////////////////////////////////////////////////////////
(null) => {
$crate::Ipld::Null
};
(true) => {
$crate::Ipld::Bool(true)
};
(false) => {
$crate::Ipld::Bool(false)
};
([]) => {
$crate::Ipld::List(ipld_internal_vec![])
};
([ $($tt:tt)+ ]) => {
$crate::Ipld::List(ipld_internal!(@array [] $($tt)+))
};
({}) => {
$crate::Ipld::Map(std::collections::BTreeMap::new())
};
({ $($tt:tt)+ }) => {
$crate::Ipld::Map({
let mut object = std::collections::BTreeMap::new();
ipld_internal!(@object object () ($($tt)+) ($($tt)+));
object
})
};
// Any Serialize type: numbers, strings, struct literals, variables etc.
// Must be below every other rule.
($other:expr) => {
{
$crate::Ipld::from($other)
}
};
}
// The json_internal macro above cannot invoke vec directly because it uses
// local_inner_macros. A vec invocation there would resolve to $crate::vec.
// Instead invoke vec here outside of local_inner_macros.
#[macro_export]
#[doc(hidden)]
macro_rules! ipld_internal_vec {
($($content:tt)*) => {
vec![$($content)*]
};
}
#[macro_export]
#[doc(hidden)]
macro_rules! ipld_unexpected {
() => {};
}

293
src/ipld/mod.rs Normal file
View File

@ -0,0 +1,293 @@
// This code was adapted from https://github.com/ipfs-rust/rust-ipld, and most of
// its code is the same as at revision b2286c53c13f3eeec2a3766387f2926838e8e4c9;
// it used to be a direct dependency, but recent updates to cid and multihash crates
// made it incompatible with them.
pub mod dag_cbor;
pub mod dag_json;
pub mod dag_pb;
#[macro_use]
pub mod ipld_macro;
use cid::{Cid, Codec};
use dag_cbor::DagCborCodec;
use dag_json::DagJsonCodec;
use dag_pb::DagPbCodec;
use multihash::Multihash;
use std::collections::BTreeMap;
use thiserror::Error;
/// Ipld
#[derive(Clone, Debug, PartialEq)]
pub enum Ipld {
/// Represents the absence of a value or the value undefined.
Null,
/// Represents a boolean value.
Bool(bool),
/// Represents an integer.
Integer(i128),
/// Represents a floating point value.
Float(f64),
/// Represents an UTF-8 string.
String(String),
/// Represents a sequence of bytes.
Bytes(Vec<u8>),
/// Represents a list.
List(Vec<Ipld>),
/// Represents a map.
Map(BTreeMap<String, Ipld>),
/// Represents a link to an Ipld node
Link(Cid),
}
macro_rules! derive_to_ipld_prim {
($enum:ident, $ty:ty, $fn:ident) => {
impl From<$ty> for Ipld {
fn from(t: $ty) -> Self {
Ipld::$enum(t.$fn() as _)
}
}
};
}
macro_rules! derive_to_ipld {
($enum:ident, $ty:ty, $fn:ident) => {
impl From<$ty> for Ipld {
fn from(t: $ty) -> Self {
Ipld::$enum(t.$fn())
}
}
};
}
derive_to_ipld!(Bool, bool, clone);
derive_to_ipld_prim!(Integer, i8, clone);
derive_to_ipld_prim!(Integer, i16, clone);
derive_to_ipld_prim!(Integer, i32, clone);
derive_to_ipld_prim!(Integer, i64, clone);
derive_to_ipld_prim!(Integer, i128, clone);
derive_to_ipld_prim!(Integer, isize, clone);
derive_to_ipld_prim!(Integer, u8, clone);
derive_to_ipld_prim!(Integer, u16, clone);
derive_to_ipld_prim!(Integer, u32, clone);
derive_to_ipld_prim!(Integer, u64, clone);
derive_to_ipld_prim!(Integer, usize, clone);
derive_to_ipld_prim!(Float, f32, clone);
derive_to_ipld_prim!(Float, f64, clone);
derive_to_ipld!(String, String, into);
derive_to_ipld!(String, &str, to_string);
derive_to_ipld!(Bytes, Vec<u8>, into);
derive_to_ipld!(Bytes, &[u8], to_vec);
derive_to_ipld!(List, Vec<Ipld>, into);
derive_to_ipld!(Map, BTreeMap<String, Ipld>, to_owned);
derive_to_ipld!(Link, Cid, clone);
derive_to_ipld!(Link, &Cid, to_owned);
/// An index into ipld
pub enum IpldIndex<'a> {
/// An index into an ipld list.
List(usize),
/// An owned index into an ipld map.
Map(String),
/// An index into an ipld map.
MapRef(&'a str),
}
impl<'a> From<usize> for IpldIndex<'a> {
fn from(index: usize) -> Self {
Self::List(index)
}
}
impl<'a> From<String> for IpldIndex<'a> {
fn from(key: String) -> Self {
Self::Map(key)
}
}
impl<'a> From<&'a str> for IpldIndex<'a> {
fn from(key: &'a str) -> Self {
Self::MapRef(key)
}
}
impl Ipld {
/// Indexes into a ipld list or map.
pub fn get<'a, T: Into<IpldIndex<'a>>>(&self, index: T) -> Option<&Ipld> {
match self {
Ipld::List(l) => match index.into() {
IpldIndex::List(i) => l.get(i),
_ => None,
},
Ipld::Map(m) => match index.into() {
IpldIndex::Map(ref key) => m.get(key),
IpldIndex::MapRef(key) => m.get(key),
_ => None,
},
_ => None,
}
}
/// Returns an iterator.
pub fn iter(&self) -> IpldIter<'_> {
IpldIter {
stack: vec![Box::new(vec![self].into_iter())],
}
}
}
/// Ipld iterator.
pub struct IpldIter<'a> {
stack: Vec<Box<dyn Iterator<Item = &'a Ipld> + 'a>>,
}
impl<'a> Iterator for IpldIter<'a> {
type Item = &'a Ipld;
fn next(&mut self) -> Option<Self::Item> {
loop {
if let Some(iter) = self.stack.last_mut() {
if let Some(ipld) = iter.next() {
match ipld {
Ipld::List(list) => {
self.stack.push(Box::new(list.iter()));
}
Ipld::Map(map) => {
self.stack.push(Box::new(map.values()));
}
_ => {}
}
return Some(ipld);
} else {
self.stack.pop();
}
} else {
return None;
}
}
}
}
/// Ipld type error.
#[derive(Debug, Error)]
pub enum IpldError {
/// Expected a boolean.
#[error("Expected a boolean.")]
NotBool,
/// Expected an integer.
#[error("Expected an integer.")]
NotInteger,
/// Expected a float.
#[error("Expected a float.")]
NotFloat,
/// Expected a string.
#[error("Expected a string.")]
NotString,
/// Expected bytes.
#[error("Expected bytes.")]
NotBytes,
/// Expected a list.
#[error("Expected a list.")]
NotList,
/// Expected a map.
#[error("Expected a map.")]
NotMap,
/// Expected a cid.
#[error("Expected a cid.")]
NotLink,
/// Expected a key.
#[error("Expected a key.")]
NotKey,
/// Index not found.
#[error("Index not found.")]
IndexNotFound,
/// Key not found.
#[error("Key not found.")]
KeyNotFound,
}
/// Block error.
#[derive(Debug, Error)]
pub enum BlockError {
/// Block exceeds MAX_BLOCK_SIZE.
#[error("Block size {0} exceeds MAX_BLOCK_SIZE.")]
BlockTooLarge(usize),
/// Hash does not match the CID.
#[error("Hash does not match the CID.")]
InvalidHash(Multihash),
/// The codec is unsupported.
#[error("Unsupported codec {0:?}.")]
UnsupportedCodec(cid::Codec),
/// The multihash is unsupported.
#[error("Unsupported multihash {0:?}.")]
UnsupportedMultihash(multihash::Code),
/// The codec returned an error.
#[error("Codec error: {0}")]
CodecError(Box<dyn std::error::Error + Send + Sync>),
/// Io error.
#[error("{0}")]
Io(std::io::Error),
/// Cid error.
#[error("{0}")]
Cid(cid::Error),
/// Link error.
#[error("Invalid link.")]
InvalidLink,
}
impl From<std::io::Error> for BlockError {
fn from(err: std::io::Error) -> Self {
Self::Io(err)
}
}
impl From<cid::Error> for BlockError {
fn from(err: cid::Error) -> Self {
Self::Cid(err)
}
}
/// The maximum block size is 1MiB.
pub const MAX_BLOCK_SIZE: usize = 1_048_576;
/// Validate a block.
pub fn validate(cid: &Cid, data: &[u8]) -> Result<(), BlockError> {
if data.len() > MAX_BLOCK_SIZE {
return Err(BlockError::BlockTooLarge(data.len()));
}
let hash = cid.hash().algorithm().digest(&data);
if hash.as_ref() != cid.hash() {
return Err(BlockError::InvalidHash(hash));
}
Ok(())
}
/// Encode ipld to bytes.
pub fn encode_ipld(ipld: &Ipld, codec: Codec) -> Result<Box<[u8]>, BlockError> {
let bytes = match codec {
Codec::DagCBOR => DagCborCodec::encode(ipld)?,
Codec::DagProtobuf => DagPbCodec::encode(ipld)?,
Codec::DagJSON => DagJsonCodec::encode(ipld)?,
Codec::Raw => {
if let Ipld::Bytes(bytes) = ipld {
bytes.to_vec().into_boxed_slice()
} else {
return Err(BlockError::CodecError(IpldError::NotBytes.into()));
}
}
_ => return Err(BlockError::UnsupportedCodec(codec)),
};
Ok(bytes)
}
/// Decode block to ipld.
pub fn decode_ipld(cid: &Cid, data: &[u8]) -> Result<Ipld, BlockError> {
let ipld = match cid.codec() {
Codec::DagCBOR => DagCborCodec::decode(data)?,
Codec::DagProtobuf => DagPbCodec::decode(data)?,
Codec::DagJSON => DagJsonCodec::decode(data)?,
Codec::Raw => Ipld::Bytes(data.to_vec()),
_ => return Err(BlockError::UnsupportedCodec(cid.codec())),
};
Ok(ipld)
}

10
src/lib.rs
View File

@ -6,6 +6,7 @@
#[macro_use]
extern crate tracing;
pub use crate::ipld::Ipld;
use anyhow::{anyhow, format_err};
use async_std::path::PathBuf;
pub use bitswap::{BitswapEvent, Block, Stats};
@ -15,7 +16,6 @@ use futures::channel::mpsc::{channel, Receiver, Sender};
use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender};
use futures::sink::SinkExt;
use futures::stream::{Fuse, Stream};
pub use libipld::ipld::Ipld;
pub use libp2p::core::{connection::ListenerId, ConnectedPoint, Multiaddr, PeerId, PublicKey};
pub use libp2p::identity::Keypair;
use tracing::Span;
@ -33,6 +33,8 @@ use std::task::{Context, Poll};
mod config;
mod dag;
pub mod error;
#[macro_use]
pub mod ipld;
pub mod ipns;
pub mod p2p;
pub mod path;
@ -1203,7 +1205,7 @@ fn could_be_bound_from_ephemeral(
#[cfg(test)]
mod tests {
use super::*;
use libipld::ipld;
use crate::make_ipld;
use libp2p::build_multiaddr;
use multihash::Sha2_256;
@ -1289,7 +1291,7 @@ mod tests {
async fn test_put_and_get_dag() {
let ipfs = Node::new("test_node").await;
let data = ipld!([-1, -2, -3]);
let data = make_ipld!([-1, -2, -3]);
let cid = ipfs.put_dag(data.clone()).await.unwrap();
let new_data = ipfs.get_dag(cid.into()).await.unwrap();
assert_eq!(data, new_data);
@ -1299,7 +1301,7 @@ mod tests {
async fn test_pin_and_unpin() {
let ipfs = Node::new("test_node").await;
let data = ipld!([-1, -2, -3]);
let data = make_ipld!([-1, -2, -3]);
let cid = ipfs.put_dag(data.clone()).await.unwrap();
ipfs.pin_block(&cid).await.unwrap();

2
src/path.rs
View File

@ -1,7 +1,7 @@
use crate::error::{Error, TryError};
use crate::ipld::Ipld;
use cid::Cid;
use core::convert::{TryFrom, TryInto};
use libipld::ipld::Ipld;
use libp2p::PeerId;
use std::fmt;
use std::str::FromStr;

3
src/unixfs/mod.rs
View File

@ -1,13 +1,12 @@
use crate::dag::IpldDag;
use crate::error::Error;
use crate::ipld::{dag_pb::PbNode, Ipld};
use crate::path::IpfsPath;
use crate::repo::RepoTypes;
use async_std::fs;
use async_std::io::ReadExt;
use async_std::path::PathBuf;
use cid::{Cid, Codec};
use libipld::ipld::Ipld;
use libipld::pb::PbNode;
use std::collections::BTreeMap;
use std::convert::TryInto;