tsarevvv/rust-ipfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge #321

Browse Source 
321: fix: make ipfs-http to use ipfs::IpfsPath r=ljedrz a=koivunej

First step towards solving the #238 

- fix `ipfs::IpfsPath`
  - swapped `ipfs::path::SubPath` for `String` (commit has reason)
  - removed `IpfsPath::push(&mut self, segment: impl Into<SubPath>)`
  - remove the implicit ending empty path segment (seemed intentional, don't know why that was there)
  - migrate tests
    - most notably add support for `cid/a/b/c` cases
    - move "good_but_unsupported" over to "good_paths" (as ipfs::IpfsPath validates more)
- make ipfs-http use `ipfs::IpfsPath`

This specifically doesn't include:

- separation of different kinds of IpfsPaths (`CidPath`, `IpnsPath`)
  - instead there is currently an expect over at http side for cid paths
- probably a good idea to refactor out the `Vec<String>` (sub)path somehow; it might be heavily reusable for unixfs for example
- moving the resolving over to ipfs from the current not so great `ipfs_http::v0::refs::walk_paths` 

These should be handled on next PRs. The end result here is not too pretty but it shouldn't break much either.

Big unsolved mysteries:

- how to implement conformance test compatible refs once this has been moved over to ipfs (probably need to adjust the test)

Co-authored-by: Joonas Koivunen <joonas@equilibrium.co>
Co-authored-by: ljedrz <ljedrz@users.noreply.github.com>
This commit is contained in:
bors[bot] 2020-08-20 09:12:49 +00:00 committed by GitHub
commit 4f762446f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 308 additions and 493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
http/src/v0/dag.rs
View File

@ -121,12 +121,16 @@ async fn inner_resolve<T: IpfsTypes>(
ipfs: Ipfs<T>,
opts: ResolveOptions,
) -> Result<impl Reply, Rejection> {
use crate::v0::refs::{walk_path, IpfsPath};
use crate::v0::refs::{walk_path, IpfsPath, WalkOptions};
use std::convert::TryFrom;
let path = IpfsPath::try_from(opts.arg.as_str()).map_err(StringError::from)?;
let (current, _, remaining) = walk_path(&ipfs, path)
let walk_opts = WalkOptions {
follow_dagpb_data: true,
};
let (current, _, remaining) = walk_path(&ipfs, &walk_opts, path)
.maybe_timeout(opts.timeout.map(StringSerialized::into_inner))
.await
.map_err(StringError::from)?

98
http/src/v0/refs.rs
View File

@ -19,7 +19,9 @@ mod format;
use format::EdgeFormatter;
pub(crate) mod path;
pub use path::{IpfsPath, WalkSuccess};
pub use ipfs::path::IpfsPath;
use path::resolve_segment;
pub use path::WalkSuccess;
use crate::v0::support::{HandledErr, StreamResponse};
@ -47,18 +49,12 @@ async fn refs_inner<T: IpfsTypes>(
formatter
);
let mut paths = opts
let paths = opts
.arg
.iter()
.map(|s| IpfsPath::try_from(s.as_str()).map_err(StringError::from))
.collect::<Result<Vec<_>, _>>()?;
for path in paths.iter_mut() {
// this is needed because the paths should not error on matching on the final Data segment,
// it just becomes projected as `Loaded::Raw(_)`, however such items can have no links.
path.set_follow_dagpb_data(true);
}
let st = refs_paths(ipfs, paths, max_depth, opts.unique)
.maybe_timeout(opts.timeout)
.await
@ -72,6 +68,9 @@ async fn refs_inner<T: IpfsTypes>(
// FIXME: there should be a total timeout arching over path walking to the stream completion.
// hyper can't do trailer errors on chunked bodies so ... we can't do much.
// FIXME: the test case 'should print nothing for non-existent hashes' is problematic as it
// expects the headers to be blocked before the timeout expires.
let st = st.map(move |res| {
let res = match res {
Ok((source, dest, link_name)) => {
@ -135,19 +134,26 @@ async fn refs_paths<T: IpfsTypes>(
use futures::stream::FuturesOrdered;
use futures::stream::TryStreamExt;
// the assumption is that futuresordered will poll the first N items until the first completes,
// buffering the others. it might not be 100% parallel but it's probably enough.
let mut walks = FuturesOrdered::new();
let opts = WalkOptions {
follow_dagpb_data: true,
};
for path in paths {
walks.push(walk_path(&ipfs, path));
}
// added braces to spell it out for borrowck that opts does not outlive this fn
let iplds = {
// the assumption is that futuresordered will poll the first N items until the first completes,
// buffering the others. it might not be 100% parallel but it's probably enough.
let mut walks = FuturesOrdered::new();
// strip out the path inside last document, we don't need it
let iplds = walks
.map_ok(|(cid, maybe_ipld, _)| (cid, maybe_ipld))
.try_collect()
.await?;
for path in paths {
walks.push(walk_path(&ipfs, &opts, path));
}
// strip out the path inside last document, we don't need it
walks
.map_ok(|(cid, maybe_ipld, _)| (cid, maybe_ipld))
.try_collect()
.await?
};
Ok(iplds_refs(ipfs, iplds, max_depth, unique))
}
@ -238,17 +244,29 @@ pub enum Loaded {
Ipld(Ipld),
}
#[derive(Default, Debug)]
pub struct WalkOptions {
pub follow_dagpb_data: bool,
}
/// Walks the `path` while loading the links.
///
/// Returns the Cid where we ended up, and an optional Ipld structure if one was projected, and the
/// path inside the last document we walked.
pub async fn walk_path<T: IpfsTypes>(
ipfs: &Ipfs<T>,
mut path: IpfsPath,
opts: &WalkOptions,
path: IpfsPath,
) -> Result<(Cid, Loaded, Vec<String>), WalkError> {
use ipfs::unixfs::ll::{MaybeResolved, ResolveError};
let mut current = path.take_root().unwrap();
let mut current = path
.root()
.cid()
.expect("unsupported: need to add an error variant for this! or design around it")
.to_owned();
let mut iter = path.path().iter();
// cache for any datastructure used in repeated hamt lookups
let mut cache = None;
@ -269,7 +287,7 @@ pub async fn walk_path<T: IpfsTypes>(
// needs to be mutable because the Ipld walk will overwrite it to project down in the
// document
let mut needle = if let Some(needle) = path.next() {
let mut needle = if let Some(needle) = iter.next() {
needle
} else {
return Ok((current, Loaded::Raw(data), Vec::new()));
@ -283,13 +301,20 @@ pub async fn walk_path<T: IpfsTypes>(
continue;
}
Ok(MaybeResolved::NotFound) => {
return handle_dagpb_not_found(current, &data, needle, &path)
return handle_dagpb_not_found(
current,
&data,
needle.to_owned(),
iter.next().is_none(),
opts,
)
}
Err(ResolveError::UnexpectedType(_)) => {
// the conformance tests use a path which would end up going through a file
// and the returned error string is tested against listed alternatives.
// unexpected type is not one of them.
let e = WalkFailed::from(path::WalkFailed::UnmatchedNamedLink(needle));
let e =
WalkFailed::from(path::WalkFailed::UnmatchedNamedLink(needle.to_owned()));
return Err(WalkError::from((e, current)));
}
Err(e) => return Err(WalkError::from((WalkFailed::from(e), current))),
@ -314,7 +339,13 @@ pub async fn walk_path<T: IpfsTypes>(
break;
}
Ok(MaybeResolved::NotFound) => {
return handle_dagpb_not_found(next, &data, needle, &path)
return handle_dagpb_not_found(
next,
&data,
needle.to_owned(),
iter.next().is_none(),
opts,
)
}
Err(e) => {
return Err(WalkError::from((
@ -336,8 +367,7 @@ pub async fn walk_path<T: IpfsTypes>(
// this needs to be stored at least temporarily to recover the path_inside_last or
// the "remaining path"
let tmp = needle.clone();
ipld = match IpfsPath::resolve_segment(needle, ipld) {
Ok(WalkSuccess::EmptyPath(_)) => unreachable!(),
ipld = match resolve_segment(&needle, ipld) {
Ok(WalkSuccess::AtDestination(ipld)) => {
path_inside_last.push(tmp);
ipld
@ -350,13 +380,13 @@ pub async fn walk_path<T: IpfsTypes>(
};
// we might resolve multiple segments inside a single document
needle = match path.next() {
needle = match iter.next() {
Some(needle) => needle,
None => break,
};
}
if path.len() == 0 {
if iter.len() == 0 {
// when done with the remaining IpfsPath we should be set with the projected Ipld
// document
path_inside_last.shrink_to_fit();
@ -370,11 +400,12 @@ fn handle_dagpb_not_found(
at: Cid,
data: &[u8],
needle: String,
path: &IpfsPath,
last_segment: bool,
opts: &WalkOptions,
) -> Result<(Cid, Loaded, Vec<String>), WalkError> {
use ipfs::unixfs::ll::dagpb::node_data;
if needle == "Data" && path.len() == 0 && path.follow_dagpb_data() {
if opts.follow_dagpb_data && last_segment && needle == "Data" {
// /dag/resolve needs to "resolve through" a dag-pb node down to the "just data" even
// though we do not need to extract it ... however this might be good to just filter with
// refs, as no refs of such path can exist as the links are in the outer structure.
@ -424,6 +455,7 @@ fn iplds_refs<T: IpfsTypes>(
return;
}
// FIXME: this should be queued_or_visited
let mut visited = HashSet::new();
let mut work = VecDeque::new();
@ -463,8 +495,8 @@ fn iplds_refs<T: IpfsTypes>(
warn!("failed to load {}, linked from {}: {}", cid, source, e);
// TODO: yield error msg
// unsure in which cases this happens, because we'll start to search the content
// and stop only when request has been cancelled (FIXME: not yet, because dropping
// all subscriptions doesn't "stop the operation.")
// and stop only when request has been cancelled (FIXME: no way to stop this
// operation)
continue;
}
};

404
http/src/v0/refs/path.rs
View File

@ -8,223 +8,46 @@
use cid::{self, Cid};
use ipfs::ipld::Ipld;
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::fmt;
#[derive(Debug)]
pub enum PathError {
InvalidCid(cid::Error),
InvalidPath,
}
impl fmt::Display for PathError {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match *self {
PathError::InvalidCid(e) => write!(fmt, "{}", e),
PathError::InvalidPath => write!(fmt, "invalid path"),
pub fn resolve_segment(key: &str, mut ipld: Ipld) -> Result<WalkSuccess, WalkFailed> {
ipld = match ipld {
Ipld::Link(cid) if key == "." => {
// go-ipfs: allows this to be skipped. let's require the dot for now.
// FIXME: this would require the iterator to be peekable in addition.
return Ok(WalkSuccess::Link(key.to_owned(), cid));
}
}
}
impl std::error::Error for PathError {}
/// Ipfs path following https://github.com/ipfs/go-path/
#[derive(Debug)]
pub struct IpfsPath {
/// Option to support moving the cid
root: Option<Cid>,
path: std::vec::IntoIter<String>,
/// True by default, to allow "finding" `Data` under dag-pb node
/// TODO: document why this matters
follow_dagpb_data: bool,
}
impl From<Cid> for IpfsPath {
/// Creates a new IpfsPath from just the Cid, which is the same as parsing from a string
/// representation of a Cid but cannot fail.
fn from(root: Cid) -> IpfsPath {
IpfsPath {
root: Some(root),
path: Vec::new().into_iter(),
follow_dagpb_data: true,
Ipld::Map(mut m) => {
if let Some(ipld) = m.remove(key) {
ipld
} else {
return Err(WalkFailed::UnmatchedMapProperty(m, key.to_owned()));
}
}
}
}
impl TryFrom<&str> for IpfsPath {
type Error = PathError;
fn try_from(path: &str) -> Result<Self, Self::Error> {
let mut split = path.splitn(2, "/ipfs/");
let first = split.next();
let (_root, path) = match first {
Some("") => {
/* started with /ipfs/ */
if let Some(x) = split.next() {
// was /ipfs/x
("ipfs", x)
Ipld::List(mut l) => {
if let Ok(index) = key.parse::<usize>() {
if index < l.len() {
l.swap_remove(index)
} else {
// just the /ipfs/
return Err(PathError::InvalidPath);
return Err(WalkFailed::ListIndexOutOfRange(l, index));
}
}
Some(x) => {
/* maybe didn't start with /ipfs/, need to check second */
if split.next().is_some() {
// x/ipfs/_
return Err(PathError::InvalidPath);
}
("", x)
}
None => return Err(PathError::InvalidPath),
};
let mut split = path.splitn(2, '/');
let root = split
.next()
.expect("first value from splitn(2, _) must exist");
let path = split
.next()
.iter()
.flat_map(|s| s.split('/').filter(|s| !s.is_empty()).map(String::from))
.collect::<Vec<_>>()
.into_iter();
let root = Some(Cid::try_from(root).map_err(PathError::InvalidCid)?);
let follow_dagpb_data = true;
Ok(IpfsPath {
root,
path,
follow_dagpb_data,
})
}
}
impl IpfsPath {
pub fn take_root(&mut self) -> Option<Cid> {
self.root.take()
}
pub fn set_follow_dagpb_data(&mut self, follow: bool) {
self.follow_dagpb_data = follow;
}
pub fn follow_dagpb_data(&self) -> bool {
self.follow_dagpb_data
}
pub fn resolve(&mut self, ipld: Ipld) -> Result<WalkSuccess, WalkFailed> {
let key = match self.next() {
Some(key) => key,
None => return Ok(WalkSuccess::EmptyPath(ipld)),
};
Self::resolve_segment(key, ipld)
}
pub fn resolve_segment(key: String, mut ipld: Ipld) -> Result<WalkSuccess, WalkFailed> {
ipld = match ipld {
Ipld::Link(cid) if key == "." => {
// go-ipfs: allows this to be skipped. let's require the dot for now.
// FIXME: this would require the iterator to be peekable in addition.
return Ok(WalkSuccess::Link(key, cid));
}
Ipld::Map(mut m) => {
if let Some(ipld) = m.remove(&key) {
ipld
} else {
return Err(WalkFailed::UnmatchedMapProperty(m, key));
}
}
Ipld::List(mut l) => {
if let Ok(index) = key.parse::<usize>() {
if index < l.len() {
l.swap_remove(index)
} else {
return Err(WalkFailed::ListIndexOutOfRange(l, index));
}
} else {
return Err(WalkFailed::UnparseableListIndex(l, key));
}
}
x => return Err(WalkFailed::UnmatchableSegment(x, key)),
};
if let Ipld::Link(next_cid) = ipld {
Ok(WalkSuccess::Link(key, next_cid))
} else {
Ok(WalkSuccess::AtDestination(ipld))
}
}
/// Walks the path depicted by self until either the path runs out or a new link needs to be
/// traversed to continue the walk. With !dag-pb documents this can result in subtree of an
/// Ipld be represented.
///
/// # Panics
///
/// If the current Ipld is from a dag-pb and the libipld has changed it's dag-pb tree structure.
// FIXME: this needs to be removed and ... we should have some generic Ipld::walk
pub fn walk(&mut self, current: &Cid, mut ipld: Ipld) -> Result<WalkSuccess, WalkFailed> {
if self.len() == 0 {
return Ok(WalkSuccess::EmptyPath(ipld));
}
if current.codec() == cid::Codec::DagProtobuf {
return Err(WalkFailed::UnsupportedWalkOnDagPbIpld);
}
loop {
ipld = match self.resolve(ipld)? {
WalkSuccess::AtDestination(ipld) => ipld,
WalkSuccess::EmptyPath(ipld) => return Ok(WalkSuccess::AtDestination(ipld)),
ret @ WalkSuccess::Link(_, _) => return Ok(ret),
};
}
}
pub fn remaining_path(&self) -> &[String] {
self.path.as_slice()
}
// Currently unused by commited code, but might become handy or easily removed later on.
#[allow(dead_code)]
pub fn debug<'a>(&'a self, current: &'a Cid) -> impl fmt::Debug + 'a {
struct DebuggableIpfsPath<'a> {
current: &'a Cid,
segments: &'a [String],
}
impl<'a> fmt::Debug for DebuggableIpfsPath<'a> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{}", self.current)?;
if !self.segments.is_empty() {
write!(fmt, "/...")?;
}
for seg in self.segments {
write!(fmt, "/{}", seg)?;
}
Ok(())
} else {
return Err(WalkFailed::UnparseableListIndex(l, key.to_owned()));
}
}
x => return Err(WalkFailed::UnmatchableSegment(x, key.to_owned())),
};
DebuggableIpfsPath {
current,
segments: self.path.as_slice(),
}
if let Ipld::Link(next_cid) = ipld {
Ok(WalkSuccess::Link(key.to_owned(), next_cid))
} else {
Ok(WalkSuccess::AtDestination(ipld))
}
}
/// The success values walking an `IpfsPath` can result to.
#[derive(Debug, PartialEq)]
pub enum WalkSuccess {
/// IpfsPath was already empty, or became empty during previous walk
// FIXME: remove this when migrating away from IpfsPath::walk
EmptyPath(Ipld),
/// IpfsPath arrived at destination, following walk attempts will return EmptyPath
AtDestination(Ipld),
/// Path segment lead to a link which needs to be loaded to continue the walk
@ -281,112 +104,16 @@ impl fmt::Display for WalkFailed {
impl std::error::Error for WalkFailed {}
impl Iterator for IpfsPath {
type Item = String;
fn next(&mut self) -> Option<String> {
self.path.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.path.size_hint()
}
}
impl ExactSizeIterator for IpfsPath {
fn len(&self) -> usize {
self.path.len()
}
}
#[cfg(test)]
mod tests {
use super::{IpfsPath, WalkSuccess};
use super::WalkFailed;
use super::{resolve_segment, WalkSuccess};
use cid::Cid;
use ipfs::{ipld::Ipld, make_ipld};
use ipfs::{ipld::Ipld, make_ipld, IpfsPath};
use std::convert::TryFrom;
// good_paths, good_but_unsupported, bad_paths from https://github.com/ipfs/go-path/blob/master/path_test.go
#[test]
fn good_paths() {
let good = [
("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", 0),
("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", 1),
(
"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
6,
),
(
"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
6,
),
("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", 0),
];
for &(good, len) in &good {
let p = IpfsPath::try_from(good).unwrap();
assert_eq!(p.len(), len);
}
}
#[test]
fn good_but_unsupported() {
let unsupported = [
"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a",
"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
];
for &unsupported in &unsupported {
// these fail from failing to parse "ipld" or "ipns" as cid
IpfsPath::try_from(unsupported).unwrap_err();
}
}
#[test]
fn bad_paths() {
let bad = [
"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a",
"/ipfs/foo",
"/ipfs/",
"ipfs/",
"ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
"/ipld/foo",
"/ipld/",
"ipld/",
"ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
];
for &bad in &bad {
IpfsPath::try_from(bad).unwrap_err();
}
}
#[test]
fn trailing_slash_is_ignored() {
let paths = [
"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/",
"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/",
];
for &path in &paths {
let p = IpfsPath::try_from(path).unwrap();
assert_eq!(p.len(), 0);
}
}
#[test]
fn multiple_slashes_are_deduplicated() {
// this is similar to behaviour in js-ipfs, as of
// https://github.com/ipfs-rust/rust-ipfs/pull/147/files#r408939850
let p =
IpfsPath::try_from("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n///a").unwrap();
assert_eq!(p.len(), 1);
}
fn example_doc_and_a_cid() -> (Ipld, Cid) {
let cid = Cid::try_from("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n").unwrap();
let doc = make_ipld!({
@ -430,23 +157,16 @@ mod tests {
];
for (path, expected) in &good_examples {
let mut p = IpfsPath::try_from(*path).unwrap();
let p = IpfsPath::try_from(*path).unwrap();
// not really the document cid but it doesn't matter; it just needs to be !dag-pb
let doc_cid = p.take_root().unwrap();
//let doc_cid = p.take_root().unwrap();
// projection
assert_eq!(
p.walk(&doc_cid, example_doc.clone()),
walk(example_doc.clone(), &p).map(|r| r.0),
Ok(WalkSuccess::AtDestination(expected.clone()))
);
// after walk the iterator has been exhausted and the path is always empty and returns
// the given value
assert_eq!(
p.walk(&doc_cid, example_doc.clone()),
Ok(WalkSuccess::EmptyPath(example_doc.clone()))
);
}
}
@ -456,11 +176,10 @@ mod tests {
let doc = make_ipld!(cid.clone());
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/./foobar";
let mut p = IpfsPath::try_from(path).unwrap();
let doc_cid = p.take_root().unwrap();
let p = IpfsPath::try_from(path).unwrap();
assert_eq!(
p.walk(&doc_cid, doc),
walk(doc, &p).map(|r| r.0),
Ok(WalkSuccess::Link(".".into(), cid))
);
}
@ -471,12 +190,11 @@ mod tests {
let doc = make_ipld!(cid);
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/foobar";
let mut p = IpfsPath::try_from(path).unwrap();
let doc_cid = p.take_root().unwrap();
let p = IpfsPath::try_from(path).unwrap();
// go-ipfs would walk over the link even without a dot, this will probably come up with
// dag/get
p.walk(&doc_cid, doc).unwrap_err();
walk(doc, &p).unwrap_err();
}
#[test]
@ -484,14 +202,15 @@ mod tests {
let (example_doc, cid) = example_doc_and_a_cid();
let path = "bafyreielwgy762ox5ndmhx6kpi6go6il3gzahz3ngagb7xw3bj3aazeita/nested/even/2/or/something_on_the_next_block";
let mut p = IpfsPath::try_from(path).unwrap();
let doc_cid = p.take_root().unwrap();
let p = IpfsPath::try_from(path).unwrap();
let (success, mut remaining) = walk(example_doc, &p).unwrap();
assert_eq!(success, WalkSuccess::Link("or".into(), cid));
assert_eq!(
p.walk(&doc_cid, example_doc),
Ok(WalkSuccess::Link("or".into(), cid))
remaining.next().map(|s| s.as_str()),
Some("something_on_the_next_block")
);
assert_eq!(p.next(), Some("something_on_the_next_block".into()));
}
#[test]
@ -505,10 +224,45 @@ mod tests {
];
for path in &mismatches {
let mut p = IpfsPath::try_from(*path).unwrap();
let doc_cid = p.take_root().unwrap();
let p = IpfsPath::try_from(*path).unwrap();
// let doc_cid = p.take_root().unwrap();
// using just unwrap_err as the context would be quite troublesome to write
p.walk(&doc_cid, example_doc.clone()).unwrap_err();
walk(example_doc.clone(), &p).unwrap_err();
}
}
fn walk(
mut doc: Ipld,
path: &'_ IpfsPath,
) -> Result<
(
WalkSuccess,
impl Iterator<Item = &'_ String> + std::fmt::Debug,
),
WalkFailed,
> {
if path.path().is_empty() {
unreachable!("empty path");
}
let current = path.root().cid().unwrap();
if current.codec() == cid::Codec::DagProtobuf {
return Err(WalkFailed::UnsupportedWalkOnDagPbIpld);
}
let mut iter = path.path().iter();
loop {
let needle = if let Some(needle) = iter.next() {
needle
} else {
return Ok((WalkSuccess::AtDestination(doc), iter));
};
doc = match resolve_segment(needle, doc)? {
WalkSuccess::AtDestination(ipld) => ipld,
ret @ WalkSuccess::Link(_, _) => return Ok((ret, iter)),
};
}
}
}

10
http/src/v0/root_files.rs
View File

@ -60,8 +60,7 @@ pub fn cat<T: IpfsTypes>(
}
async fn cat_inner<T: IpfsTypes>(ipfs: Ipfs<T>, args: CatArgs) -> Result<impl Reply, Rejection> {
let mut path = IpfsPath::try_from(args.arg.as_str()).map_err(StringError::from)?;
path.set_follow_dagpb_data(false);
let path = IpfsPath::try_from(args.arg.as_str()).map_err(StringError::from)?;
let range = match (args.offset, args.length) {
(Some(start), Some(len)) => Some(start..(start + len)),
@ -73,7 +72,7 @@ async fn cat_inner<T: IpfsTypes>(ipfs: Ipfs<T>, args: CatArgs) -> Result<impl Re
// FIXME: this is here until we have IpfsPath back at ipfs
// FIXME: this timeout here is ... not great; the end user could be waiting for 2*timeout
let (cid, _, _) = walk_path(&ipfs, path)
let (cid, _, _) = walk_path(&ipfs, &Default::default(), path)
.maybe_timeout(args.timeout.clone().map(StringSerialized::into_inner))
.await
.map_err(StringError::from)?
@ -118,12 +117,11 @@ pub fn get<T: IpfsTypes>(
async fn get_inner<T: IpfsTypes>(ipfs: Ipfs<T>, args: GetArgs) -> Result<impl Reply, Rejection> {
use futures::stream::TryStreamExt;
let mut path = IpfsPath::try_from(args.arg.as_str()).map_err(StringError::from)?;
path.set_follow_dagpb_data(false);
let path = IpfsPath::try_from(args.arg.as_str()).map_err(StringError::from)?;
// FIXME: this is here until we have IpfsPath back at ipfs
// FIXME: this timeout is only for the first step, should be for the whole walk!
let (cid, _, _) = walk_path(&ipfs, path)
let (cid, _, _) = walk_path(&ipfs, &Default::default(), path)
.maybe_timeout(args.timeout.map(StringSerialized::into_inner))
.await
.map_err(StringError::from)?

64
src/dag.rs
View File

@ -1,6 +1,6 @@
use crate::error::Error;
use crate::ipld::{decode_ipld, encode_ipld, Ipld};
use crate::path::{IpfsPath, IpfsPathError, SubPath};
use crate::path::IpfsPath;
use crate::repo::RepoTypes;
use crate::Ipfs;
use bitswap::Block;
@ -37,11 +37,7 @@ impl<Types: RepoTypes> IpldDag<Types> {
};
let mut ipld = decode_ipld(&cid, self.ipfs.repo.get_block(&cid).await?.data())?;
for sub_path in path.iter() {
if !can_resolve(&ipld, sub_path) {
let path = sub_path.to_owned();
return Err(IpfsPathError::ResolveError { ipld, path }.into());
}
ipld = resolve(ipld, sub_path);
ipld = try_resolve(ipld, sub_path)?;
ipld = match ipld {
Ipld::Link(cid) => decode_ipld(&cid, self.ipfs.repo.get_block(&cid).await?.data())?,
ipld => ipld,
@ -51,43 +47,27 @@ impl<Types: RepoTypes> IpldDag<Types> {
}
}
fn can_resolve(ipld: &Ipld, sub_path: &SubPath) -> bool {
match sub_path {
SubPath::Key(key) => {
if let Ipld::Map(ref map) = ipld {
if map.contains_key(key) {
return true;
}
}
}
SubPath::Index(index) => {
if let Ipld::List(ref vec) = ipld {
if *index < vec.len() {
return true;
}
}
}
fn try_resolve(ipld: Ipld, segment: &str) -> Result<Ipld, Error> {
match ipld {
Ipld::Map(mut map) => map
.remove(segment)
.ok_or_else(|| anyhow::anyhow!("no such segment: {:?}", segment)),
Ipld::List(mut vec) => match segment.parse::<usize>() {
Ok(index) if index < vec.len() => Ok(vec.swap_remove(index)),
Ok(_) => Err(anyhow::anyhow!(
"index out of range 0..{}: {:?}",
vec.len(),
segment
)),
Err(_) => Err(anyhow::anyhow!("invalid list index: {:?}", segment)),
},
link @ Ipld::Link(_) if segment == "." => Ok(link),
other => Err(anyhow::anyhow!(
"cannot resolve {:?} through: {:?}",
segment,
other
)),
}
false
}
fn resolve(ipld: Ipld, sub_path: &SubPath) -> Ipld {
match sub_path {
SubPath::Key(key) => {
if let Ipld::Map(mut map) = ipld {
return map.remove(key).unwrap();
}
}
SubPath::Index(index) => {
if let Ipld::List(mut vec) = ipld {
return vec.swap_remove(*index);
}
}
}
panic!(
"Failed to resolved ipld: {:?} sub_path: {:?}",
ipld, sub_path
);
}
#[cfg(test)]

217
src/path.rs
View File

@ -10,7 +10,7 @@ use thiserror::Error;
#[derive(Clone, Debug, PartialEq)]
pub struct IpfsPath {
root: PathRoot,
path: Vec<SubPath>,
path: Vec<String>,
}
impl FromStr for IpfsPath {
@ -18,21 +18,32 @@ impl FromStr for IpfsPath {
fn from_str(string: &str) -> Result<Self, Error> {
let mut subpath = string.split('/');
let empty = subpath.next();
let root_type = subpath.next();
let key = subpath.next();
let empty = subpath.next().expect("there's always the first split");
let root = match (empty, root_type, key) {
(Some(""), Some("ipfs"), Some(key)) => PathRoot::Ipld(Cid::try_from(key)?),
(Some(""), Some("ipld"), Some(key)) => PathRoot::Ipld(Cid::try_from(key)?),
(Some(""), Some("ipns"), Some(key)) => match PeerId::from_str(key).ok() {
Some(peer_id) => PathRoot::Ipns(peer_id),
None => PathRoot::Dns(key.to_string()),
},
_ => return Err(IpfsPathError::InvalidPath(string.to_owned()).into()),
let root = if !empty.is_empty() {
// by default if there is no prefix it's an ipfs or ipld path
PathRoot::Ipld(Cid::try_from(empty)?)
} else {
let root_type = subpath.next();
let key = subpath.next();
match (empty, root_type, key) {
("", Some("ipfs"), Some(key)) => PathRoot::Ipld(Cid::try_from(key)?),
("", Some("ipld"), Some(key)) => PathRoot::Ipld(Cid::try_from(key)?),
("", Some("ipns"), Some(key)) => match PeerId::from_str(key).ok() {
Some(peer_id) => PathRoot::Ipns(peer_id),
None => PathRoot::Dns(key.to_string()),
},
_ => {
//todo!("empty: {:?}, root: {:?}, key: {:?}", empty, root_type, key);
return Err(IpfsPathError::InvalidPath(string.to_owned()).into());
}
}
};
let mut path = IpfsPath::new(root);
path.push_str(&subpath.collect::<Vec<&str>>().join("/"))?;
path.push_split(subpath)
.map_err(|_| IpfsPathError::InvalidPath(string.to_owned()))?;
Ok(path)
}
}
@ -53,24 +64,28 @@ impl IpfsPath {
self.root = root;
}
pub fn push<T: Into<SubPath>>(&mut self, sub_path: T) {
self.path.push(sub_path.into());
}
pub fn push_str(&mut self, string: &str) -> Result<(), Error> {
if string.is_empty() {
return Ok(());
}
for sub_path in string.split('/') {
self.push_split(string.split('/'))
.map_err(|_| IpfsPathError::InvalidPath(string.to_owned()).into())
}
fn push_split<'a>(&mut self, split: impl Iterator<Item = &'a str>) -> Result<(), ()> {
let mut split = split.peekable();
while let Some(sub_path) = split.next() {
if sub_path == "" {
return Err(IpfsPathError::InvalidPath(string.to_owned()).into());
}
let index = sub_path.parse::<usize>();
if let Ok(index) = index {
self.push(index);
} else {
self.push(sub_path);
return if split.peek().is_none() {
// trim trailing
Ok(())
} else {
// no empty segments in the middle
Err(())
};
}
self.path.push(sub_path.to_owned());
}
Ok(())
}
@ -86,9 +101,13 @@ impl IpfsPath {
Ok(self)
}
pub fn iter(&self) -> impl Iterator<Item = &SubPath> {
pub fn iter(&self) -> impl Iterator<Item = &String> {
self.path.iter()
}
pub fn path(&self) -> &[String] {
&self.path
}
}
impl fmt::Display for IpfsPath {
@ -137,13 +156,25 @@ impl TryInto<PeerId> for IpfsPath {
}
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, PartialEq)]
pub enum PathRoot {
Ipld(Cid),
Ipns(PeerId),
Dns(String),
}
impl fmt::Debug for PathRoot {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
use PathRoot::*;
match self {
Ipld(cid) => write!(fmt, "{}", cid),
Ipns(pid) => write!(fmt, "{}", pid),
Dns(name) => write!(fmt, "{:?}", name),
}
}
}
impl PathRoot {
pub fn is_ipld(&self) -> bool {
matches!(self, PathRoot::Ipld(_))
@ -227,75 +258,20 @@ impl TryInto<PeerId> for PathRoot {
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum SubPath {
Key(String),
Index(usize),
}
impl From<String> for SubPath {
fn from(key: String) -> Self {
SubPath::Key(key)
}
}
impl From<&str> for SubPath {
fn from(key: &str) -> Self {
SubPath::from(key.to_string())
}
}
impl From<usize> for SubPath {
fn from(index: usize) -> Self {
SubPath::Index(index)
}
}
impl SubPath {
pub fn is_key(&self) -> bool {
matches!(*self, SubPath::Key(_))
}
pub fn to_key(&self) -> Option<&String> {
match self {
SubPath::Key(ref key) => Some(key),
_ => None,
}
}
pub fn is_index(&self) -> bool {
matches!(self, SubPath::Index(_))
}
pub fn to_index(&self) -> Option<usize> {
match self {
SubPath::Index(index) => Some(*index),
_ => None,
}
}
}
impl fmt::Display for SubPath {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match *self {
SubPath::Key(ref key) => write!(fmt, "{}", key),
SubPath::Index(index) => write!(fmt, "{}", index),
}
}
}
#[derive(Debug, Error)]
pub enum IpfsPathError {
#[error("Invalid path {0:?}")]
InvalidPath(String),
#[error("Can't resolve {path:?}")]
ResolveError { ipld: Ipld, path: SubPath },
ResolveError { ipld: Ipld, path: String },
#[error("Expected ipld path but found ipns path.")]
ExpectedIpldPath,
}
#[cfg(test)]
mod tests {
use super::IpfsPath;
use std::convert::TryFrom;
/*use super::*;
use bitswap::Block;
@ -347,4 +323,75 @@ mod tests {
let res = "/ipfs/QmRN6wdp1S2A5EtjW9A3M1vKSBuQQGcgvuhoMUoEz4iiT5/key/3";
assert_eq!(path.to_string(), res);
}*/
#[test]
fn good_paths() {
let good = [
("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", 0),
("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", 1),
(
"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
6,
),
(
"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
6,
),
("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", 0),
("/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", 0),
("/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", 1),
(
"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f",
6,
),
("/ipns/QmSrPmbaUKA3ZodhzPWZnpFgcPMFWF4QsxXbkWfEptTBJd", 0),
(
"/ipns/QmSrPmbaUKA3ZodhzPWZnpFgcPMFWF4QsxXbkWfEptTBJd/a/b/c/d/e/f",
6,
),
];
for &(good, len) in &good {
let p = IpfsPath::try_from(good).unwrap();
assert_eq!(p.path().len(), len);
}
}
#[test]
fn bad_paths() {
let bad = [
"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a",
"/ipfs/foo",
"/ipfs/",
"ipfs/",
"ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
"/ipld/foo",
"/ipld/",
"ipld/",
"ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n",
];
for &bad in &bad {
IpfsPath::try_from(bad).unwrap_err();
}
}
#[test]
fn trailing_slash_is_ignored() {
let paths = [
"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/",
"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/",
];
for &path in &paths {
let p = IpfsPath::try_from(path).unwrap();
assert_eq!(p.path().len(), 0, "{:?} from {:?}", p, path);
}
}
#[test]
fn multiple_slashes_are_not_deduplicated() {
// this used to be the behaviour in ipfs-http
IpfsPath::try_from("/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n///a").unwrap_err();
}
}