From 055fe8208e6a2c73d59c20145669ec08ec74d8e7 Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@equilibrium.co>
Date: Tue, 25 Aug 2020 14:27:42 +0300
Subject: [PATCH] fix: check for unique nodes before queueing

---
 http/src/v0/refs.rs | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/http/src/v0/refs.rs b/http/src/v0/refs.rs
index 80f63fbd..fc16aafb 100644
--- a/http/src/v0/refs.rs
+++ b/http/src/v0/refs.rs
@@ -201,12 +201,15 @@ fn iplds_refs<T: IpfsTypes>(
             return;
         }
 
-        // FIXME: this should be queued_or_visited
-        let mut visited = HashSet::new();
+        let mut queued_or_visited = HashSet::new();
         let mut work = VecDeque::new();
 
         for (origin, ipld) in iplds {
             for (link_name, next_cid) in ipld_links(&origin, ipld) {
+                if unique && !queued_or_visited.insert(next_cid.clone()) {
+                    trace!("skipping already queued {}", next_cid);
+                    continue;
+                }
                 work.push_back((0, next_cid, origin.clone(), link_name));
             }
         }
@@ -222,11 +225,6 @@ fn iplds_refs<T: IpfsTypes>(
                 _ => true
             };
 
-            if unique && !visited.insert(cid.clone()) {
-                trace!("skipping already visited {}", cid);
-                continue;
-            }
-
             let data = match ipfs.get_block(&cid).await {
                 Ok(Block { data, .. }) => data,
                 Err(e) => {
@@ -243,7 +241,6 @@ fn iplds_refs<T: IpfsTypes>(
                 Ok(ipld) => ipld,
                 Err(e) => {
                     warn!("failed to parse {}, linked from {}: {}", cid, source, e);
-                    // TODO: yield error msg
                     // go-ipfs on raw Qm hash:
                     // > failed to decode Protocol Buffers: incorrectly formatted merkledag node: unmarshal failed. proto: illegal wireType 6
                     yield Err(e.to_string());
@@ -253,11 +250,11 @@ fn iplds_refs<T: IpfsTypes>(
 
             if traverse_links {
                 for (link_name, next_cid) in ipld_links(&cid, ipld) {
-                    if unique && visited.contains(&next_cid) {
-                        // skip adding already yielded documents
+                    if unique && !queued_or_visited.insert(next_cid.clone()) {
+                        trace!("skipping already queued {}", next_cid);
                         continue;
                     }
-                    // TODO: could also have a hashset for queued destinations ...
+
                     work.push_back((depth + 1, next_cid, cid.clone(), link_name));
                 }
             }