How to speed up the ipfs data retrieval?

1

Please consider the following code which retrieves json data from some IPFS hashes:

const IPFS      = require('ipfs-mini');
const ipfs      = new IPFS({ host: 'ipfs.infura.io', port: 5001, protocol: 'https' });

var mergeJSON   = require("merge-json");   

async function mergeJSONs( hashArry ){

    var mergedJSON  = {}
    var dataArray   = [];

    for(var hash of hashArry){
            var data = await ipfsRetrieve(hash)
            dataArray.push(data);        
     };

     dataArray.forEach(function(JSON){
         mergedJSON  = mergeJSON.merge(mergedJSON, JSON);
     });

    console.log(mergedJSON);   
}


function ipfsRetrieve( ipfsHash ){
return new Promise( function( resolve, reject ) {
    ipfs.catJSON( ipfsHash, (err, result) => {
        if (err){                
            reject(err);
            return
        }
        resolve(result); 
    });
});    
}

Now, when I call:

var hashArray   = [ "Qmds...nZTDcbbfgcWMKPo6qfi",
                                 "QmZB...bhY2cpK2Ue3JYYvdfgg",
                                 "QmRfr...f9YAN1CtyKztALjj3Vgjg",
                                 "Qmba...czvxHGGPrSWQfN8S9xq",
                                 "QmXH...8vrJP8Eq6bvggggsefsq",
                                 "QmXx...t9RAL6vNZctooR76nXfe",
                                 "QmcH...3c5wW3AMoyypkGmAg",
              ]
mergeJSONs( hashArray )

It takes me some sensible seconds to see the result.How can I speed up this IPFS retrieving specially for large number of hashes?

2

One idea is to retrieve in parallel and skip the dataArray.

const dataPromises = hashArry.map(hash => ipfsRetrieve(hash));
for (const dataPromise of dataPromises) {
  const dataJSON = await dataPromise;
  mergedJSON = mergeJSON.merge(mergedJSON, dataJSON);
}

Of course this could cause a big waste: if a failure happens early, it will still retrieve the rest.

1 Like
3

Thanks for your attention and reply.

I used your suggestion, but again the time consuming part is still getting data from IPFS through hashes. Is there any way for accelerating this?