Failed to fetch certain files from ipfs cluster

Help IPFS
1

Because public IPFS gateway is too slow, I set up own ipfs cluster using kubernetes on AWS. However, when I tried to get files from the cluster, I succeeded for some files but failed for others consistently(failed one kept failing).

How do I debug this? Did I make mistake on configuration? Here’s the configuration I used.

{
    "API": {
        "HTTPHeaders": {
            "Access-Control-Allow-Methods": [
                "PUT",
                "POST"
            ],
            "Access-Control-Allow-Origin": [
                "http://localhost:3000",
                "http://127.0.0.1:5001",
                "https://webui.ipfs.io"
            ]
        }
    },
    "Addresses": {
        "API": "/ip4/0.0.0.0/tcp/5001",
        "Announce": [],
        "AppendAnnounce": [],
        "Gateway": "/ip4/0.0.0.0/tcp/8080",
        "NoAnnounce": [],
        "Swarm": [
            "/ip4/0.0.0.0/tcp/4001",
            "/ip6/::/tcp/4001",
            "/ip4/0.0.0.0/udp/4001/quic",
            "/ip6/::/udp/4001/quic"
        ]
    },
    "AutoNAT": {},
    "Bootstrap": [
        "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
        "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
        "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
        "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
        "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
        "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
    ],
    "DNS": {
        "Resolvers": {}
    },
    "Datastore": {
        "BloomFilterSize": 0,
        "GCPeriod": "1h",
        "HashOnRead": false,
        "Spec": {
            "mounts": [
                {
                    "child": {
                        "path": "blocks",
                        "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
                        "sync": true,
                        "type": "flatfs"
                    },
                    "mountpoint": "/blocks",
                    "prefix": "flatfs.datastore",
                    "type": "measure"
                },
                {
                    "child": {
                        "compression": "none",
                        "path": "datastore",
                        "type": "levelds"
                    },
                    "mountpoint": "/",
                    "prefix": "leveldb.datastore",
                    "type": "measure"
                }
            ],
            "type": "mount"
        },
        "StorageGCWatermark": 90,
        "StorageMax": "10GB"
    },
    "Discovery": {
        "MDNS": {
            "Enabled": true,
            "Interval": 10
        }
    },
    "Experimental": {
        "AcceleratedDHTClient": false,
        "FilestoreEnabled": false,
        "GraphsyncEnabled": false,
        "Libp2pStreamMounting": false,
        "P2pHttpProxy": false,
        "StrategicProviding": false,
        "UrlstoreEnabled": false
    },
    "Gateway": {
        "APICommands": [],
        "HTTPHeaders": {
            "Access-Control-Allow-Headers": [
                "X-Requested-With",
                "Range",
                "User-Agent"
            ],
            "Access-Control-Allow-Methods": [
                "GET"
            ],
            "Access-Control-Allow-Origin": [
                "*"
            ]
        },
        "NoDNSLink": false,
        "NoFetch": false,
        "PathPrefixes": [],
        "PublicGateways": null,
        "RootRedirect": "",
        "Writable": false
    },
    "Identity": {
        "PeerID": "<intentionally hide>"
    },
    "Internal": {},
    "Ipns": {
        "RecordLifetime": "",
        "RepublishPeriod": "",
        "ResolveCacheSize": 128
    },
    "Migration": {
        "DownloadSources": [],
        "Keep": ""
    },
    "Mounts": {
        "FuseAllowOther": false,
        "IPFS": "/ipfs",
        "IPNS": "/ipns"
    },
    "Peering": {
        "Peers": null
    },
    "Pinning": {
        "RemoteServices": {}
    },
    "Plugins": {
        "Plugins": null
    },
    "Provider": {
        "Strategy": ""
    },
    "Pubsub": {
        "DisableSigning": false,
        "Router": ""
    },
    "Reprovider": {
        "Interval": "12h",
        "Strategy": "all"
    },
    "Routing": {
        "Type": "dht"
    },
    "Swarm": {
        "AddrFilters": null,
        "ConnMgr": {
            "GracePeriod": "20s",
            "HighWater": 900,
            "LowWater": 600,
            "Type": "basic"
        },
        "DisableBandwidthMetrics": false,
        "DisableNatPortMap": false,
        "RelayClient": {
            "Enabled": true
        },
        "RelayService": {
            "Enabled": true
        },
        "Transports": {
            "Multiplexers": {},
            "Network": {},
            "Security": {}
        }
    }
}

Thanks in advance!

1 Like
2

Hi mate, have you worked it around?

3

I would inquire as to the nature of the files used. Were they uniform in size? What sequence were they committed to IPFS/IPNS? Could the fail file be a media container (like Matroska) that is digitally watermarked? I had a problem before transferring a Matroska from phone to pC or NAS. It would get to same point then fail. The movie had a digital watermark that was invisible and cause it to silently fail…