Indexer-in-Docker dies occasionally on M1 #278
Assignees
Comments
|
@mieubrisse We don't run indexer-for-explorer on M1, but if you can give us a minimum reproducible example, we can test it on our side. Otherwise, it is hard to tell anything. Will you consider having arm64 image? nearcore and indexer-for-explorer compiles just fine on Mac M1 these days (even though it uses a different Wasm VM since wasmer single-pass backend still does not support ARM, but it should not be noticeable in local testing) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This has been reported by several users using NEARtosis. Logs of a sample dead container:
[2mJun 01 20:21:50.150[0m [32m INFO[0m indexer_for_explorer: Block height 7666
[2mJun 01 20:21:50.663[0m [32m INFO[0m indexer_for_explorer: Block height 7667
[2mJun 01 20:21:51.674[0m [32m INFO[0m indexer_for_explorer: Block height 7668
[2mJun 01 20:21:52.201[0m [32m INFO[0m indexer_for_explorer: Block height 7669
[2mJun 01 20:21:52.708[0m [32m INFO[0m indexer_for_explorer: Block height 7670
[2mJun 01 20:21:53.234[0m [32m INFO[0m indexer_for_explorer: Block height 7671
[2mJun 01 20:21:54.248[0m [32m INFO[0m indexer_for_explorer: Block height 7672
[2mJun 01 20:21:54.779[0m [32m INFO[0m indexer_for_explorer: Block height 7673
run-in-docker.sh: line 113: 62 Killed DATABASE_URL="$
{database_url}
" "$
{indexer_binary_filepath}
" --home-dir "$
{LOCALNET_NEAR_DIRPATH}
" run --store-genesis sync-from-latest${1+"$
{@}
"}
Container inspect output:
{ "Id": "c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e", "Created": "2022-06-01T18:59:15.444782549Z", "Path": "/bin/sh", "Args": [ "-c", "bash run-in-docker.sh ./diesel \"$ {DATABASE_URL}\" ./indexer-explorer" ], "State": { "Status": "exited", "Running": false, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 0, "ExitCode": 137, "Error": "", "StartedAt": "2022-06-01T18:59:15.74320059Z", "FinishedAt": "2022-06-01T20:21:59.089780258Z" }, "Image": "sha256:6d52a82748f28559fb1b2977cde523ed993cc93cbecd36e5bac6e9399a26448c", "ResolvConfPath": "/var/lib/docker/containers/c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e/resolv.conf", "HostnamePath": "/var/lib/docker/containers/c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e/hostname", "HostsPath": "/var/lib/docker/containers/c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e/hosts", "LogPath": "/var/lib/docker/containers/c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e/c05bf675c39548e735ccd7f4e888ca205267a81c209145dedcb3e98b9452ad3e-json.log", "Name": "/near-kurtosis-module- ~1654109944~ ~user-service~-indexer-node-1654109954", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": {} }, "NetworkMode": "default", "PortBindings": { "24567/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "0" } ], "3030/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "0" } ] }, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": [], "CapDrop": null, "CgroupnsMode": "private", "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": [], "GroupAdd": null, "IpcMode": "private", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": null, "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": null, "DeviceCgroupRules": null, "DeviceRequests": null, "KernelMemory": 0, "KernelMemoryTCP": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": null, "PidsLimit": null, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0, "MaskedPaths": [ "/proc/asound", "/proc/acpi", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware" ], "ReadonlyPaths": [ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger" ] }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/2070ec3394d2a812cc4cac885be979bb1f9366c2d2d815e138532562368d79b6-init/diff:/var/lib/docker/overlay2/6e78631e5adf6b9e25e29aa34b841012941d3b60f745717bfa059be911c247d6/diff:/var/lib/docker/overlay2/e1b2d6a6f0db5ed1105d4eb20463e68a729bc1ba3d4cf08331d31f6adb8b0eaa/diff:/var/lib/docker/overlay2/9fc17cf2c893fb2bf6b557e9103131e7786a6d6a70df657448d8bfd8c85dd25f/diff:/var/lib/docker/overlay2/14060890fbd19f9a272c09156781eb441b1f45697e4327132b39bde2e6f6fc79/diff:/var/lib/docker/overlay2/fc2bac9e14b3d1603522da8c926122948eac13bc2d2b21f422d0bf0aa7a40df2/diff:/var/lib/docker/overlay2/6627adbcafdcfe334bbfbfeee76def58850d9c8bbe7e1b20d5f82ab04740d03c/diff:/var/lib/docker/overlay2/7468d847dfeb0dfe67d4c290915772736dd2bb968dfd5487170b67cb4d50cc12/diff", "MergedDir": "/var/lib/docker/overlay2/2070ec3394d2a812cc4cac885be979bb1f9366c2d2d815e138532562368d79b6/merged", "UpperDir": "/var/lib/docker/overlay2/2070ec3394d2a812cc4cac885be979bb1f9366c2d2d815e138532562368d79b6/diff", "WorkDir": "/var/lib/docker/overlay2/2070ec3394d2a812cc4cac885be979bb1f9366c2d2d815e138532562368d79b6/work" }, "Name": "overlay2" }, "Mounts": [], "Config": { "Hostname": "c05bf675c395", "Domainname": "", "User": "", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "ExposedPorts": { "24567/tcp": {}, "3030/tcp": {} }, "Tty": false, "OpenStdin": true, "StdinOnce": false, "Env": [ "DATABASE_URL=postgres://near:near@contract-helper-db:5432/indexer", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ], "Cmd": null, "Image": "kurtosistech/near-indexer-for-explorer:7510e7f", "Volumes": null, "WorkingDir": "/near/indexer-explorer", "Entrypoint": [ "/bin/sh", "-c", "bash run-in-docker.sh ./diesel \"${DATABASE_URL} \" ./indexer-explorer" ], "OnBuild": null, "Labels": { "com.kurtosistech.app-id": "kurtosis", "com.kurtosistech.container-type": "user-service", "com.kurtosistech.enclave-id": "near-kurtosis-module--1654109944", "com.kurtosistech.guid": "indexer-node-1654109954", "com.kurtosistech.id": "indexer-node", "com.kurtosistech.ports": "rpc:3030/TCP,gossip:24567/TCP", "com.kurtosistech.private-ip": "92.55.32.5" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "d8940c61f49404e229e734aec553f55131df588d46f184c948028298d1f25af7", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": {}, "SandboxKey": "/var/run/docker/netns/d8940c61f494", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": { "bridge": { "IPAMConfig": null, "Links": null, "Aliases": null, "NetworkID": "82cec4b74b1c477c378554da59b90adbad15ae887c1aa74e346656644b7c0711", "EndpointID": "", "Gateway": "", "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "", "DriverOpts": null } , "near-kurtosis-module--1654109944": { "IPAMConfig": { "IPv4Address": "92.55.32.5" } , "Links": null, "Aliases": [ "indexer-node", "c05bf675c395" ], "NetworkID": "d7cc27b2835a15de3e243ade6d18bbbeff83a1a917abc82ad171608f44568ce9", "EndpointID": "", "Gateway": "", "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "", "DriverOpts": null } } } }This is happening for two users, both of whom have 8gb of memory associated, so it doesn't look to be an OOM error (and the "OOMKilled" flag isn't set in the container). Both users have said that the same thing happens using LocalTerra on M1.
The image is built by the Kurtosis team, using a CircleCI x86-64 Linux machine.
The text was updated successfully, but these errors were encountered: