I can see only these logs on ECS.
{
"action": "startup",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"default_vectorizer_module": "none",
"level": "info",
"msg": "the default vectorizer modules is set to \"none\", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer",
"time": "2024-09-23T09:02:28Z"
}
{
"action": "startup",
"auto_schema_enabled": true,
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "auto schema enabled setting is set to \"true\"",
"time": "2024-09-23T09:02:28Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "Limiting resources: memory: 80%, cores: all but one",
"time": "2024-09-23T09:02:28Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"limit": 6871947673,
"msg": "Set memory limit",
"time": "2024-09-23T09:02:28Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "module offload-s3 is enabled",
"time": "2024-09-23T09:02:28Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "open cluster service",
"servers": {
"dev-csp-ecs-weaviate-service.dev-csp-sddns": 8300
},
"time": "2024-09-23T09:07:34Z"
}
{
"address": "10.12.79.57:8301",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "starting cloud rpc server ...",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "starting raft sub-system ...",
"time": "2024-09-23T09:07:34Z"
}
{
"address": "10.12.79.57:8300",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "tcp transport",
"tcpMaxPool": 3,
"tcpTimeout": 10000000000,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "loading local db",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "local DB successfully loaded",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "schema manager loaded",
"n": 0,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"metadata_only_voters": false,
"msg": "construct a new raft node",
"name": "dev-csp-ecs-weaviate-service.dev-csp-sddns",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "19-19-1727082094548",
"last-index": 19,
"last-term": 19,
"level": "info",
"msg": "starting restore from snapshot",
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "restoring schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "successfully restored schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "reload local db: update schema ...",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"last_applied_index": 19,
"last_snapshot_index": 19,
"last_store_log_applied_index": 19,
"level": "info",
"msg": "successfully reloaded indexes from snapshot",
"n": 0,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "19-19-1727082094548",
"last-index": 19,
"last-term": 19,
"level": "info",
"msg": "snapshot restore progress",
"percent-complete": "[100.00]%",
"read-bytes": 106,
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "19-19-1727082094548",
"last-index": 19,
"last-term": 19,
"level": "info",
"msg": "restored from snapshot",
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"index": 1,
"level": "info",
"msg": "raft initial configuration",
"servers": "[[{Suffrage:Voter ID:dev-csp-ecs-weaviate-service.dev-csp-sddns Address:10.12.118.10:8301}]]",
"time": "2024-09-23T09:07:34Z"
}
{
"action": "raft_cluster_recovery",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"existed_single_cluster_node": {
"Suffrage": 0,
"ID": "dev-csp-ecs-weaviate-service.dev-csp-sddns",
"Address": "10.12.118.10:8301"
},
"level": "info",
"msg": "perform cluster recovery",
"new_single_cluster_node": {
"Suffrage": 0,
"ID": "dev-csp-ecs-weaviate-service.dev-csp-sddns",
"Address": "10.12.79.57:8301"
},
"time": "2024-09-23T09:07:34Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"follower": {},
"leader-address": "",
"leader-id": "",
"level": "info",
"msg": "raft entering follower state",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "restoring schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "successfully restored schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "reload local db: update schema ...",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"last_applied_index": 19,
"last_snapshot_index": 19,
"last_store_log_applied_index": 19,
"level": "info",
"msg": "successfully reloaded indexes from snapshot",
"n": 0,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "19-19-1727082094548",
"last-index": 19,
"last-term": 19,
"level": "info",
"msg": "snapshot restore progress",
"percent-complete": "[100.00]%",
"read-bytes": 106,
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "persisting snapshot",
"time": "2024-09-23T09:07:34Z"
}
2024-09-23T09:07:34.800Z [INFO] snapshot: creating new snapshot: path=/var/lib/weaviate/raft/snapshots/20-20-1727082454800.tmp
2024-09-23T09:07:34.860Z [INFO] snapshot: reaping snapshot: path=/var/lib/weaviate/raft/snapshots/17-17-1727081368609
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "20-20-1727082454800",
"last-index": 20,
"last-term": 20,
"level": "info",
"msg": "starting restore from snapshot",
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "restoring schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "successfully restored schema from snapshot",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "reload local db: update schema ...",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"last_applied_index": 19,
"last_snapshot_index": 20,
"last_store_log_applied_index": 19,
"level": "info",
"msg": "successfully reloaded indexes from snapshot",
"n": 0,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "20-20-1727082454800",
"last-index": 20,
"last-term": 20,
"level": "info",
"msg": "snapshot restore progress",
"percent-complete": "[100.00]%",
"read-bytes": 106,
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"id": "20-20-1727082454800",
"last-index": 20,
"last-term": 20,
"level": "info",
"msg": "restored from snapshot",
"size-in-bytes": 106,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"index": 1,
"level": "info",
"msg": "raft initial configuration",
"servers": "[[{Suffrage:Voter ID:dev-csp-ecs-weaviate-service.dev-csp-sddns Address:10.12.79.57:8301}]]",
"time": "2024-09-23T09:07:34Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"follower": {},
"leader-address": "",
"leader-id": "",
"level": "info",
"msg": "raft entering follower state",
"time": "2024-09-23T09:07:34Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"last_snapshot_index": 20,
"last_store_applied_index_on_start": 19,
"level": "info",
"msg": "raft node constructed",
"raft_applied_index": 20,
"raft_last_index": 20,
"time": "2024-09-23T09:07:34Z"
}
{
"action": "bootstrap",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"error": "could not join a cluster from [10.12.79.57:8300]",
"level": "warning",
"msg": "failed to join cluster, will notify next if voter",
"servers": [
"10.12.79.57:8300"
],
"time": "2024-09-23T09:07:35Z",
"voter": true
}
{
"action": "bootstrap",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"candidates": [
{
"Suffrage": 0,
"ID": "dev-csp-ecs-weaviate-service.dev-csp-sddns",
"Address": "10.12.79.57:8300"
}
],
"level": "info",
"msg": "starting cluster bootstrapping",
"time": "2024-09-23T09:07:35Z"
}
{
"action": "bootstrap",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "warning",
"msg": "bootstrapping cluster",
"time": "2024-09-23T09:07:35Z",
"warn": "bootstrap only works on new clusters"
}
{
"action": "bootstrap",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "notified peers this node is ready to join as voter",
"servers": [
"10.12.79.57:8300"
],
"time": "2024-09-23T09:07:35Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"last-leader-addr": "",
"last-leader-id": "",
"level": "warning",
"msg": "raft heartbeat timeout reached, starting election",
"time": "2024-09-23T09:07:36Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "raft entering candidate state",
"node": {},
"term": 21,
"time": "2024-09-23T09:07:36Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "raft election won",
"tally": 1,
"term": 21,
"time": "2024-09-23T09:07:36Z"
}
{
"action": "raft",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"leader": {},
"level": "info",
"msg": "raft entering leader state",
"time": "2024-09-23T09:07:36Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"docker_image_tag": "1.26.4",
"level": "info",
"msg": "configured versions",
"server_version": "1.26.4",
"time": "2024-09-23T09:07:36Z"
}
{
"action": "grpc_startup",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "grpc server listening at [::]:50051",
"time": "2024-09-23T09:07:36Z"
}
{
"action": "restapi_management",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"docker_image_tag": "1.26.4",
"level": "info",
"msg": "Serving weaviate at http://[::]:8080",
"time": "2024-09-23T09:07:36Z"
}
{
"address": "10.12.79.57:8300",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "current Leader",
"time": "2024-09-23T09:07:36Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "starting migration from old schema",
"time": "2024-09-23T09:07:36Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "legacy schema is empty, nothing to migrate",
"time": "2024-09-23T09:07:36Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "migration from the old schema has been successfully completed",
"time": "2024-09-23T09:07:36Z"
}
{
"action": "bootstrap",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "node reporting ready, node has probably recovered cluster from raft config. Exiting bootstrap process",
"time": "2024-09-23T09:07:36Z"
}
{
"action": "telemetry_push",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "telemetry started",
"payload": "&{MachineID:cb0a4689-502b-431e-bb35-3a6fee4f1be8 Type:INIT Version:1.26.4 NumObjects:0 OS:linux Arch:amd64 UsedModules:[]}",
"time": "2024-09-23T09:07:37Z"
}
{
"action": "restapi_management",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"docker_image_tag": "1.26.4",
"level": "info",
"msg": "Shutting down... ",
"time": "2024-09-23T09:13:55Z"
}
{
"action": "restapi_management",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"docker_image_tag": "1.26.4",
"level": "info",
"msg": "Stopped serving weaviate at http://[::]:8080",
"time": "2024-09-23T09:13:55Z"
}
{
"action": "telemetry_push",
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "telemetry terminated",
"payload": "&{MachineID:cb0a4689-502b-431e-bb35-3a6fee4f1be8 Type:TERMINATE Version:1.26.4 NumObjects:0 OS:linux Arch:amd64 UsedModules:[]}",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing raft FSM store ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "shutting down raft sub-system ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "transferring leadership to another server",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"error": "cannot find peer",
"level": "error",
"msg": "transferring leadership",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing raft-net ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing log store ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing data store ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing loaded database ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing raft-rpc client ...",
"time": "2024-09-23T09:13:56Z"
}
{
"build_git_commit": "584532a",
"build_go_version": "go1.21.13",
"build_image_tag": "1.26.4",
"build_wv_version": "1.26.4",
"level": "info",
"msg": "closing raft-rpc server ...",
"time": "2024-09-23T09:13:56Z"
}