Manual server deployment issue

Updated to the latest server but can now no longer login on the frontend with a 502 bad gateway error.

Docker logs look like this

speckle-server_1      | node:internal/modules/cjs/loader:1080
speckle-server_1      |   throw err;
speckle-server_1      |   ^
speckle-server_1      |
speckle-server_1      | Error: Cannot find module '/speckle-server/packages/server/bash'
speckle-server_1      |     at Module._resolveFilename (node:internal/modules/cjs/loader:1077:15)
speckle-server_1      |     at Module._load (node:internal/modules/cjs/loader:922:27)
speckle-server_1      |     at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12)
speckle-server_1      |     at node:internal/main/run_main_module:23:47 {
speckle-server_1      |   code: 'MODULE_NOT_FOUND',
speckle-server_1      |   requireStack: []
speckle-server_1      | }
speckle-server_1      |
speckle-server_1      | Node.js v18.17.1
webhook-service_1     | {"level":"info","time":"2023-10-09T09:19:09.414Z","component":"webhook-service","msg":"Starting Webhook Service..."}
webhook-service_1     | {"level":"error","time":"2023-10-09T09:19:09.550Z","component":"webhook-service","err":{"type":"Error","message":"connect ECONNREFUSED 172.18.0.6:5432","stack":"Error: connect ECONNREFUSED 172.18.0.6:5432\n    at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1494:16)","errno":-111,"code":"ECONNREFUSED","syscall":"connect","address":"172.18.0.6","port":5432},"msg":"Error executing task"}

And the docker compose configuration I am using looks like this

version: "2"
services:
  ####
  # Speckle Server dependencies
  #######
  postgres:
    image: "postgres:13.1-alpine"
    restart: always
    environment:
      POSTGRES_DB: speckle
      POSTGRES_USER: speckle
      POSTGRES_PASSWORD: speckle
    volumes:
      - ./postgres-data:/var/lib/postgresql/data/
    ports:
      - "127.0.0.1:5432:5432"

  redis:
    image: "redis:6.0-alpine"
    restart: always
    volumes:
      - ./redis-data:/data
    ports:
      - "127.0.0.1:6379:6379"

  minio:
    image: "minio/minio"
    command: server /data --console-address ":9001"
    restart: always
    volumes:
      - ./minio-data:/data
    ports:
      - "127.0.0.1:9000:9000"
      - "127.0.0.1:9001:9001"

  ####
  # Speckle Server
  #######
  speckle-frontend:
    image: speckle/speckle-frontend:2
    restart: always
    ports:
      - "0.0.0.0:80:8080"
    environment:
      FILE_SIZE_LIMIT_MB: 100

  speckle-server:
    image: speckle/speckle-server:2
    restart: always
    command: ["bash", "-c", "/wait && node bin/www"]
    environment:
      # TODO: Change this to the URL of the speckle server, as accessed from the network
      CANONICAL_URL: "https://speckle-test.ramboll.com"

      # TODO: Change this to a unique secret for this server
      SESSION_SECRET: "SECRETKEYHERE"

      STRATEGY_LOCAL: "true"
      DEBUG: "speckle:*"

      POSTGRES_URL: "postgres"
      POSTGRES_USER: "speckle"
      POSTGRES_PASSWORD: "speckle"
      POSTGRES_DB: "speckle"

      REDIS_URL: "redis://redis"
      
      S3_ENDPOINT: "http://minio:9000"
      S3_ACCESS_KEY: "minioadmin"
      S3_SECRET_KEY: "minioadmin"
      S3_BUCKET: "speckle-server"
      S3_CREATE_BUCKET: "true"

      FILE_SIZE_LIMIT_MB: 100
      
      WAIT_HOSTS: postgres:5432, redis:6379, minio:9000
      EMAIL: "true"
      EMAIL_HOST: "smtp.sendgrid.net"
      EMAIL_PORT: "587"
      EMAIL_USERNAME: "SECRETUNAME"
      EMAIL_PASSWORD: "SECRETPASSWORD"
      EMAIL_FROM: "XXXXXXXXX@ramboll.com"
      DEFAULT_FROM_EMAIL: "XXXXXXXXX@ramboll.com"

  preview-service:
    image: speckle/speckle-preview-service:2
    restart: always
    mem_limit: "1000m"
    memswap_limit: "1000m"
    command: ["bash", "-c", "/wait && node bin/www"]
    environment:
      DEBUG: "preview-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

  webhook-service:
    image: speckle/speckle-webhook-service:2
    restart: always
    command: ["bash", "-c", "/wait && node main.js"]
    environment:
      DEBUG: "webhook-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

  fileimport-service:
    image: speckle/speckle-fileimport-service:2
    restart: always
    command: ["bash", "-c", "/wait && node src/daemon.js"]
    environment:
      DEBUG: "fileimport-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

      S3_ENDPOINT: "http://minio:9000"
      S3_ACCESS_KEY: "minioadmin"
      S3_SECRET_KEY: "minioadmin"
      S3_BUCKET: "speckle-server"

      SPECKLE_SERVER_URL: "http://speckle-server:3000"
1 Like

Hi @imanwarsame - your docker compose file is out of date and should be updated to match the latest documentation: Deploying a Server - manual setup | Speckle Docs

Specifically, the command for the speckle-server container is out of date.

Iain

Hi @iainsproat, thanks for getting back to me. I’m getting an error saying

ERROR: The Compose file './docker-compose.yml' is invalid because:
'name' does not match any of the regexes: '^x-'

You might be seeing this error because you're using the wrong Compose file version. Either specify a supported version (e.g "2.2" or "3.3") and place your service definitions under the `services` key, or omit the `version` key and place your service definitions at the root of the file to use version 1.
For more on the Compose file format versions, see https://docs.docker.com/compose/compose-file/

When I remove the name line, the server is unhealthy

1 Like

Hi @imanwarsame

Can you confirm which version of docker compose you are running? Please paste the output of the command docker compose version here.

The docker compose file version should be 2.3, which requires docker engine 17.06.0+ or greater Compose file versions and upgrading | Docker Docs

If you are still encountering issues, please post your docker compose file here (omitting any sensitive secrets, if any). Please also post the log messages or errors produced by the unhealthy services.

Iain

I now see that the top level name field may not be supported in version 2.3. Try removing the name property, and we can debug the failing service instead.

Hi @iainsproat, my docker compose version is v2.21.0. I’ve removed the name line.
The error is

Creating network "speckle-server" with the default driver
WARNING: Found orphan containers (speckle_reverse-proxy_1) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up.
Creating speckle_postgres_1         ... done
Creating speckle_speckle-frontend_1 ... done
Creating speckle_redis_1            ... done
Creating speckle_minio_1            ... done

ERROR: for speckle-server  Container "02336ea188d3" is unhealthy.
ERROR: Encountered errors while bringing up the project.

My docker-compose file looks like this atm

version: "2.3"

services:
  ####
  # Speckle Server dependencies
  #######
  postgres:
    image: "postgres:14.5-alpine"
    restart: always
    environment:
      POSTGRES_DB: speckle
      POSTGRES_USER: speckle
      POSTGRES_PASSWORD: speckle
    volumes:
      - ./postgres-data:/var/lib/postgresql/data/
    healthcheck:
      # the -U user has to match the POSTGRES_USER value
      test: ["CMD-SHELL", "pg_isready -U speckle"]
      interval: 5s
      timeout: 5s
      retries: 30
    ports:
      - "127.0.0.1:5432:5432"

  redis:
    image: "redis:6.0-alpine"
    restart: always
    volumes:
      - ./redis-data:/data
    healthcheck:
      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
      interval: 5s
      timeout: 5s
      retries: 30
    ports:
      - "127.0.0.1:6379:6379"

  minio:
    image: "minio/minio"
    command: server /data --console-address ":9001"
    restart: always
    volumes:
      - ./minio-data:/data
    healthcheck:
      test: ["CMD-SHELL", "curl -s -o /dev/null http://127.0.0.1:9000/minio/index.html"]
      interval: 5s
      timeout: 30s
      retries: 30
      start_period: 10s
    ports:
      - "127.0.0.1:9000:9000"
      - "127.0.0.1:9001:9001"

  ####
  # Speckle Server
  #######
  speckle-frontend:
    image: speckle/speckle-frontend:2
    restart: always
    ports:
      - "0.0.0.0:80:8080"
    environment:
      FILE_SIZE_LIMIT_MB: 100

  speckle-server:
    image: speckle/speckle-server:2
    restart: always
    healthcheck:
      test: ["CMD", "node", "-e", "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/graphql?query={serverInfo{version}}', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(res.statusCode != 200 || body.toLowerCase().includes('error'));}); }).end(); } catch { process.exit(1); }"]
      interval: 10s
      timeout: 3s
      retries: 30

    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
      minio:
        condition: service_healthy
    environment:
      # TODO: Change this to the URL of the speckle server, as accessed from the network
      CANONICAL_URL: "https://speckle-test.ramboll.com"
      SPECKLE_AUTOMATE_URL: 'https://speckle-test.ramboll.com:3030'

      REDIS_URL: "redis://redis"

      S3_ENDPOINT: "http://minio:9000"
      S3_ACCESS_KEY: "minioadmin"
      S3_SECRET_KEY: "minioadmin"
      S3_BUCKET: "speckle-server"
      S3_CREATE_BUCKET: "true"

      FILE_SIZE_LIMIT_MB: 100

      # TODO: Change this to a unique secret for this server
      SESSION_SECRET: "THISISASECRET"

      STRATEGY_LOCAL: "true"
      DEBUG: "speckle:*"

      POSTGRES_URL: "postgres"
      POSTGRES_USER: "speckle"
      POSTGRES_PASSWORD: "speckle"
      POSTGRES_DB: "speckle"
      ENABLE_MP: 'false'

      WAIT_HOSTS: postgres:5432, redis:6379, minio:9000
      EMAIL: "true"
      EMAIL_HOST: "smtp.sendgrid.net"
      EMAIL_PORT: "587"
      EMAIL_USERNAME: "apikey"
      EMAIL_PASSWORD: "YETANOTHERSECRET"
      EMAIL_FROM: "speckle@ramboll.com"
      DEFAULT_FROM_EMAIL: "speckle@ramboll.com"

  preview-service:
    image: speckle/speckle-preview-service:2
    restart: always
    depends_on:
      speckle-server:
        condition: service_healthy
    mem_limit: "1000m"
    memswap_limit: "1000m"
    environment:
      DEBUG: "preview-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

  webhook-service:
    image: speckle/speckle-webhook-service:2
    restart: always
    depends_on:
      speckle-server:
        condition: service_healthy
    environment:
      DEBUG: "webhook-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

  fileimport-service:
    image: speckle/speckle-fileimport-service:2
    restart: always
    depends_on:
      speckle-server:
        condition: service_healthy
    environment:
      DEBUG: "fileimport-service:*"
      PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle"
      WAIT_HOSTS: postgres:5432

      S3_ENDPOINT: "http://minio:9000"
      S3_ACCESS_KEY: "minioadmin"
      S3_SECRET_KEY: "minioadmin"
      S3_BUCKET: "speckle-server"

      SPECKLE_SERVER_URL: "http://speckle-server:3000"

networks:
  default:
    name: speckle-server

volumes:
  postgres-data:
  redis-data:
  minio-data:
1 Like

I’ve been able to run the docker compose file you provided without any issues.

Could you view the speckle-server container logs and paste them here? It may provide more information that will help us.

Iain

Hi @iainsproat, I tried sudo docker logs speckle-server and got the response Error response from daemon: No such container: speckle-server

Could you provide the output of docker ps -a to show what is running?

CONTAINER ID   IMAGE                        COMMAND                  CREATED             STATUS                          PORTS                                NAMES
02336ea188d3   postgres:14.5-alpine         "docker-entrypoint.s…"   About an hour ago   Restarting (1) 42 seconds ago                                        speckle_postgres_1
a5dc8ea0ca21   minio/minio                  "/usr/bin/docker-ent…"   About an hour ago   Up About an hour (healthy)      127.0.0.1:9000-9001->9000-9001/tcp   speckle_minio_1
3fe4cba56c8d   speckle/speckle-frontend:2   "/opt/bitnami/openre…"   About an hour ago   Up About an hour                8443/tcp, 0.0.0.0:80->8080/tcp       speckle_speckle-frontend_1
038ed4010790   redis:6.0-alpine             "docker-entrypoint.s…"   About an hour ago   Up About an hour (healthy)      127.0.0.1:6379->6379/tcp             speckle_redis_1
60a11e6963b2   traefik:v2.5                 "/entrypoint.sh --pr…"   6 months ago        Exited (0) 6 months ago                                              speckle_reverse-proxy_1

Unfortunately we don’t really have enough information to diagnose the problem. speckle-server is unhealthy for some unknown reason on your machine.

Try removing the healthcheck section of the speckle-server and running docker compose up again. Are there any more error messages when running docker compose up?

Hi @iainsproat, I’ve removed all health checks and dependency checks. Snip of sections of the docker output below. Thank you!

postgres_1            |
postgres_1            | PostgreSQL Database directory appears to contain a database; Skipping initialization
postgres_1            |
postgres_1            | 2023-10-10 11:10:22.331 UTC [1] FATAL:  database files are incompatible with server
postgres_1            | 2023-10-10 11:10:22.331 UTC [1] DETAIL:  The data directory was initialized by PostgreSQL version 13, which is not compatible with this version 14.5.
redis_1               | 1:C 10 Oct 2023 11:10:22.710 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
redis_1               | 1:C 10 Oct 2023 11:10:22.710 # Redis version=6.0.20, bits=64, commit=00000000, modified=0, pid=1, just started
redis_1               | 1:C 10 Oct 2023 11:10:22.711 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
redis_1               | 1:M 10 Oct 2023 11:10:22.713 * Running mode=standalone, port=6379.
redis_1               | 1:M 10 Oct 2023 11:10:22.713 # Server initialized
redis_1               | 1:M 10 Oct 2023 11:10:22.713 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
redis_1               | 1:M 10 Oct 2023 11:10:22.714 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled (set to 'madvise' or 'never').
redis_1               | 1:M 10 Oct 2023 11:10:22.720 * Loading RDB produced by version 6.0.20
redis_1               | 1:M 10 Oct 2023 11:10:22.721 * RDB age 4 seconds
redis_1               | 1:M 10 Oct 2023 11:10:22.721 * RDB memory usage when created 0.77 Mb
redis_1               | 1:M 10 Oct 2023 11:10:22.721 * DB loaded from disk: 0.001 seconds
redis_1               | 1:M 10 Oct 2023 11:10:22.722 * Ready to accept connections
preview-service_1     | {"level":"info","time":"2023-10-10T11:09:06.142Z","component":"preview-service","component":"preview-service/server","msg":"Listening on port 3001"}
preview-service_1     | {"level":"info","time":"2023-10-10T11:09:06.144Z","component":"preview-service","msg":"📸 Started Preview Service"}
preview-service_1     | {"level":"error","time":"2023-10-10T11:09:06.183Z","component":"preview-service","err":{"type":"Error","message":"getaddrinfo ENOTFOUND postgres","stack":"Error: getaddrinfo ENOTFOUND postgres\n    at GetAddrInfoReqWrap.onlookup [as oncomplete] (node:dns:108:26)","errno":-3008,"code":"ENOTFOUND","syscall":"getaddrinfo","hostname":"postgres"},"msg":"Error executing task"}

speckle-frontend_1    | {"time_local":"10/Oct/2023:11:10:41 +0000","remote_addr":"10.8.158.87","remote_user":"","request":"GET /graphql HTTP/1.1","status": "502","body_bytes_sent":"982","request_time":"0.000","http_referrer":"","http_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"}
speckle-frontend_1    | 2023/10/10 11:10:41 [error] 33#33: *2 connect() failed (111: Connection refused) while connecting to upstream, client: 10.8.158.87, server: , request: "GET /graphql HTTP/1.1", upstream: "http://172.22.0.3:3000/graphql", host: "speckle-test.ramboll.com"

webhook-service_1     | {"level":"error","time":"2023-10-10T11:10:43.592Z","component":"webhook-service","err":{"type":"Error","message":"getaddrinfo ENOTFOUND postgres","stack":"Error: getaddrinfo ENOTFOUND postgres\n    at GetAddrInfoReqWrap.onlookup [as oncomplete] (node:dns:107:26)","errno":-3008,"code":"ENOTFOUND","syscall":"getaddrinfo","hostname":"postgres"},"msg":"Error executing task"}
1 Like

This is the root cause of your issue. Due to this, postgres is in a failing state and no Speckle service can connect to the database causing them to fail.

Please make sure that you have followed all the instructions in our postgres upgrade guide. You may need to deploy the previous version in order to complete all necessary instructions before upgrading. Database backup, upgrade, and restore | Speckle Docs

Iain

1 Like

Hi @iainsproat, I’m afraid I’m getting 403 forbidden errors for new users trying to setup and trying to push or pull from streams (either programatically or using grasshopper). The server page is running and I can log in and view streams

Hi @imanwarsame - sorry to hear that you’re continuing to experience errors.

Do the server logs show any error messages which may provide some insight?

When you mentioned ‘users trying to setup’, do you mean when the new user is registering with the server for the first time, or when they are logged in and creating a stream?

Are you able to create streams?

Iain

Hi @iainsproat, seems it is back up and running. We had some IT rules that were causing issues but even after that it was still complaining. Ended up downgrading then upgrading and restarting a bunch of times and then it started working. Not sure why but happy it’s up!

4 Likes