Hello!

We're struggling getting our installation upgraded. We're using a Docker Swarm with GlusterFS handling the volume mount. We were able to go from 10.1 to 10.6.5 successfully following these steps, but going to 11 fails.

We edited our docker-compose.yml file to source a new image:

 cat docker-compose.yml
version: "3.3"
services:
  frontend:
    #image: openproject/community:8.3.1
    #image: openproject/community:9.0.0
    #image: openproject/community:10.1
    #image: openproject/community:10.6.5
    image: openproject/community:11.2.4
    environment:
      - SECRET_KEY_BASE=***********************************
      - EMAIL_DELIVERY_METHOD=smtp
      - SMTP_ENABLE_STARTTLS_AUTO="true"
      - SMTP_ADDRESS=********************
      - SMTP_PORT=587
      - SMTP_DOMAIN=********************
    volumes:
      - /mnt/gv0/volumes/open-project/postgres:/var/openproject/pgdata
      - /mnt/gv0/volumes/open-project/logs:/var/log/supervisor
      - /mnt/gv0/volumes/open-project/static:/var/openproject/assets
    networks:
      - traefik-public
    deploy:
      replicas: 1
      labels:
        - "traefik.http.routers.openproject.rule=Host(`project.allpointsbroadband.net`)"
        - "traefik.http.services.openproject.loadbalancer.server.port=80"
        - "traefik.http.routers.openproject.service=openproject"
        - "traefik.http.routers.openproject.tls.certresolver=letsencrypt-dns"
        - "traefik.http.routers.openproject.tls.domains[0].main=*.allpointsbroadband.net"
        - "traefik.http.routers.openproject.tls.domains[0].sans=allpointsbroadband.net"
        - "traefik.http.routers.openproject.entrypoints=websecure"

networks:
  traefik-public:
    external: true

Then we run docker stack deploy -c docker-compose.yml open-project  which should rebuild and redeploy the containers

The log file shows this, and the container exits. Any ideas?

-----> Starting the all-in-one OpenProject setup at /app/docker/prod/supervisord...
-----> Database cluster already exists, not modifying.
      waiting for server to start..................... done
      server started
      Trying to contact PostgreSQL server instance or waiting for it to come online.
      Starting memcached: memcached.
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/railtie.rb:152:in `block (3 levels) in <class:Railtie>': Ignoring /app/db/schema_cache.yml because it has expired. The current schema version is 20201005120137, but the one in the cache is 20210219092709. (StructuredWarnings::StandardWarning)
rake aborted!
StandardError: An error has occurred, this and all later migrations canceled:

Caused by:
PG::InternalError: ERROR:  unexpected data beyond EOF in block 36 of relation base/16385/329763
HINT:  This has been seen to occur with buggy kernels; consider updating your system.
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:47:in `exec'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:47:in `block (2 levels) in execute'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/dependencies/interlock.rb:48:in `block in permit_concurrent_loads'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/share_lock.rb:187:in `yield_shares'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/dependencies/interlock.rb:47:in `permit_concurrent_loads'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:46:in `block in execute'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract_adapter.rb:696:in `block (2 levels) in log'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:26:in `block (2 levels) in synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract_adapter.rb:695:in `block in log'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/notifications/instrumenter.rb:24:in `instrument'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract_adapter.rb:687:in `log'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/postgresql/database_statements.rb:45:in `execute'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:929:in `block in method_missing'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:897:in `block in say_with_time'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:897:in `say_with_time'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:918:in `method_missing'
/app/db/migrate/20201005184411_rename_timestamps.rb:46:in `block (2 levels) in add_timestamp_column'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:752:in `up'
/app/db/migrate/20201005184411_rename_timestamps.rb:45:in `block in add_timestamp_column'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:788:in `block in reversible'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1016:in `execute_block'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:788:in `reversible'
/app/db/migrate/20201005184411_rename_timestamps.rb:44:in `add_timestamp_column'
/app/db/migrate/20201005184411_rename_timestamps.rb:27:in `change'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:867:in `exec_migration'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:851:in `block (2 levels) in migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:850:in `block in migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/connection_pool.rb:462:in `with_connection'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:849:in `migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1037:in `migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1329:in `block in execute_migration_in_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1380:in `block in ddl_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/database_statements.rb:320:in `block in transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/transaction.rb:310:in `block in within_new_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:26:in `block (2 levels) in synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `handle_interrupt'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:25:in `block in synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `handle_interrupt'
/app/vendor/bundle/ruby/2.7.0/gems/activesupport-6.1.3.2/lib/active_support/concurrency/load_interlock_aware_monitor.rb:21:in `synchronize'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/transaction.rb:308:in `within_new_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/database_statements.rb:320:in `transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/transactions.rb:209:in `transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1380:in `ddl_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1328:in `execute_migration_in_transaction'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1302:in `each'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1302:in `migrate_without_lock'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1251:in `block in migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1401:in `block in with_advisory_lock'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1416:in `block in with_advisory_lock_connection'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/connection_adapters/abstract/connection_pool.rb:462:in `with_connection'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1416:in `with_advisory_lock_connection'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1397:in `with_advisory_lock'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1251:in `migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1086:in `up'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/migration.rb:1061:in `migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/tasks/database_tasks.rb:237:in `migrate'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/railties/databases.rake:92:in `block (3 levels) in <top (required)>'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/railties/databases.rake:90:in `each'
/app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.1.3.2/lib/active_record/railties/databases.rake:90:in `block (2 levels) in <top (required)>'
/app/vendor/bundle/ruby/2.7.0/gems/rake-13.0.3/exe/rake:27:in `<top (required)>'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli/exec.rb:63:in `load'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli/exec.rb:63:in `kernel_load'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli/exec.rb:28:in `run'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli.rb:476:in `exec'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/vendor/thor/lib/thor/invocation.rb:127:in `invoke_command'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/vendor/thor/lib/thor.rb:399:in `dispatch'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli.rb:30:in `dispatch'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/vendor/thor/lib/thor/base.rb:476:in `start'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/cli.rb:24:in `start'
/usr/local/bundle/gems/bundler-2.1.4/exe/bundle:46:in `block in <top (required)>'
/usr/local/bundle/gems/bundler-2.1.4/lib/bundler/friendly_errors.rb:123:in `with_friendly_errors'
/usr/local/bundle/gems/bundler-2.1.4/exe/bundle:34:in `<top (required)>'
/usr/local/bundle/bin/bundle:23:in `load'
/usr/local/bundle/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
      I, [2021-05-19T16:32:34.894765 #76]  INFO -- : Migrating to RenameTimestamps (20201005184411)
      == 20201005184411 RenameTimestamps: migrating =================================
      -- rename_column(:comments, :created_on, :created_at)
         -> 0.0177s
      -- change_column_default(:comments, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398ea46d0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0231s
      -- rename_column(:comments, :updated_on, :updated_at)
         -> 0.0031s
      -- change_column_default(:comments, :updated_at, {:from=>nil, :to=>#<Proc:0x0000562397692ba0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0021s
      -- rename_column(:messages, :created_on, :created_at)
         -> 0.0104s
      -- change_column_default(:messages, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398e9a9f0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0031s
      -- rename_column(:messages, :updated_on, :updated_at)
         -> 0.0052s
      -- change_column_default(:messages, :updated_at, {:from=>nil, :to=>#<Proc:0x0000562398ecebd8 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0026s
      -- rename_column(:versions, :created_on, :created_at)
         -> 0.0083s
      -- change_column_default(:versions, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398ed8a70 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0032s
      -- rename_column(:versions, :updated_on, :updated_at)
         -> 0.0025s
      -- change_column_default(:versions, :updated_at, {:from=>nil, :to=>#<Proc:0x00005623976b2fe0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0026s
      -- rename_column(:users, :created_on, :created_at)
         -> 0.0093s
      -- change_column_default(:users, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398eee2f8 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0041s
      -- rename_column(:users, :updated_on, :updated_at)
         -> 0.0041s
      -- change_column_default(:users, :updated_at, {:from=>nil, :to=>#<Proc:0x0000562398efc600 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0036s
      -- rename_column(:wiki_pages, :created_on, :created_at)
         -> 0.0052s
      -- change_column_default(:wiki_pages, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398f178b0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0024s
      -- rename_column(:wiki_redirects, :created_on, :created_at)
         -> 0.0025s
      -- change_column_default(:wiki_redirects, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398f1eac0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0020s
      -- rename_column(:tokens, :created_on, :created_at)
         -> 0.0027s
      -- change_column_default(:tokens, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398f2b9a0 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0020s
      -- rename_column(:settings, :updated_on, :updated_at)
         -> 0.0019s
      -- change_column_default(:settings, :updated_at, {:from=>nil, :to=>#<Proc:0x0000562398f33448 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0035s
      -- rename_column(:cost_queries, :created_on, :created_at)
         -> 0.0020s
      -- change_column_default(:cost_queries, :created_at, {:from=>nil, :to=>#<Proc:0x0000562398f30518 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0034s
      -- rename_column(:cost_queries, :updated_on, :updated_at)
         -> 0.0015s
      -- change_column_default(:cost_queries, :updated_at, {:from=>nil, :to=>#<Proc:0x0000562398f39168 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0022s
      -- rename_column(:wiki_contents, :updated_on, :updated_at)
         -> 0.0045s
      -- change_column_default(:wiki_contents, :updated_at, {:from=>nil, :to=>#<Proc:0x000056239761df80 /app/db/migrate/20201005184411_rename_timestamps.rb:38 (lambda)>})
         -> 0.0032s
      -- add_column(:journals, :updated_at, :timestamp, {:default=>#<Proc:0x0000562398f49900 /app/db/migrate/20201005184411_rename_timestamps.rb:42 (lambda)>})
         -> 0.7121s
      -- execute("UPDATE\n  journals\nSET updated_at = created_at\n")