docker-compose.stack.yml

# Production-shaped stack overlay (profile: stack), used by scripts/stack/*.sh
# and `make verify-stack`. It adds what the two-cell dev stack lacks and
# production has:
#   - a second Foghorn replica in cell B, so both cells have a leader and a
#     non-leader;
#   - an offchain Livepeer orchestrator+transcoder and gateway per cell, fronted
#     by nginx with the production media_ingest shape and discovered through
#     Quartermaster as a physical gateway (scripts/stack/seed.sql);
#   - a webhook receiver, Mailpit as SMTP, and the in-network scenario runner.
# Run it through scripts/stack/common.sh, which layers a generated overlay that
# resets every container_name and host port so several slots can coexist.

# scripts/stack/up.sh sets GO_LIVEPEER_IMAGE (in the slot env file) to the go-livepeer release
# scripts/resolve-golivepeer-release.sh resolves. The fallback only lets commands that create no
# container (down, logs) interpolate this file on a slot that was never brought up.
x-go-livepeer-image: &go-livepeer-image ${GO_LIVEPEER_IMAGE:-go-livepeer-unresolved-run-scripts-stack-up}

# Gateway telemetry to Decklog, as the CLI renders it for a production
# livepeer-gateway (cli/cmd/cluster_provision.go): the platform service token,
# Decklog's gRPC address, and the owner tenant of both cells' clusters. Each
# gateway adds its host (gateway ID), region and cluster. The namespace is the
# network observer's stack namespace: the CLI would label an offchain gateway
# chain 0, but here the stack chain (livepeer-chain) stands in for Arbitrum, so
# the gateways publish under its 42161 for Bridge to project their rows.
x-stack-gateway-telemetry: &stack-gateway-telemetry
  SERVICE_TOKEN: ${SERVICE_TOKEN}
  FRAMEWORKS_DECKLOG_GRPC_ADDR: decklog:18006
  FRAMEWORKS_DECKLOG_TLS_MODE: disabled
  FRAMEWORKS_CLUSTER_OWNER_TENANT_ID: 5eed517e-ba5e-da7a-517e-ba5eda7a0001
  FRAMEWORKS_ENVIRONMENT: stack
  FRAMEWORKS_NETWORK_ID: "42161"

x-stack-logging: &stack-logging
  driver: "json-file"
  options:
    max-size: "50m"
    max-file: "3"

# Durable storage: one in-network S3 service (hostname s3) with one bucket per
# cell, so each cell keeps its own immutable backend. Every storage-aware
# service carries the literal tuple rather than interpolated env, so a service
# the runner recreates during fault injection keeps the same backend. Path-style
# presigned URLs (http://s3:8080/<bucket>/...) resolve for edges and the runner.
x-stack-s3-cell-a: &stack-s3-cell-a
  STORAGE_S3_ENDPOINT: http://s3:8080
  STORAGE_S3_BUCKET: fw-stack-central-primary
  STORAGE_S3_REGION: us-east-1
  STORAGE_S3_ACCESS_KEY: frameworks
  STORAGE_S3_SECRET_KEY: frameworks-secret
  STORAGE_S3_PREFIX: ""

x-stack-s3-cell-b: &stack-s3-cell-b
  STORAGE_S3_ENDPOINT: http://s3:8080
  STORAGE_S3_BUCKET: fw-stack-us-primary
  STORAGE_S3_REGION: us-east-1
  STORAGE_S3_ACCESS_KEY: frameworks
  STORAGE_S3_SECRET_KEY: frameworks-secret
  STORAGE_S3_PREFIX: ""

services:
  # Second control-plane replicas, on their own node (central-node-2, from
  # scripts/stack/seed.sql) as production runs them on separate hosts. The
  # network alias puts each behind the same name as the first replica, so every
  # client that dials commodore:19001 or quartermaster:19002 resolves both, the
  # way <service>.internal resolves every instance on the mesh.
  quartermaster-2:
    extends:
      file: docker-compose.yml
      service: quartermaster
    profiles: ["stack"]
    environment:
      QUARTERMASTER_HOST: quartermaster-2
      NODE_ID: central-node-2
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      frameworks:
        aliases: [quartermaster]

  commodore-2:
    extends:
      file: docker-compose.yml
      service: commodore
    profiles: ["stack"]
    environment:
      COMMODORE_HOST: commodore-2
      NODE_ID: central-node-2
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      FROM_EMAIL: stack@frameworks.network
    depends_on:
      postgres:
        condition: service_healthy
      quartermaster:
        condition: service_healthy
    networks:
      frameworks:
        aliases: [commodore]

  # Second Bridge replica. It has no shared alias: scenarios reach it as
  # bridge-2 to hold a WebSocket on each replica, while every other client
  # keeps using bridge.
  bridge-2:
    extends:
      file: docker-compose.yml
      service: bridge
    profiles: ["stack"]
    environment:
      BRIDGE_HOST: bridge-2
      NODE_ID: central-node-2
    depends_on:
      quartermaster:
        condition: service_healthy
      commodore:
        condition: service_started
    networks: [frameworks]

  # Second cell-B replica. Its Quartermaster instance row and cluster
  # assignments come from scripts/stack/seed.sql before it starts.
  foghorn-b-2:
    extends:
      file: docker-compose.yml
      service: foghorn-b
    profiles: ["stack"]
    environment:
      <<: *stack-s3-cell-b
      FOGHORN_RELAY_ADVERTISE_ADDR: foghorn-b-2:18019
      FOGHORN_INSTANCE_ID: foghorn-b-2
      FOGHORN_HOST: foghorn-b-2
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
    depends_on:
      postgres:
        condition: service_healthy
      quartermaster:
        condition: service_healthy
      foghorn-redis-b:
        condition: service_healthy

  # Offchain Livepeer, cell A: orchestrator+transcoder on CPU, gateway with the
  # production auth webhook into the cell's Foghorn, nginx front on the
  # physical infra FQDN Quartermaster advertises. Each orchestrator quotes 1500
  # wei per 1000 pixels without tickets, so the gateway charges its work at
  # that price with zero ticket credit (scenario 35).
  livepeer-orch-a:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -orchestrator
      - -transcoder
      - -network=offchain
      - -pricePerUnit=1500
      - -pixelsPerUnit=1000
      - -serviceAddr=livepeer-orch-a:8935
      - -httpAddr=0.0.0.0:8935
      - -cliAddr=127.0.0.1:7935
      - -orchSecret=stack-orch-secret
      - -v=4
    networks: [frameworks]
    cpus: 2.0
    mem_limit: 2g
    logging: *stack-logging

  livepeer-gateway-a:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -gateway
      - -network=offchain
      - -httpAddr=0.0.0.0:8935
      - -httpIngest
      - -cliAddr=127.0.0.1:7935
      - -rtmpAddr=127.0.0.1:1936
      - -orchAddr=livepeer-orch-a:8935
      - -authWebhookUrl=http://foghorn:18008/webhooks/livepeer/auth
      - -trustedProxyCIDRs=${STACK_SUBNET:-10.231.1.0/24}
      - -v=4
    environment:
      <<: *stack-gateway-telemetry
      FRAMEWORKS_GATEWAY_ID: stack-lp-a
      FRAMEWORKS_GATEWAY_REGION: eu-west
      FRAMEWORKS_CLUSTER_ID: central-primary
    depends_on: [livepeer-orch-a]
    networks: [frameworks]
    mem_limit: 1g
    logging: *stack-logging

  livepeer-proxy-a:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      GATEWAY_FQDN: livepeer-gateway.stack-lp-a.infra.frameworks.network
      GATEWAY_UPSTREAM: livepeer-gateway-a:8935
    volumes:
      - ./scripts/stack/livepeer-proxy.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/a.crt:/etc/frameworks/certs/gateway.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/a.key:/etc/frameworks/certs/gateway.key:ro
    depends_on: [livepeer-gateway-a]
    networks:
      frameworks:
        aliases: [livepeer-gateway.stack-lp-a.infra.frameworks.network]
    logging: *stack-logging

  # Offchain Livepeer, cell B.
  livepeer-orch-b:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -orchestrator
      - -transcoder
      - -network=offchain
      - -pricePerUnit=1500
      - -pixelsPerUnit=1000
      - -serviceAddr=livepeer-orch-b:8935
      # The identity scripts/stack/livepeer-chain.py lists for this
      # orchestrator, so the real network-prober's discovery is attributed.
      - -ethOrchAddr=0x5eed0000000000000000000000000000000000c3
      - -httpAddr=0.0.0.0:8935
      - -cliAddr=127.0.0.1:7935
      - -orchSecret=stack-orch-secret
      - -v=4
    networks: [frameworks]
    cpus: 2.0
    mem_limit: 2g
    logging: *stack-logging

  livepeer-gateway-b:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -gateway
      - -network=offchain
      - -httpAddr=0.0.0.0:8935
      - -httpIngest
      - -cliAddr=127.0.0.1:7935
      - -rtmpAddr=127.0.0.1:1936
      - -orchAddr=livepeer-orch-b:8935
      - -authWebhookUrl=http://foghorn-b:18008/webhooks/livepeer/auth
      - -trustedProxyCIDRs=${STACK_SUBNET:-10.231.1.0/24}
      - -v=4
    environment:
      <<: *stack-gateway-telemetry
      FRAMEWORKS_GATEWAY_ID: stack-lp-b
      FRAMEWORKS_GATEWAY_REGION: us-east
      FRAMEWORKS_CLUSTER_ID: us-primary
    depends_on: [livepeer-orch-b]
    networks: [frameworks]
    mem_limit: 1g
    logging: *stack-logging

  livepeer-proxy-b:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      GATEWAY_FQDN: livepeer-gateway.stack-lp-b.infra.frameworks.network
      GATEWAY_UPSTREAM: livepeer-gateway-b:8935
    volumes:
      - ./scripts/stack/livepeer-proxy.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/b.crt:/etc/frameworks/certs/gateway.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/b.key:/etc/frameworks/certs/gateway.key:ro
    depends_on: [livepeer-gateway-b]
    networks:
      frameworks:
        aliases: [livepeer-gateway.stack-lp-b.infra.frameworks.network]
    logging: *stack-logging

  # Cell A's public control name in front of each of its Foghorn instances,
  # for the edge scenario 30 switches to TLS: the name resolves to both fronts,
  # as a production cell's foghorn.<cluster>.<root> resolves to its instances.
  # See scripts/stack/foghorn-control-tls.conf.template.
  foghorn-tls-a-1:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      FOGHORN_CONTROL_FQDN: foghorn.demo-media.stack.frameworks.network
      FOGHORN_UPSTREAM: foghorn:18029
    volumes:
      - ./scripts/stack/foghorn-control-tls.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.crt:/etc/frameworks/certs/foghorn.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.key:/etc/frameworks/certs/foghorn.key:ro
    networks:
      frameworks:
        aliases: [foghorn.demo-media.stack.frameworks.network]
    logging: *stack-logging

  foghorn-tls-a-2:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      FOGHORN_CONTROL_FQDN: foghorn.demo-media.stack.frameworks.network
      FOGHORN_UPSTREAM: foghorn-2:18029
    volumes:
      - ./scripts/stack/foghorn-control-tls.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.crt:/etc/frameworks/certs/foghorn.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.key:/etc/frameworks/certs/foghorn.key:ro
    networks:
      frameworks:
        aliases: [foghorn.demo-media.stack.frameworks.network]
    logging: *stack-logging

  webhook-receiver:
    image: python:3.13-alpine
    profiles: ["stack"]
    command: ["python3", "/app/webhook-receiver.py"]
    volumes:
      - ./scripts/stack/webhook-receiver.py:/app/webhook-receiver.py:ro
    networks: [frameworks]
    logging: *stack-logging

  # Chaos lets the registration scenario make SMTP fail once to exercise the
  # account-email outbox retry.
  mailpit:
    image: axllent/mailpit:v1.27
    profiles: ["stack"]
    command: ["--enable-chaos"]
    networks: [frameworks]
    logging: *stack-logging

  # Scenarios run here (docker compose exec). The Docker socket is for fault
  # injection into this slot's own project only: scripts/stack/lib.sh always
  # passes -p "$COMPOSE_PROJECT_NAME".
  stack-runner:
    build:
      context: scripts/stack
      dockerfile: runner.Dockerfile
    profiles: ["stack"]
    # Publishers are started from $(...) subshells and orphaned to PID 1; an init
    # reaps them so wait_publisher sees them exit instead of a lingering zombie.
    init: true
    environment:
      STACK_SLOT: ${STACK_SLOT:-1}
      STACK_STATE_DIR: /tmp/fw-stack
      COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-fw-stack-1}
      COMPOSE_PROFILES: ${COMPOSE_PROFILES:-edge,two-cell,stack,stack-s3-ceph,network-observations}
      COMPOSE_FILE: /repo/docker-compose.yml:/repo/docker-compose.stack.yml:/repo/.stack/slot-${STACK_SLOT:-1}/reset.yml
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      SERVICE_TOKEN: ${SERVICE_TOKEN}
      JWT_SECRET: ${JWT_SECRET}
      NETWORK_OBSERVER_SECRET: ${NETWORK_OBSERVER_SECRET}
      GRPC_ALLOW_INSECURE: "true"
      SUPPLIER_NAME: ${SUPPLIER_NAME}
      SUPPLIER_ADDRESS: ${SUPPLIER_ADDRESS}
      SUPPLIER_VAT_NUMBER: ${SUPPLIER_VAT_NUMBER}
      SUPPLIER_REGISTRATION_NUMBER: ${SUPPLIER_REGISTRATION_NUMBER}
      SUPPLIER_COUNTRY: ${SUPPLIER_COUNTRY}
    volumes:
      - ./:/repo:ro
      - /var/run/docker.sock:/var/run/docker.sock
    networks: [frameworks]
    logging: *stack-logging

  # Stack-wide wiring into the existing services. The edge image uses the
  # production edge OS: the Mist tree scripts/stack/mist.Dockerfile builds links
  # against Ubuntu 24.04's C runtime.
  edge:
    image: ${STACK_EDGE_IMAGE:-frameworks-edge:dev}
    build:
      args:
        EDGE_BASE_IMAGE: ubuntu:24.04
    # Every Foghorn instance of the cell, as production renders the node's
    # control address list: a node whose instance goes away fails over to the
    # other one instead of staying without control.
    environment:
      FOGHORN_CONTROL_ADDR: foghorn:18029,foghorn-2:18029
    # The dev edge serves the repo's demo recordings from a bind mount; a stack
    # edge must never write into the source tree, so it gets a slot volume.
    volumes:
      - edge_a_storage:/data/storage
  edge-b:
    image: ${STACK_EDGE_IMAGE:-frameworks-edge:dev}
    environment:
      FOGHORN_CONTROL_ADDR: foghorn-b:18029,foghorn-b-2:18029
  foghorn:
    environment:
      <<: *stack-s3-cell-a
  foghorn-2:
    environment:
      <<: *stack-s3-cell-a
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
      # Mist's balancer source for the edges this replica controls. The dev
      # fallback is <FOGHORN_HOST>:18008, but this replica serves public HTTP
      # on 18018.
      FOGHORN_URL: http://foghorn-2:18018
  foghorn-b:
    environment:
      <<: *stack-s3-cell-b
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
  chandler:
    environment:
      <<: *stack-s3-cell-a
  storage-init:
    environment:
      <<: *stack-s3-cell-a
  # Cell B's descriptor: the same clean, one-time establishment storage-init
  # does for cell A, for the us-primary cell the two-cell seed creates.
  storage-init-b:
    extends:
      file: docker-compose.yml
      service: storage-init
    profiles: ["stack"]
    environment:
      <<: *stack-s3-cell-b
      CLUSTER_ID: us-primary

  # Ceph RGW is the S3 implementation production's provider runs, pinned as in
  # the storage conformance tests; its demo image is amd64-only, so arm64
  # engines get the pinned SeaweedFS instead. scripts/stack/common.sh enables
  # exactly one backend profile. The transparent test proxy answers s3:8080.
  s3-proxy:
    image: python:3.13-alpine
    profiles: ["stack"]
    command: ["python3", "/app/s3-fault-proxy.py"]
    environment:
      STACK_S3_UPSTREAM: s3-${STACK_S3:-ceph}:8080
      SERVICE_TOKEN: ${SERVICE_TOKEN}
    volumes:
      - ./scripts/stack/s3-fault-proxy.py:/app/s3-fault-proxy.py:ro
    networks:
      frameworks:
        aliases: [s3]
    logging: *stack-logging
  s3-ceph:
    image: quay.io/ceph/demo@sha256:522483cf07cfce6386b8e18a3edfa88a1b32c688dee231d0a6962b1513557723
    profiles: ["stack-s3-ceph"]
    environment:
      CEPH_DAEMON: demo
      DEMO_DAEMONS: osd rgw
      MON_IP: 127.0.0.1
      CEPH_PUBLIC_NETWORK: 0.0.0.0/0
      RGW_NAME: localhost
      CEPH_DEMO_UID: frameworks
      CEPH_DEMO_ACCESS_KEY: frameworks
      CEPH_DEMO_SECRET_KEY: frameworks-secret
    networks: [frameworks]
    logging: *stack-logging
  s3-seaweed:
    image: chrislusf/seaweedfs@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
    profiles: ["stack-s3-seaweed"]
    # weed server's volume server defaults to 8080; move it so S3 answers there.
    command: ["server", "-s3", "-dir=/data", "-s3.port=8080", "-volume.port=18081"]
    networks: [frameworks]
    logging: *stack-logging
  commodore:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      FROM_EMAIL: stack@frameworks.network
  purser:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
  lookout:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
  bosun:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      BOSUN_ALLOW_PRIVATE_DESTINATIONS: "true"

  # Livepeer network observations: the coordinator (profile
  # network-observations, which scripts/stack/common.sh enables) reads its
  # orchestrator catalog from the stack chain instead of Arbitrum, and Bridge
  # publishes the demo tenant's observations under the stack namespace.
  livepeer-chain:
    image: python:3.13-alpine
    profiles: ["stack"]
    command: ["python3", "/app/livepeer-chain.py"]
    volumes:
      - ./scripts/stack/livepeer-chain.py:/app/livepeer-chain.py:ro
    networks: [frameworks]
    healthcheck:
      test:
        [
          "CMD",
          "python3",
          "-c",
          "import urllib.request; urllib.request.urlopen('http://localhost:8545/orchestrators', timeout=3)",
        ]
      interval: 5s
      timeout: 5s
      retries: 6
    logging: *stack-logging
  network-observer:
    environment:
      LIVEPEER_ETH_RPC_URL: http://livepeer-chain:8545
      LIVEPEER_CONTROLLER: "0x5eed00000000000000000000000000000000c0de"
      NETWORK_OBSERVER_ENVIRONMENT: stack
    depends_on:
      livepeer-chain:
        condition: service_healthy
  # The real prober probes the stack's own orchestrators, which sit on the slot's
  # private subnet. Its ID sorts after scenario 35's protocol prober
  # (stack-prober-eu), so the coordinator, which assigns an endpoint without
  # known geography to the lowest-ID healthy prober, hands this one every
  # endpoint except while the scenario's prober is polling.
  network-prober:
    environment:
      NETWORK_PROBER_ID: stack-prober-real
      NETWORK_PROBER_REGION: stack-us
      NETWORK_PROBER_ALLOWED_PRIVATE_CIDRS: ${STACK_SUBNET:-10.231.1.0/24}
    depends_on:
      network-observer:
        condition: service_healthy
  bridge:
    environment:
      NETWORK_OBSERVER_URL: http://network-observer:18024
      NETWORK_OBSERVER_SECRET: ${NETWORK_OBSERVER_SECRET}
      NETWORK_PUBLIC_TENANT_IDS: 5eed517e-ba5e-da7a-517e-ba5eda7a0001
      NETWORK_OBSERVER_ENVIRONMENT: stack

networks:
  frameworks:
    ipam:
      config:
        - subnet: ${STACK_SUBNET:-10.231.1.0/24}

volumes:
  edge_a_storage: