diff --git a/e2e/README.md b/e2e/README.md index 92049b742c..eabbe1560a 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -49,10 +49,10 @@ e2e/stack/up.sh --fresh # destroy volumes first — a genuine cold start e2e/stack/down.sh # stop (--volumes to wipe) ``` -It is trimmed to what the e2e path exercises: postgres, redis, frontend-loader, -auth, rest-api, staff-api, nginx, init. Dropped from PlaceOS/local: core, edge, -triggers, dispatch, source, influx, chronograf, mosquitto, minio and the -loki/grafana profile — roughly half the containers. +It is trimmed to what the e2e path exercises: postgres, elasticsearch, redis, +search-ingest, frontend-loader, auth, rest-api, staff-api, nginx, init. Dropped +from PlaceOS/local: core, edge, triggers, dispatch, source, influx, chronograf, +mosquitto, minio and the loki/grafana profile — roughly half the containers. Two things a cold start taught us that a long-lived stack hides: @@ -178,10 +178,10 @@ at or below it. | A **tenant** for the backend domain | staff-api rejects *every* `/bookings` and `/events` call with "domain does not have a tenant configured" until one exists. | | One **non-admin** user per worker | Permission gating needs a non-admin; parallel mutation needs distinct identities. | -`seed.ts` polls briefly for the authority rather than reading it once: the row -is created by `init start`, which may not have finished on a cold stack. (The -API reads it straight from Postgres since PPT-2644 — the old Elasticsearch -index lag is gone.) +`seed.ts` polls for the authority rather than reading it once: `/domains` is +served from Elasticsearch, so on a cold stack the row exists in Postgres before +the API can see it. Failing fast there is the single most likely way a CI run +breaks, and the error looks nothing like the cause. It bootstraps through the `backoffice` application, which `init` always creates — otherwise registering an OAuth app would require a token that requires an OAuth app. diff --git a/e2e/stack/SELF_HOSTED_RUNNER.md b/e2e/stack/SELF_HOSTED_RUNNER.md index c6896bd474..56a37e57bc 100644 --- a/e2e/stack/SELF_HOSTED_RUNNER.md +++ b/e2e/stack/SELF_HOSTED_RUNNER.md @@ -87,7 +87,8 @@ Verified on this VM, so no extra provisioning is needed: | | | |---|---| -| `vm.max_map_count` | **1048576** — no sysctl step needed | +| `vm.max_map_count` | **1048576** — above Elasticsearch's 262144, so no sysctl step | +| cgroup version | **v2** — the condition that crashes ES 7.17.6; the stack pins 7.17.28 for exactly this reason. Do not downgrade it. | | Published ports | forwarded to the host, so `localhost:9443` works | ### 2. Register the runner @@ -179,6 +180,7 @@ stack-bring-up failure. | `docker compose` not a valid command | The compose plugin is not registered — step 1. | | Hangs on **Bring up the local PlaceOS stack** | `up.sh` prints `compose ps` plus every service's logs on failure; read the step output first. Usually Colima not running (`colima status`) or the registry unreachable. | | `port is already allocated` | Previous run died. The reclaim step handles it; otherwise `./down.sh --volumes`. | +| `elastic-1 is unhealthy` | Give Colima more memory. ES is pinned to 7.17.28 to avoid the cgroup v2 JDK crash — do not downgrade. | | Can't SSH the machine from the LAN | Expected and irrelevant to CI. It sits on a different subnet with inter-VLAN traffic filtered; use Tailscale for admin. | ## Reverting to GitHub-hosted diff --git a/e2e/stack/docker-compose.yml b/e2e/stack/docker-compose.yml index 46d4e4b7ed..611b71004b 100644 --- a/e2e/stack/docker-compose.yml +++ b/e2e/stack/docker-compose.yml @@ -24,6 +24,7 @@ volumes: postgres-data: + elastic-data: redis-data: nginx-data: www: @@ -44,9 +45,18 @@ x-postgresdb-client-env: &postgresdb-client-env PG_DATABASE: placeos PG_DATABASE_URL: postgresql://placeos:development@postgres:5432/placeos +x-elastic-client-env: &elastic-client-env + ELASTIC_HOST: elastic + ELASTIC_PORT: 9200 + ES_HOST: elastic + ES_PORT: 9200 + x-redis-client-env: &redis-client-env REDIS_URL: redis://redis:6379 +x-search-ingest-client-env: &search-ingest-client-env + PLACE_SEARCH_INGEST_URI: http://search-ingest:3000 + x-logging: &std-logging logging: driver: json-file @@ -74,6 +84,79 @@ services: POSTGRES_DB: placeos TZ: ${TZ:-UTC} + elastic: + # STILL HERE ON PURPOSE — do not delete without reading this. + # + # PPT-2644 moved rest-api's search off Elasticsearch and onto Postgres, and + # this service was removed on that basis in 16dc58f4 (merged 2026-08-13). + # That merge broke CI for 11 days, because the stack pins + # `${PLACEOS_TAG:-latest}` and `latest` is the RELEASE channel, not develop: + # it was `placeos-2.2608.1`, cut before PPT-2644 landed. A rest-api without + # PG search and without an `elastic` service falls back to its default host + # and every index route 500s with + # + # Error connecting to '127.0.0.1:9200': Connection refused + # + # which surfaces as the seed's `GET /oauth_apps` failing in `up.sh`. + # + # The rule this violated: THE STACK'S COMPOSITION MUST MATCH THE IMAGE + # CHANNEL IT PINS. Removing this is safe only once the image that + # `${PLACEOS_TAG:-latest}` resolves to actually contains PPT-2644. Check + # before you do, do not infer it from the rest-api repo: + # + # docker pull placeos/rest-api:latest + # docker image inspect placeos/rest-api:latest \ + # --format '{{index .Config.Labels "org.opencontainers.image.version"}}' + # # then, in the rest-api clone: + # git merge-base --is-ancestor 51faa1a + # + # As of 2026-08-24 that is false for `latest` and true for `nightly`. When it + # flips, re-apply 16dc58f4 (it reverts cleanly) rather than redoing the work. + # + # A LATE 7.17 patch, not the 7.17.6 that PlaceOS/local pins. + # + # 7.17.6 bundles a JDK with the cgroup v2 NPE bug: its launcher dies in + # `JvmOptionsParser` -> `DefaultSystemMemoryInfo` with "Cannot invoke + # CgroupInfo.getMountPoint() because anyController is null" before the JVM even + # starts, so no ES_JAVA_OPTS setting can work around it. GitHub runners use + # cgroup v2; Docker Desktop's VM does not, which is why this passes locally and + # failed the first two CI runs. Staying on the 7.x line keeps client + # compatibility with rest-api and search-ingest, which is what PlaceOS/local + # exercises. + image: elasticsearch:${E2E_ELASTIC_VERSION:-7.17.28} + hostname: elastic + restart: always + <<: *std-logging + healthcheck: + test: curl --silent --fail localhost:9200/_cat/health + interval: 10s + start_period: 60s + retries: 20 + volumes: + - elastic-data:/usr/share/elasticsearch/data + # `bootstrap.memory_lock` is OFF deliberately. + # + # PlaceOS/local enables it, but it requires an unlimited `memlock` rlimit or + # Elasticsearch refuses to boot ("memory locking requested ... but memory is + # not locked") — the container exits, and `compose --wait` reports it as + # unhealthy within seconds rather than after its retry budget. Docker Desktop + # grants that rlimit by default, so this only bites in CI: it is what failed + # the first GitHub Actions run. Memory locking is production tuning and buys + # a throwaway test stack nothing. The ulimits below make it work either way. + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + environment: + bootstrap.memory_lock: 'false' + cluster.routing.allocation.disk.threshold_enabled: 'false' + discovery.type: single-node + ES_JAVA_OPTS: -Xms512m -Xmx512m + TZ: ${TZ:-UTC} + redis: # Pinned: keydb is not the system under test, so a moving tag here can only # ever add noise. The PlaceOS services deliberately stay on ${PLACEOS_TAG} @@ -95,6 +178,17 @@ services: environment: TZ: ${TZ:-UTC} + search-ingest: + image: placeos/search-ingest:${PLACEOS_TAG:-latest} + hostname: search-ingest + restart: always + <<: *std-logging + depends_on: + elastic: { condition: service_healthy } + postgres: { condition: service_healthy } + environment: + <<: [*deployment-env, *postgresdb-client-env, *elastic-client-env] + # Populates the shared `www` volume from PlaceOS/www-core — the source of the # platform /login page that the real-login spec drives. frontend-loader: @@ -137,6 +231,7 @@ services: <<: *std-logging depends_on: postgres: { condition: service_healthy } + elastic: { condition: service_healthy } redis: { condition: service_started } env_file: - .secrets/.env.public_key @@ -146,7 +241,9 @@ services: [ *deployment-env, *postgresdb-client-env, + *elastic-client-env, *redis-client-env, + *search-ingest-client-env, ] staff-api: @@ -211,7 +308,7 @@ services: env_file: - .secrets/.env.secret_key environment: - <<: [*deployment-env, *postgresdb-client-env] + <<: [*deployment-env, *postgresdb-client-env, *elastic-client-env] PLACE_DOMAIN: ${E2E_DOMAIN:-localhost:9443} PLACE_APPLICATION: backoffice PLACE_AUTH_HOST: auth:8080 diff --git a/e2e/stack/up.sh b/e2e/stack/up.sh index 8945ccafa1..4af8e58c5c 100755 --- a/e2e/stack/up.sh +++ b/e2e/stack/up.sh @@ -63,9 +63,11 @@ fi # Self-diagnose on failure. # # `compose --wait` reports only "container X is unhealthy" and exits. On a CI -# runner that is the whole of the evidence unless something dumps more — the -# reason is usually in a container log nobody had collected. Print state and -# logs for EVERY service here, so the step output alone explains the failure. +# runner that is the whole of the evidence unless something dumps more, and the +# first GitHub Actions run failed with exactly that one line — the reason +# (Elasticsearch refusing to boot without a memlock rlimit) was in a container log +# nobody had collected. Print state and logs for EVERY service here, so the step +# output alone explains the failure. diagnose() { echo echo "=== compose ps ===" @@ -79,8 +81,8 @@ trap 'rc=$?; [[ $rc -ne 0 ]] && diagnose; exit $rc' ERR step "starting services" # Bounded so a stuck container fails with a clear message rather than hanging # until the job timeout. -dc up -d --wait --wait-timeout 300 postgres redis -dc up -d frontend-loader auth rest-api staff-api nginx +dc up -d --wait --wait-timeout 300 postgres elastic redis +dc up -d search-ingest frontend-loader auth rest-api staff-api nginx step "waiting for the API" for i in $(seq 1 60); do diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index f824ab018d..6eead301f1 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,8 +1,9 @@ /** * Is the local PlaceOS stack actually up? * - * The stack is a PRECONDITION, not something Playwright starts — it is a - * dozen-odd containers, so a `webServer` block would be the wrong tool. Instead fail in about a second with a message that + * The stack is a PRECONDITION, not something Playwright starts — it is ~20 + * containers and Elasticsearch alone wants several GB, so a `webServer` block + * would be the wrong tool. Instead fail in about a second with a message that * says what to do, rather than after 90s of unexplained navigation timeouts. */ import { request as pwRequest } from '@playwright/test'; diff --git a/e2e/support/seed.ts b/e2e/support/seed.ts index 365826fddd..2538f4571e 100644 --- a/e2e/support/seed.ts +++ b/e2e/support/seed.ts @@ -56,12 +56,13 @@ async function json(api: APIRequestContext, path: string, params?: Record deadline) { throw new Error( `no authority visible via ${ENGINE_API}/domains after ${timeoutMs / 1000}s.\n` + - `The row is created by \`init start\` — check that init completed ` + - `successfully (docker compose logs init).`, + `The row is created by \`init start\`, but this endpoint is served from ` + + `Elasticsearch — check that search-ingest is running and has built its ` + + `indices (docker compose logs search-ingest).`, ); } await new Promise((r) => setTimeout(r, 2000)); @@ -102,9 +104,9 @@ async function ensureOAuthApp(api: APIRequestContext, appUrl: string, name: stri }); if (!res.ok()) { const body = await res.text(); - // A concurrent seed run may have created the row between our existence - // check and this POST. A uniqueness rejection means it is there — not - // an error. + // The existence check above reads from Elasticsearch, so a seed re-run + // within a second or two of the first can miss a row that Postgres + // already has. A uniqueness rejection means it is there — not an error. if (alreadyExists(body)) return { uid, redirect_uri, created: false }; throw new Error(`create oauth_app failed: HTTP ${res.status()} ${body}`); }