CCOR-13193 - test against oss server in ci

.env.example

@@ -6,4 +6,7 @@ CONDUCTOR_AUTH_SECRET=""
 CONDUCTOR_MAX_HTTP2_CONNECTIONS=
 
 CONDUCTOR_TLS_INSECURE=
-CONDUCTOR_DISABLE_HTTP2=
+CONDUCTOR_DISABLE_HTTP2=
+
+# Hostname of the httpbin service reachable from the Conductor server (default: httpbin-server)
+HTTPBIN_SERVICE_HOSTNAME=

.github/workflows/pull_request.yml

@@ -7,8 +7,14 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linter:
+    # TEMP: disabled while iterating on integration-tests-oss. Remove this line to re-enable.
+    if: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -33,6 +39,8 @@ jobs:
         run: npm run lint
 
   unit-tests:
+    # TEMP: disabled while iterating on integration-tests-oss. Remove this line to re-enable.
+    if: false
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -79,19 +87,19 @@ jobs:
           name: codecov-unit-node-${{ matrix.node-version }}
           fail_ci_if_error: false
 
-  # Integration tests (v5): lower max-parallel reduces 502/503 from the shared Conductor server
-  # but makes CI slower without eliminating flakes entirely — feel free to experiment.
-  # Sharding (--shard i/N) splits the suite so each job runs ~1/N of tests.
-  integration-tests:
+  # Integration tests (v5 sdkdev): mirrors integration-tests but targets the sdkdev environment.
+  integration-tests-v5-sdkdev:
+    # TEMP: disabled while iterating on integration-tests-oss. Remove this line to re-enable.
+    if: false
     runs-on: ubuntu-latest
     timeout-minutes: 25
     strategy:
       fail-fast: false
-      max-parallel: 2
+      max-parallel: 3
       matrix:
         node-version: [20, 22, 24]
         shard: [1, 2, 3]
-    name: Node.js v${{ matrix.node-version }} - integration v5 (shard ${{ matrix.shard }}/3)
+    name: Node.js v${{ matrix.node-version }} - integration v5 sdkdev (shard ${{ matrix.shard }}/3)
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -111,45 +119,47 @@ jobs:
       - name: Install Dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: npm ci
-      - name: Run integration tests (v5) shard ${{ matrix.shard }}/3
+      - name: Run integration tests (v5 sdkdev) shard ${{ matrix.shard }}/3
         run: npm run test:integration:v5 -- --ci --coverage --runInBand --testTimeout=120000 --shard=${{ matrix.shard }}/3 --reporters=default --reporters=github-actions --reporters=jest-junit
         env:
           ORKES_BACKEND_VERSION: "5"
-          CONDUCTOR_SERVER_URL: ${{ vars.SERVER_URL }}
-          CONDUCTOR_AUTH_KEY: ${{ secrets.AUTH_KEY }}
-          CONDUCTOR_AUTH_SECRET: ${{ secrets.AUTH_SECRET }}
+          CONDUCTOR_SERVER_URL: ${{ vars.SDKDEV_V5_SERVER_URL }}
+          CONDUCTOR_AUTH_KEY: ${{ vars.SDKDEV_V5_AUTH_KEY }}
+          CONDUCTOR_AUTH_SECRET: ${{ secrets.SDKDEV_V5_AUTH_SECRET }}
           CONDUCTOR_REQUEST_TIMEOUT_MS: "120000"
           CONDUCTOR_RETRY_SERVER_ERRORS: "true"
-          JEST_JUNIT_OUTPUT_NAME: integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          HTTPBIN_SERVICE_HOSTNAME: "httpbin"
+          JEST_JUNIT_OUTPUT_NAME: integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
       - name: Publish Test Results
         uses: dorny/test-reporter@v2
         if: ${{ !cancelled() }}
         with:
-          name: integration v5 Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
-          path: reports/integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          name: integration v5 sdkdev Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
+          path: reports/integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
           reporter: jest-junit
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/lcov.info
-          flags: integration-v5
-          name: codecov-integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
+          flags: integration-v5-sdkdev
+          name: codecov-integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
           fail_ci_if_error: false
 
-  # Integration tests (v4): same sharding as v5. v4 fails in CI (passes locally); do not block PRs.
-  integration-tests-v4:
+  # Integration tests (v4 sm): mirrors integration-tests-v4 but targets the sm environment.
+  integration-tests-v4-sm:
+    # TEMP: disabled while iterating on integration-tests-oss. Remove this line to re-enable.
+    if: false
     runs-on: ubuntu-latest
     timeout-minutes: 25
-    continue-on-error: true
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
         node-version: [20, 22, 24]
         shard: [1, 2, 3]
-    name: Node.js v${{ matrix.node-version }} - integration v4 (shard ${{ matrix.shard }}/3)
+    name: Node.js v${{ matrix.node-version }} - integration v4 sm (shard ${{ matrix.shard }}/3)
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -169,29 +179,115 @@ jobs:
       - name: Install Dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: npm ci
-      - name: Run integration tests (v4) shard ${{ matrix.shard }}/3
+      - name: Run integration tests (v4 sm) shard ${{ matrix.shard }}/3
         run: npm run test:integration:v4 -- --ci --coverage --runInBand --testTimeout=120000 --shard=${{ matrix.shard }}/3 --reporters=default --reporters=github-actions --reporters=jest-junit
         env:
           ORKES_BACKEND_VERSION: "4"
-          CONDUCTOR_SERVER_URL: ${{ vars.SERVER_URL_V4 }}
-          CONDUCTOR_AUTH_KEY: ${{ secrets.AUTH_KEY_V4 }}
-          CONDUCTOR_AUTH_SECRET: ${{ secrets.AUTH_SECRET_V4 }}
+          CONDUCTOR_SERVER_URL: ${{ vars.SM_V4_SERVER_URL }}
+          CONDUCTOR_AUTH_KEY: ${{ vars.SM_V4_AUTH_KEY }}
+          CONDUCTOR_AUTH_SECRET: ${{ secrets.SM_V4_AUTH_SECRET }}
           CONDUCTOR_REQUEST_TIMEOUT_MS: "120000"
           CONDUCTOR_RETRY_SERVER_ERRORS: "true"
-          JEST_JUNIT_OUTPUT_NAME: integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          JEST_JUNIT_OUTPUT_NAME: integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
       - name: Publish Test Results
         uses: dorny/test-reporter@v2
         if: ${{ !cancelled() }}
         with:
-          name: integration v4 Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
-          path: reports/integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          name: integration v4 sm Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
+          path: reports/integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
           reporter: jest-junit
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/lcov.info
-          flags: integration-v4
-          name: codecov-integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
+          flags: integration-v4-sm
+          name: codecov-integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
           fail_ci_if_error: false
+
+  # Integration tests (OSS): spins up Conductor OSS + Postgres via an inline
+  # docker-compose and runs the integration suite unauthenticated, with Orkes-only
+  # tests gated out via CONDUCTOR_SERVER_TYPE=oss (see test:integration:oss).
+  integration-tests-oss:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    name: Node.js v22 - integration oss
+    env:
+      CONDUCTOR_SERVER_URL: http://localhost:8080/api
+      CONDUCTOR_SERVER_TYPE: oss
+      CONDUCTOR_REQUEST_TIMEOUT_MS: "120000"
+      CONDUCTOR_RETRY_SERVER_ERRORS: "true"
+      HTTPBIN_SERVICE_HOSTNAME: httpbin
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: "npm"
+      - name: Cache node_modules
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: npm-22-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            npm-22-
+      - name: Install Dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: npm ci
+      - name: Write docker-compose file
+        run: |
+          cat <<'EOF' > docker-compose-oss.yaml
+          services:
+            conductor-server:
+              image: conductoross/conductor:latest
+              environment:
+                - CONFIG_PROP=config-postgres.properties
+              ports:
+                - "8080:8080"
+              healthcheck:
+                test: ["CMD", "curl", "-I", "-XGET", "http://localhost:8080/health"]
+                interval: 10s
+                timeout: 10s
+                retries: 20
+              links:
+                - conductor-postgres:postgresdb
+              depends_on:
+                conductor-postgres:
+                  condition: service_healthy
+            conductor-postgres:
+              image: postgres:16
+              environment:
+                - POSTGRES_USER=conductor
+                - POSTGRES_PASSWORD=conductor
+              healthcheck:
+                test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432'
+                interval: 5s
+                timeout: 5s
+                retries: 12
+            httpbin:
+              image: ghcr.io/orkes-io/test-utils/httpbin:latest
+              expose:
+                - "8081"
+          EOF
+      - name: Start Conductor OSS stack
+        run: docker compose -f docker-compose-oss.yaml up -d
+      - name: Wait for Conductor to be healthy
+        run: timeout 120 bash -c 'until curl -sf http://localhost:8080/health; do sleep 5; done'
+      - name: Run integration tests (OSS)
+        run: npm run test:integration:oss -- --ci --runInBand --testTimeout=120000 --reporters=default --reporters=github-actions --reporters=jest-junit
+        env:
+          JEST_JUNIT_OUTPUT_NAME: integration-oss-test-results.xml
+      - name: Dump Conductor logs
+        if: failure()
+        run: docker compose -f docker-compose-oss.yaml logs conductor-server
+      - name: Publish Test Results
+        uses: dorny/test-reporter@v2
+        if: ${{ !cancelled() }}
+        with:
+          name: integration oss
+          path: reports/integration-oss-test-results.xml
+          reporter: jest-junit

SDK_DEVELOPMENT.md

@@ -416,19 +416,89 @@ npm run test:unit       # Unit tests only
 
 ### Integration tests (E2E)
 
-Against a real Conductor server. Location: `src/integration-tests/`.
+Against a real Orkes Conductor (enterprise) server. Location: `src/integration-tests/`.
 
 ```bash
 # Full suite
-CONDUCTOR_SERVER_URL=http://localhost:8080 \
-CONDUCTOR_AUTH_KEY=key CONDUCTOR_AUTH_SECRET=secret \
-ORKES_BACKEND_VERSION=5 \
-npm run test:integration:orkes-v5
+npm run test:integration:v5
 
 # Single file
-npx jest --force-exit --testPathPatterns="AuthorizationClient"
+npx jest --force-exit --detectOpenHandles --testPathPatterns="AuthorizationClient"
 ```
 
+Server credentials are read from environment variables. You can export them in your
+shell or put them in a `.env` file in the repo root (loaded automatically by
+`jest.config.mjs` via `dotenv`). To use a `.env` file, copy the example:
+
+```bash
+cp .env.example .env
+```
+
+Required variables plus two that CI sets for flake resilience:
+
+```bash
+CONDUCTOR_SERVER_URL="https://your-cluster.orkesconductor.io/api"
+CONDUCTOR_AUTH_KEY="your-key"
+CONDUCTOR_AUTH_SECRET="your-secret"
+CONDUCTOR_REQUEST_TIMEOUT_MS=120000
+CONDUCTOR_RETRY_SERVER_ERRORS=true
+```
+
+### Replicating CI matrix shards locally
+
+CI runs integration tests as a matrix of **3 Node versions (20, 22, 24) x 3 shards**,
+using Jest's built-in `--shard=i/N` flag. Each shard runs ~1/3 of the test files
+deterministically, so shard 1/3 always contains the same files for a given test suite.
+
+To reproduce a specific CI cell locally you need two things: the right Node version
+and the right shard flag.
+
+#### Node version switching with fnm
+
+Install [fnm](https://github.com/Schniz/fnm) (Fast Node Manager) if you don't have it.
+If you currently have Node installed directly via Homebrew, remove it first so the two
+don't conflict:
+
+```bash
+brew remove node                    # remove Homebrew-managed Node if present
+brew install fnm
+eval "$(fnm env --use-on-cd)"      # add this line to ~/.zshrc for persistence
+```
+
+Then install the versions CI uses:
+
+```bash
+fnm install 20 && fnm install 22 && fnm install 24
+```
+
+#### Running a shard
+
+Example: reproduce **"Node.js v22 — integration v5 (shard 1/3)"**:
+
+```bash
+fnm use 22
+node --version                     # confirm v22.x.x
+
+npm run test:integration:v5 -- --runInBand --testTimeout=120000 --shard=1/3
+```
+
+Run all three shards sequentially to cover the full suite:
+
+```bash
+for shard in 1 2 3; do
+  npm run test:integration:v5 -- --runInBand --testTimeout=120000 --shard=$shard/3
+done
+```
+
+Switch back to your default Node when done:
+
+```bash
+fnm use default                    # or: fnm use system
+node --version                     # confirm you're back to your usual version
+```
+
+The same pattern works for v4 — swap `test:integration:v5` with `test:integration:v4`.
+
 ### Integration test template
 
 ```typescript

jest.config.mjs

@@ -10,6 +10,7 @@ export default {
     "!src/**/generated/**",
     "!src/**/spec/**",
     "!src/**/*.test.{ts,tsx}",
+    "!src/integration-tests/**",
     "!src/**/index.ts",
     "!src/**/types.ts",
     "!src/**/*.types.ts",

package.json

@@ -51,6 +51,7 @@
     "test:integration:base": "jest --force-exit --detectOpenHandles --testMatch='**/src/integration-tests/*.test.[jt]s?(x)'",
     "test:integration:v5": "cross-env ORKES_BACKEND_VERSION=5 npm run test:integration:base --",
     "test:integration:v4": "cross-env ORKES_BACKEND_VERSION=4 npm run test:integration:base --",
+    "test:integration:oss": "cross-env ORKES_BACKEND_VERSION=5 CONDUCTOR_SERVER_TYPE=oss npm run test:integration:base --",
     "test:integration:v5:batch": "cross-env ORKES_BACKEND_VERSION=5 node scripts/run-integration-batch.mjs",
     "test:integration:v4:batch": "cross-env ORKES_BACKEND_VERSION=4 node scripts/run-integration-batch.mjs",
     "ci": "npm run lint && npm run test",