diff --git a/commands/build.ts b/commands/build.ts index 3d9ed1b..22c856f 100644 --- a/commands/build.ts +++ b/commands/build.ts @@ -30,11 +30,16 @@ export const command = new Command() } done(sArt, recipe.artifacts ? `${recipe.artifacts.generator}/${recipe.artifacts.fork}` : "no artifacts"); const sp = step(`rendering ${r}`); - const { name, binaries } = await buildOne(r); + const { name, binaries, binaryBuilds } = await buildOne(r); done(sp, `manifests/${name}/`); - const missing = missingBinaries(binaries); + // Binaries built from source land at `up` time; don't flag them missing here. + const managed = new Set(binaryBuilds); + const missing = missingBinaries(binaries.filter((b) => !managed.has(b))); if (missing.length > 0) { console.log(` ${warn("!")} host binaries not found: ${missing.join(", ")}`); } + if (binaryBuilds.length > 0) { + console.log(` ${warn("!")} binaries built from source on 'up': ${binaryBuilds.length}`); + } } }); diff --git a/commands/test.ts b/commands/test.ts index 8af7d62..e2c598d 100644 --- a/commands/test.ts +++ b/commands/test.ts @@ -7,8 +7,13 @@ import { Command } from "jsr:@cliffy/command@^1.0.0-rc.7"; import { type AddressLike, + concat, + decodeRlp, + encodeRlp, getBytes, + hexlify, keccak256, + sha256, toUtf8String, Transaction, Wallet, @@ -20,9 +25,99 @@ const STATIC_PREFUNDED = [ "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", ]; +export type TxType = "legacy" | "blob"; + const DEFAULT_VALUE_WEI = 100_000_000_000_000_000n; // 0.1 ETH const DEFAULT_GAS_LIMIT = 21000n; const DEFAULT_GAS_PRICE = 1_000_000_000n; // 1 gwei +// EIP-1559/blob fee fields (blob tx is type 3, so it can't use gasPrice). +const DEFAULT_PRIORITY_FEE = 1_000_000_000n; // 1 gwei +const DEFAULT_MAX_FEE = 10_000_000_000n; // 10 gwei — covers devnet base fee +const DEFAULT_MAX_BLOB_FEE = 1_000_000_000n; // 1 gwei — well above the 1 wei min + +const BLOB_SIZE = 4096 * 32; // one EIP-4844 blob: 4096 field elements × 32 bytes + +// kzg-wasm, loaded lazily so the legacy path never pays for the WASM init. Its +// trusted setup is network-agnostic, so the bundled mainnet setup is correct on +// any devnet. v1 exposes the EIP-7594 cell-proof ops required post-Osaka. +type KzgApi = { + blobToKZGCommitment: (blob: string) => string; + computeCellsAndProofs: (blob: string) => [string[], string[]]; +}; +let kzgLib: Promise | null = null; +function getKzg(): Promise { + if (!kzgLib) { + kzgLib = import("npm:kzg-wasm@^1.0.0").then((m) => m.loadKZG() as Promise); + } + return kzgLib; +} + +const hx = (s: string): string => (s.startsWith("0x") ? s : `0x${s}`); + +// A single valid blob: a short marker in the first field element (leading byte +// left 0 so the element stays below the BLS modulus), the rest zero-filled. +function makeBlob(marker: string): Uint8Array { + const blob = new Uint8Array(BLOB_SIZE); + blob.set(new TextEncoder().encode(marker).subarray(0, 31), 1); + return blob; +} + +// versioned hash = 0x01 ‖ sha256(commitment)[1:] (VERSIONED_HASH_VERSION_KZG) +function blobVersionedHash(commitment: string): string { + const h = getBytes(sha256(hx(commitment))); + h[0] = 0x01; + return hexlify(h); +} + +// Build a post-Osaka (EIP-7594/PeerDAS) blob tx. ethers only serializes the +// legacy EIP-4844 sidecar, so we let it produce the signed canonical tx +// (0x03 ‖ rlp(body)) and re-wrap it as the cell-proof network form it can't +// emit: 0x03 ‖ rlp([body, wrapper_version=1, blobs, commitments, cell_proofs]) +async function signBlobTx(wallet: Wallet, to: string, nonce: number, cid: bigint): Promise { + const kzg = await getKzg(); + const blob = hexlify(makeBlob("decker blob tx")); + const commitment = hx(kzg.blobToKZGCommitment(blob)); + const cellProofs = kzg.computeCellsAndProofs(blob)[1].map(hx); // 128 proofs per blob + + const tx = new Transaction(); + tx.type = 3; + tx.to = to; + tx.value = DEFAULT_VALUE_WEI; + tx.nonce = nonce; + tx.chainId = cid; + tx.gasLimit = DEFAULT_GAS_LIMIT; + tx.maxPriorityFeePerGas = DEFAULT_PRIORITY_FEE; + tx.maxFeePerGas = DEFAULT_MAX_FEE; + tx.maxFeePerBlobGas = DEFAULT_MAX_BLOB_FEE; + tx.blobVersionedHashes = [blobVersionedHash(commitment)]; + // Sign the instance directly so we keep the canonical (sidecar-free) signed tx. + tx.signature = wallet.signingKey.sign(tx.unsignedHash); + + const body = decodeRlp(hx(tx.serialized.slice(4))); // strip 0x03, decode rlp(body) + const wrapper = encodeRlp([body, "0x01", [blob], [commitment], cellProofs]); + return concat(["0x03", wrapper]); +} + +async function signTx( + type: TxType, + wallet: Wallet, + to: string, + nonce: number, + cid: bigint, +): Promise { + if (type === "blob") return await signBlobTx(wallet, to, nonce, cid); + const tx = Transaction.from({ + type: 0, + to, + value: DEFAULT_VALUE_WEI, + gasLimit: DEFAULT_GAS_LIMIT, + gasPrice: DEFAULT_GAS_PRICE, + nonce, + chainId: cid, + data: "0x", + }); + return await wallet.signTransaction(tx); +} function hhmmss(d: Date): string { const pad = (n: number) => String(n).padStart(2, "0"); @@ -102,6 +197,7 @@ export type TestOpts = { retries: number; // 0 = retry forever insecure: boolean; expectedExtraData?: string; + type: TxType; }; export async function runTest(opts: TestOpts): Promise { @@ -120,19 +216,10 @@ export async function runTest(opts: TestOpts): Promise { const nonce = await pendingNonce(el, wallet.address); console.log(`${dim("Nonce:")} ${accent(String(nonce))}`); - const tx = Transaction.from({ - type: 0, - to: toAddress, - value: DEFAULT_VALUE_WEI, - gasLimit: DEFAULT_GAS_LIMIT, - gasPrice: DEFAULT_GAS_PRICE, - nonce, - chainId: cid, - data: "0x", - }); - const signed = await wallet.signTransaction(tx); + const signed = await signTx(opts.type, wallet, toAddress, nonce, cid); - console.log(`${dim("Sending transaction at")} ${accent(hhmmss(new Date()))}`); + const label = opts.type === "blob" ? "blob transaction" : "transaction"; + console.log(`${dim(`Sending ${label} at`)} ${accent(hhmmss(new Date()))}`); const txHash = await sendRawTx(target, signed); console.log(`${dim("TX Hash:")} ${accent(txHash)}`); @@ -205,7 +292,14 @@ export const command = new Command() .option("--expected-extra-data ", "Verify block extra data matches this string", { default: "", }) + .option("--type ", "Transaction type to send: legacy or blob", { + default: "legacy", + }) .action(async (opts) => { + if (opts.type !== "legacy" && opts.type !== "blob") { + console.error(err(`unknown --type ${JSON.stringify(opts.type)} (expected: legacy, blob)`)); + Deno.exit(2); + } Deno.exit(await runTest({ rpc: opts.rpc, elRpc: opts.elRpc, @@ -213,6 +307,7 @@ export const command = new Command() retries: opts.retries, insecure: opts.insecure, expectedExtraData: opts.expectedExtraData, + type: opts.type, })); }); diff --git a/commands/up.ts b/commands/up.ts index 0ef579f..f3a913f 100644 --- a/commands/up.ts +++ b/commands/up.ts @@ -2,6 +2,7 @@ import { Command } from "jsr:@cliffy/command@^1.0.0-rc.7"; import { isAbsolute, join, toFileUrl } from "jsr:@std/path@^1.0.0"; import { generateArtifacts, loadRecipe, missingBinaries } from "../utils/build.ts"; import { cleanRuntime, emit } from "../utils/emit.ts"; +import { ensureBinaries } from "../utils/binary-build.ts"; import { ensureImages } from "../utils/image-build.ts"; import { DEFAULT_MANIFEST, ensureClone, loadManifest } from "../utils/manifest.ts"; import { dim, done, fail, note, red, rule, step, summary } from "../utils/term.ts"; @@ -118,7 +119,9 @@ export async function upRecipe( paths = emitted.paths; done(sEmit, renderers.map((r) => r.name).join(" + ")); - const missing = missingBinaries(emitted.binaries); + // Binaries built from source (binaryBuilds) are produced below, so they are + // not expected to exist yet — only check the unmanaged ones. + const missing = missingBinaries(emitted.binaries.filter((b) => !emitted.binaryBuilds.has(b))); if (missing.length > 0) { console.error(""); console.error(red("✗ host binaries not found:")); @@ -128,6 +131,22 @@ export async function upRecipe( return { code: 1, renderers, paths }; } + if (emitted.binaryBuilds.size > 0) { + rule("binaries"); + const t = performance.now(); + try { + const built = await ensureBinaries(emitted.binaryBuilds); + const skipped = emitted.binaryBuilds.size - built.length; + const extra = built.length > 0 + ? `built ${built.length}${skipped > 0 ? `, cached ${skipped}` : ""}` + : `cached ${skipped}`; + note("✓", `binaries ready ${dim(`(${extra})`)}`, t); + } catch (e) { + console.error(red(`✗ binary build failed: ${(e as Error).message}`)); + return { code: 1, renderers, paths }; + } + } + if (imageBuilds.size > 0) { rule("images"); const t = performance.now(); diff --git a/containers/rbuilder.ts b/containers/rbuilder.ts index 8f1bcf2..a0e61c7 100644 --- a/containers/rbuilder.ts +++ b/containers/rbuilder.ts @@ -17,7 +17,7 @@ export const ports = { full_telemetry: { port: DEFAULT_HTTP_PORT + 2, protocol: "TCP" as const, service: false }, }; -const rbuilderConfigFor = ( +export const rbuilderConfigFor = ( name: string, chainPath: string, rethDatadir: string, diff --git a/containers/reth-rbuilder.ts b/containers/reth-rbuilder.ts new file mode 100644 index 0000000..95b42ec --- /dev/null +++ b/containers/reth-rbuilder.ts @@ -0,0 +1,91 @@ +import type { BinaryBuildSpec, HostCtx, Ports, ProcessDef, ProcessResult } from "../utils/types.ts"; +import { portNum } from "../utils/types.ts"; +import { binaryBuildPath } from "../utils/binary-build.ts"; +import { rbuilderConfigFor } from "./rbuilder.ts"; + +// reth + rbuilder as a single host binary: vanilla reth runs as the node with +// rbuilder spawned in-process (`reth-rbuilder node --rbuilder.config `). +// Built from the rbuilder reth 2.2 migration fork — decker clones and compiles +// it on first `up`, then caches the binary (see utils/binary-build.ts). Override +// with `binary: "/path/to/reth-rbuilder"` on the process to skip the build. +const BUILD: BinaryBuildSpec = { + repo: "https://github.com/faheelsattar/rbuilder", + ref: "faheel/reth-2.2-migration", + // CMAKE_POLICY_VERSION_MINIMUM=3.5: a transitive C dep (runng-sys → nng) pins + // an ancient cmake_minimum_required that CMake >= 4 rejects; this lets it + // configure anyway. Harmless on older CMake (honored since 3.31). + cmd: "CMAKE_POLICY_VERSION_MINIMUM=3.5 cargo build --release --bin reth-rbuilder", + artifact: "target/release/reth-rbuilder", +}; + +const DEFAULT_HTTP_PORT = 8745; + +export const ports: Ports = { + // reth node + rpc: 8545, + authrpc: 8551, + metrics: 9090, + // rbuilder (telemetry servers: redacted = http+1, full = http+2) + http: { port: DEFAULT_HTTP_PORT, protocol: "TCP", service: false }, + redacted_telemetry: { port: DEFAULT_HTTP_PORT + 1, protocol: "TCP", service: false }, + full_telemetry: { port: DEFAULT_HTTP_PORT + 2, protocol: "TCP", service: false }, +}; + +function refs(def: ProcessDef) { + const beacon = def.refs?.beacon; + const relay = def.refs?.relay; + if (!beacon) throw new Error(`reth-rbuilder ${def.name}: missing refs.beacon`); + if (!relay) throw new Error(`reth-rbuilder ${def.name}: missing refs.relay`); + return { beacon, relay }; +} + +export function buildProcess(def: ProcessDef, ctx: HostCtx): ProcessResult { + const { beacon, relay } = refs(def); + const ps: Ports = { ...ports, ...((def.config?.ports as Ports | undefined) ?? {}) }; + const dataDir = ctx.dataPath(def.name, "data"); + const ipcPath = `${dataDir}/reth.ipc`; + + // rbuilder runs in-process, so it reads reth's state from the same datadir/IPC. + const toml = rbuilderConfigFor( + def.name, + `${ctx.artifactsPath}/genesis.json`, + dataDir, + ipcPath, + "0.0.0.0", + ctx.url(beacon, "http"), + relay, + ctx.url(relay, "http"), + portNum(ps.http), + ); + const tomlPath = ctx.configPath(def.name, "rbuilder.toml"); + + return { + process: { + command: [ + def.binary ?? binaryBuildPath(BUILD), + "node", + "--chain", `${ctx.artifactsPath}/genesis.json`, + "--datadir", dataDir, + "--color", "never", + "--addr", "0.0.0.0", + "--port", "30303", + "--ipcpath", ipcPath, + "--http", + "--http.addr", "0.0.0.0", + "--http.api", "admin,eth,web3,net,rpc,mev,flashbots", + "--http.port", String(portNum(ps.rpc)), + "--authrpc.port", String(portNum(ps.authrpc)), + "--authrpc.addr", "0.0.0.0", + "--authrpc.jwtsecret", `${ctx.artifactsPath}/jwtsecret`, + "--metrics", `0.0.0.0:${portNum(ps.metrics)}`, + "--engine.persistence-threshold", "0", + "--engine.memory-block-buffer-target", "0", + "-vvv", + "--disable-discovery", + "--rbuilder.config", tomlPath, + ], + }, + configs: [{ filename: "rbuilder.toml", content: toml }], + binaryBuild: def.binary ? undefined : BUILD, + }; +} diff --git a/e2e/helpers.ts b/e2e/helpers.ts index 8b1177a..b0f29bb 100644 --- a/e2e/helpers.ts +++ b/e2e/helpers.ts @@ -44,11 +44,29 @@ export async function withTmp(fn: (dir: string) => Promise): Promise } } +// Ref to pin in test manifests. CI isn't always on `main` (feature-branch and +// detached-HEAD PR builds are common), so resolve REPO_ROOT's real state: its +// branch if checked out on one, else the HEAD commit. Both resolve in the clone +// — a branch via origin/, a SHA directly (a local-path clone copies the +// full object store, so even a detached commit is present). +async function repoRef(): Promise { + const git = async (...args: string[]): Promise => { + const { code, stdout } = await new Deno.Command("git", { + args: ["-C", REPO_ROOT, ...args], + stdout: "piped", + stderr: "null", + }).output(); + return code === 0 ? new TextDecoder().decode(stdout).trim() : ""; + }; + return (await git("branch", "--show-current")) || (await git("rev-parse", "HEAD")) || "main"; +} + // A decker.ts manifest pinned at the local checkout, so pull/auto-pull e2e runs // offline and fast instead of hitting GitHub. export async function writeLocalManifest(dir: string, recipe = "l1"): Promise { + const ref = await repoRef(); const body = `type P = { decker: { source: string; ref: string; into?: string }; recipe: string }; -export const project: P = { decker: { source: ${JSON.stringify(REPO_ROOT)}, ref: "main", into: ".decker" }, recipe: ${JSON.stringify(recipe)} }; +export const project: P = { decker: { source: ${JSON.stringify(REPO_ROOT)}, ref: ${JSON.stringify(ref)}, into: ".decker" }, recipe: ${JSON.stringify(recipe)} }; `; await Deno.writeTextFile(join(dir, "decker.ts"), body); } diff --git a/recipes/rbuilder-reth2.ts b/recipes/rbuilder-reth2.ts new file mode 100644 index 0000000..9177b9c --- /dev/null +++ b/recipes/rbuilder-reth2.ts @@ -0,0 +1,57 @@ +import type { Recipe } from "../utils/types.ts"; +import { relayWarmup } from "../scripts/relay-warmup.ts"; + +// reth + rbuilder run as a single host binary (`reth-rbuilder`); everything else +// runs in containers. The binary is built from the rbuilder reth 2.2 migration +// fork and decker clones + compiles it automatically on first `up`: +// https://github.com/faheelsattar/rbuilder/tree/faheel/reth-2.2-migration +// (see containers/reth-rbuilder.ts and utils/binary-build.ts). The first run +// triggers a full cargo build; the binary is then cached under cache/bins/. +export const recipe: Recipe = { + artifacts: { generator: "l1", fork: "fulu" }, + scripts: [ + relayWarmup({ + relays: [{ container: "mev-boost-relay-1" }], + }), + ], + pods: [ + { + name: "beacon-1", + containers: [ + { name: "beacon-1", prototype: "lighthouse-beacon", refs: { el: "el-1", builder: "mev-boost-relay-1" } }, + ], + }, + { + name: "validator-1", + containers: [ + { name: "validator-1", prototype: "lighthouse-validator", refs: { beacon: "beacon-1" } }, + ], + }, + { + name: "mev-boost-relay-1", + containers: [ + { name: "pg-mb-1", prototype: "mev-boost-relay-postgres" }, + { name: "redis-mb-1", prototype: "redis" }, + { + name: "housekeeper-mb-1", + prototype: "mev-boost-housekeeper", + refs: { beacon: "beacon-1", postgres: "pg-mb-1", redis: "redis-mb-1" }, + }, + { + name: "mev-boost-relay-1", + prototype: "mev-boost-relay", + refs: { beacon: "beacon-1", postgres: "pg-mb-1", redis: "redis-mb-1", el: "el-1" }, + }, + ], + }, + ], + processes: [ + // reth node + rbuilder in one process; serves as the EL (el-1) for the + // beacon (authrpc) and the relay's block simulation (rpc). + { + name: "el-1", + prototype: "reth-rbuilder", + refs: { beacon: "beacon-1", relay: "mev-boost-relay-1" }, + }, + ], +}; diff --git a/renderers/process-compose.ts b/renderers/process-compose.ts index 96a36a6..8d78f12 100644 --- a/renderers/process-compose.ts +++ b/renderers/process-compose.ts @@ -1,6 +1,7 @@ import { stringify } from "jsr:@std/yaml@^1.0.5"; import { lookup, makeHostCtx } from "../utils/resolve.ts"; import type { + BinaryBuildSpec, ProcessSpec, Recipe, RenderCtx, @@ -34,13 +35,18 @@ function build(recipe: Recipe, ctx: RenderCtx): RenderResult { const procs: Record = {}; const files: RenderResult["files"] = []; const binaries: string[] = []; + const binaryBuilds = new Map(); for (const def of processes) { const proto = lookup(def.prototype); if (!proto.buildProcess) { throw new Error(`process ${def.name} has no buildProcess()`); } const built = proto.buildProcess(def, hostCtx); - if (built.process.command.length > 0) binaries.push(built.process.command[0]); + const bin = built.process.command[0]; + if (bin) { + binaries.push(bin); + if (built.binaryBuild) binaryBuilds.set(bin, built.binaryBuild); + } procs[def.name] = procEntry(built.process); for (const cf of built.configs ?? []) { files.push({ @@ -55,7 +61,7 @@ function build(recipe: Recipe, ctx: RenderCtx): RenderResult { content: stringify({ version: "0.5", processes: procs }, yamlOpts), }); - return { files, binaries }; + return { files, binaries, binaryBuilds }; } async function start(paths: RendererPaths): Promise { diff --git a/utils/binary-build.ts b/utils/binary-build.ts new file mode 100644 index 0000000..57c2736 --- /dev/null +++ b/utils/binary-build.ts @@ -0,0 +1,97 @@ +import { basename, dirname } from "jsr:@std/path@^1.0.0"; +import type { BinaryBuildSpec } from "./types.ts"; + +import { DECKER_ROOT } from "./root.ts"; +const BIN_CACHE = `${DECKER_ROOT}/cache/bins`; +const SRC_CACHE = `${DECKER_ROOT}/cache/sources`; + +// Stable location of the built binary. Returned with a literal ${DECKER_ROOT} +// placeholder so it can be embedded in rendered manifests (materializeRuntime +// expands it on the way to runtime/); ensureBinaries expands it back to build. +export function binaryBuildPath(spec: BinaryBuildSpec): string { + return `\${DECKER_ROOT}/cache/bins/${relDir(spec)}/${basename(spec.artifact)}`; +} + +function outputPath(spec: BinaryBuildSpec): string { + return `${BIN_CACHE}/${relDir(spec)}/${basename(spec.artifact)}`; +} + +function relDir(spec: BinaryBuildSpec): string { + return `${repoBasename(spec.repo)}-${slug(spec.ref)}`; +} + +function repoBasename(repo: string): string { + const last = repo.replace(/\.git$/, "").replace(/\/$/, "").split("/").pop() ?? repo; + return slug(last); +} + +function slug(s: string): string { + return s.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); +} + +async function exists(path: string): Promise { + try { + await Deno.stat(path); + return true; + } catch { + return false; + } +} + +async function dirExists(path: string): Promise { + try { + return (await Deno.stat(path)).isDirectory; + } catch { + return false; + } +} + +async function run(cmd: string[], opts: { cwd?: string } = {}): Promise { + const proc = await new Deno.Command(cmd[0], { + args: cmd.slice(1), + cwd: opts.cwd, + stdout: "inherit", + stderr: "inherit", + }).output(); + if (proc.code !== 0) throw new Error(`${cmd.join(" ")} exited with code ${proc.code}`); +} + +async function ensureClone(spec: BinaryBuildSpec): Promise { + const cloneDir = `${SRC_CACHE}/${repoBasename(spec.repo)}`; + await Deno.mkdir(SRC_CACHE, { recursive: true }); + if (!(await dirExists(`${cloneDir}/.git`))) { + try { + await Deno.remove(cloneDir, { recursive: true }); + } catch { /* fine */ } + await run(["git", "clone", spec.repo, cloneDir]); + } + await run(["git", "fetch", "origin", spec.ref], { cwd: cloneDir }); + await run(["git", "checkout", spec.ref], { cwd: cloneDir }); + await run(["git", "reset", "--hard", `origin/${spec.ref}`], { cwd: cloneDir }); + return cloneDir; +} + +async function buildOne(spec: BinaryBuildSpec): Promise { + const cloneDir = await ensureClone(spec); + await run(["sh", "-c", spec.cmd], { cwd: cloneDir }); + const artifact = `${cloneDir}/${spec.artifact}`; + if (!(await exists(artifact))) { + throw new Error(`build for ${basename(spec.artifact)} ran but ${spec.artifact} is missing`); + } + const out = outputPath(spec); + await Deno.mkdir(dirname(out), { recursive: true }); + await Deno.copyFile(artifact, out); + await Deno.chmod(out, 0o755); +} + +export async function ensureBinaries( + specs: Map, +): Promise { + const built: string[] = []; + for (const spec of specs.values()) { + if (await exists(outputPath(spec))) continue; + await buildOne(spec); + built.push(outputPath(spec)); + } + return built; +} diff --git a/utils/build.ts b/utils/build.ts index f040bdb..347e89f 100644 --- a/utils/build.ts +++ b/utils/build.ts @@ -50,10 +50,12 @@ export async function generateArtifacts(recipe: Recipe): Promise { await mod.generate({ outDir: out, ...spec }); } -export async function buildOne(target: string): Promise<{ name: string; binaries: string[] }> { +export async function buildOne( + target: string, +): Promise<{ name: string; binaries: string[]; binaryBuilds: string[] }> { const { name, recipe } = await loadRecipe(target); - const { binaries } = await emit(name, recipe); - return { name, binaries }; + const { binaries, binaryBuilds } = await emit(name, recipe); + return { name, binaries, binaryBuilds: [...binaryBuilds.keys()] }; } export function missingBinaries(binaries: string[]): string[] { diff --git a/utils/emit.ts b/utils/emit.ts index 7e19633..8250080 100644 --- a/utils/emit.ts +++ b/utils/emit.ts @@ -1,6 +1,6 @@ import { dirname, isAbsolute } from "jsr:@std/path@^1.0.0"; import { rendererFor } from "./renderers.ts"; -import type { ImageBuildSpec, Recipe, Renderer, RendererPaths } from "./types.ts"; +import type { BinaryBuildSpec, ImageBuildSpec, Recipe, Renderer, RendererPaths } from "./types.ts"; import { DECKER_ROOT } from "./root.ts"; const RUNTIME_DIR = `${DECKER_ROOT}/runtime`; @@ -30,6 +30,7 @@ function validate(recipe: Recipe) { export type EmitResult = { binaries: string[]; imageBuilds: Map; + binaryBuilds: Map; selected: Renderer[]; paths: RendererPaths; }; @@ -58,6 +59,7 @@ export async function emit( await Deno.mkdir(manifestDir, { recursive: true }); const imageBuilds = new Map(); + const binaryBuilds = new Map(); const binaries: string[] = []; for (const r of selected) { const out = r.render(recipe, ctx); @@ -78,11 +80,26 @@ export async function emit( } } } + if (out.binaryBuilds) { + for (const [path, spec] of out.binaryBuilds) { + const existing = binaryBuilds.get(path); + if (existing) { + if ( + existing.repo !== spec.repo || existing.ref !== spec.ref || + existing.cmd !== spec.cmd || existing.artifact !== spec.artifact + ) { + throw new Error(`binary ${path} produced by conflicting BinaryBuildSpec`); + } + } else { + binaryBuilds.set(path, spec); + } + } + } if (out.binaries) binaries.push(...out.binaries); } await materializeRuntime(manifestDir, runtimeDir); - return { binaries, imageBuilds, selected, paths: { runtimeDir, manifestDir } }; + return { binaries, imageBuilds, binaryBuilds, selected, paths: { runtimeDir, manifestDir } }; } // Callers run this before generating artifacts, so the dir is empty when diff --git a/utils/types.ts b/utils/types.ts index ffee8ad..a37bb87 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -37,6 +37,16 @@ export type ImageBuildSpec = { cmd: string; }; +// A host binary built from a git source, the process-side analogue of +// ImageBuildSpec. `cmd` runs in the clone root; `artifact` is the built +// binary's path within the clone (e.g. "target/release/reth-rbuilder"). +export type BinaryBuildSpec = { + repo: string; + ref: string; + cmd: string; + artifact: string; +}; + export type ImageEngine = "podman" | "docker"; export type Container = { @@ -76,6 +86,9 @@ export type ProcessSpec = { export type ProcessResult = { process: ProcessSpec; configs?: { filename: string; content: string }[]; + // Build this process's binary from source. When set, command[0] must be the + // path that binaryBuildPath(binaryBuild) resolves to. + binaryBuild?: BinaryBuildSpec; }; // A ContainerDef is a recipe-level instance of a prototype placed under a Pod. @@ -163,6 +176,7 @@ export type RenderedFile = { export type RenderResult = { files: RenderedFile[]; imageBuilds?: Map; + binaryBuilds?: Map; binaries?: string[]; };