Recipe for rbuilder reth 2.x testing

commands/build.ts

@@ -30,11 +30,16 @@ export const command = new Command()
       }
       done(sArt, recipe.artifacts ? `${recipe.artifacts.generator}/${recipe.artifacts.fork}` : "no artifacts");
       const sp = step(`rendering ${r}`);
-      const { name, binaries } = await buildOne(r);
+      const { name, binaries, binaryBuilds } = await buildOne(r);
       done(sp, `manifests/${name}/`);
-      const missing = missingBinaries(binaries);
+      // Binaries built from source land at `up` time; don't flag them missing here.
+      const managed = new Set(binaryBuilds);
+      const missing = missingBinaries(binaries.filter((b) => !managed.has(b)));
       if (missing.length > 0) {
         console.log(`  ${warn("!")} host binaries not found: ${missing.join(", ")}`);
       }
+      if (binaryBuilds.length > 0) {
+        console.log(`  ${warn("!")} binaries built from source on 'up': ${binaryBuilds.length}`);
+      }
     }
   });

commands/test.ts

@@ -7,8 +7,13 @@
 import { Command } from "jsr:@cliffy/command@^1.0.0-rc.7";
 import {
   type AddressLike,
+  concat,
+  decodeRlp,
+  encodeRlp,
   getBytes,
+  hexlify,
   keccak256,
+  sha256,
   toUtf8String,
   Transaction,
   Wallet,
@@ -20,9 +25,99 @@ const STATIC_PREFUNDED = [
   "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d",
 ];
 
+export type TxType = "legacy" | "blob";
+
 const DEFAULT_VALUE_WEI = 100_000_000_000_000_000n; // 0.1 ETH
 const DEFAULT_GAS_LIMIT = 21000n;
 const DEFAULT_GAS_PRICE = 1_000_000_000n; // 1 gwei
+// EIP-1559/blob fee fields (blob tx is type 3, so it can't use gasPrice).
+const DEFAULT_PRIORITY_FEE = 1_000_000_000n; // 1 gwei
+const DEFAULT_MAX_FEE = 10_000_000_000n; // 10 gwei — covers devnet base fee
+const DEFAULT_MAX_BLOB_FEE = 1_000_000_000n; // 1 gwei — well above the 1 wei min
+
+const BLOB_SIZE = 4096 * 32; // one EIP-4844 blob: 4096 field elements × 32 bytes
+
+// kzg-wasm, loaded lazily so the legacy path never pays for the WASM init. Its
+// trusted setup is network-agnostic, so the bundled mainnet setup is correct on
+// any devnet. v1 exposes the EIP-7594 cell-proof ops required post-Osaka.
+type KzgApi = {
+  blobToKZGCommitment: (blob: string) => string;
+  computeCellsAndProofs: (blob: string) => [string[], string[]];
+};
+let kzgLib: Promise<KzgApi> | null = null;
+function getKzg(): Promise<KzgApi> {
+  if (!kzgLib) {
+    kzgLib = import("npm:kzg-wasm@^1.0.0").then((m) => m.loadKZG() as Promise<KzgApi>);
+  }
+  return kzgLib;
+}
+
+const hx = (s: string): string => (s.startsWith("0x") ? s : `0x${s}`);
+
+// A single valid blob: a short marker in the first field element (leading byte
+// left 0 so the element stays below the BLS modulus), the rest zero-filled.
+function makeBlob(marker: string): Uint8Array {
+  const blob = new Uint8Array(BLOB_SIZE);
+  blob.set(new TextEncoder().encode(marker).subarray(0, 31), 1);
+  return blob;
+}
+
+// versioned hash = 0x01 ‖ sha256(commitment)[1:]  (VERSIONED_HASH_VERSION_KZG)
+function blobVersionedHash(commitment: string): string {
+  const h = getBytes(sha256(hx(commitment)));
+  h[0] = 0x01;
+  return hexlify(h);
+}
+
+// Build a post-Osaka (EIP-7594/PeerDAS) blob tx. ethers only serializes the
+// legacy EIP-4844 sidecar, so we let it produce the signed canonical tx
+// (0x03 ‖ rlp(body)) and re-wrap it as the cell-proof network form it can't
+// emit:  0x03 ‖ rlp([body, wrapper_version=1, blobs, commitments, cell_proofs])
+async function signBlobTx(wallet: Wallet, to: string, nonce: number, cid: bigint): Promise<string> {
+  const kzg = await getKzg();
+  const blob = hexlify(makeBlob("decker blob tx"));
+  const commitment = hx(kzg.blobToKZGCommitment(blob));
+  const cellProofs = kzg.computeCellsAndProofs(blob)[1].map(hx); // 128 proofs per blob
+
+  const tx = new Transaction();
+  tx.type = 3;
+  tx.to = to;
+  tx.value = DEFAULT_VALUE_WEI;
+  tx.nonce = nonce;
+  tx.chainId = cid;
+  tx.gasLimit = DEFAULT_GAS_LIMIT;
+  tx.maxPriorityFeePerGas = DEFAULT_PRIORITY_FEE;
+  tx.maxFeePerGas = DEFAULT_MAX_FEE;
+  tx.maxFeePerBlobGas = DEFAULT_MAX_BLOB_FEE;
+  tx.blobVersionedHashes = [blobVersionedHash(commitment)];
+  // Sign the instance directly so we keep the canonical (sidecar-free) signed tx.
+  tx.signature = wallet.signingKey.sign(tx.unsignedHash);
+
+  const body = decodeRlp(hx(tx.serialized.slice(4))); // strip 0x03, decode rlp(body)
+  const wrapper = encodeRlp([body, "0x01", [blob], [commitment], cellProofs]);
+  return concat(["0x03", wrapper]);
+}
+
+async function signTx(
+  type: TxType,
+  wallet: Wallet,
+  to: string,
+  nonce: number,
+  cid: bigint,
+): Promise<string> {
+  if (type === "blob") return await signBlobTx(wallet, to, nonce, cid);
+  const tx = Transaction.from({
+    type: 0,
+    to,
+    value: DEFAULT_VALUE_WEI,
+    gasLimit: DEFAULT_GAS_LIMIT,
+    gasPrice: DEFAULT_GAS_PRICE,
+    nonce,
+    chainId: cid,
+    data: "0x",
+  });
+  return await wallet.signTransaction(tx);
+}
 
 function hhmmss(d: Date): string {
   const pad = (n: number) => String(n).padStart(2, "0");
@@ -102,6 +197,7 @@ export type TestOpts = {
   retries: number; // 0 = retry forever
   insecure: boolean;
   expectedExtraData?: string;
+  type: TxType;
 };
 
 export async function runTest(opts: TestOpts): Promise<number> {
@@ -120,19 +216,10 @@ export async function runTest(opts: TestOpts): Promise<number> {
   const nonce = await pendingNonce(el, wallet.address);
   console.log(`${dim("Nonce:")}    ${accent(String(nonce))}`);
 
-  const tx = Transaction.from({
-    type: 0,
-    to: toAddress,
-    value: DEFAULT_VALUE_WEI,
-    gasLimit: DEFAULT_GAS_LIMIT,
-    gasPrice: DEFAULT_GAS_PRICE,
-    nonce,
-    chainId: cid,
-    data: "0x",
-  });
-  const signed = await wallet.signTransaction(tx);
+  const signed = await signTx(opts.type, wallet, toAddress, nonce, cid);
 
-  console.log(`${dim("Sending transaction at")} ${accent(hhmmss(new Date()))}`);
+  const label = opts.type === "blob" ? "blob transaction" : "transaction";
+  console.log(`${dim(`Sending ${label} at`)} ${accent(hhmmss(new Date()))}`);
   const txHash = await sendRawTx(target, signed);
   console.log(`${dim("TX Hash:")}  ${accent(txHash)}`);
 
@@ -205,14 +292,22 @@ export const command = new Command()
   .option("--expected-extra-data <s:string>", "Verify block extra data matches this string", {
     default: "",
   })
+  .option("--type <type:string>", "Transaction type to send: legacy or blob", {
+    default: "legacy",
+  })
   .action(async (opts) => {
+    if (opts.type !== "legacy" && opts.type !== "blob") {
+      console.error(err(`unknown --type ${JSON.stringify(opts.type)} (expected: legacy, blob)`));
+      Deno.exit(2);
+    }
     Deno.exit(await runTest({
       rpc: opts.rpc,
       elRpc: opts.elRpc,
       timeoutMs: parseDuration(opts.timeout),
       retries: opts.retries,
       insecure: opts.insecure,
       expectedExtraData: opts.expectedExtraData,
+      type: opts.type,
     }));
   });
 

commands/up.ts

@@ -2,6 +2,7 @@ import { Command } from "jsr:@cliffy/command@^1.0.0-rc.7";
 import { isAbsolute, join, toFileUrl } from "jsr:@std/path@^1.0.0";
 import { generateArtifacts, loadRecipe, missingBinaries } from "../utils/build.ts";
 import { cleanRuntime, emit } from "../utils/emit.ts";
+import { ensureBinaries } from "../utils/binary-build.ts";
 import { ensureImages } from "../utils/image-build.ts";
 import { DEFAULT_MANIFEST, ensureClone, loadManifest } from "../utils/manifest.ts";
 import { dim, done, fail, note, red, rule, step, summary } from "../utils/term.ts";
@@ -118,7 +119,9 @@ export async function upRecipe(
   paths = emitted.paths;
   done(sEmit, renderers.map((r) => r.name).join(" + "));
 
-  const missing = missingBinaries(emitted.binaries);
+  // Binaries built from source (binaryBuilds) are produced below, so they are
+  // not expected to exist yet — only check the unmanaged ones.
+  const missing = missingBinaries(emitted.binaries.filter((b) => !emitted.binaryBuilds.has(b)));
   if (missing.length > 0) {
     console.error("");
     console.error(red("✗ host binaries not found:"));
@@ -128,6 +131,22 @@ export async function upRecipe(
     return { code: 1, renderers, paths };
   }
 
+  if (emitted.binaryBuilds.size > 0) {
+    rule("binaries");
+    const t = performance.now();
+    try {
+      const built = await ensureBinaries(emitted.binaryBuilds);
+      const skipped = emitted.binaryBuilds.size - built.length;
+      const extra = built.length > 0
+        ? `built ${built.length}${skipped > 0 ? `, cached ${skipped}` : ""}`
+        : `cached ${skipped}`;
+      note("✓", `binaries ready ${dim(`(${extra})`)}`, t);
+    } catch (e) {
+      console.error(red(`✗ binary build failed: ${(e as Error).message}`));
+      return { code: 1, renderers, paths };
+    }
+  }
+
   if (imageBuilds.size > 0) {
     rule("images");
     const t = performance.now();

containers/rbuilder.ts

@@ -17,7 +17,7 @@ export const ports = {
   full_telemetry: { port: DEFAULT_HTTP_PORT + 2, protocol: "TCP" as const, service: false },
 };
 
-const rbuilderConfigFor = (
+export const rbuilderConfigFor = (
   name: string,
   chainPath: string,
   rethDatadir: string,

containers/reth-rbuilder.ts

@@ -0,0 +1,91 @@
+import type { BinaryBuildSpec, HostCtx, Ports, ProcessDef, ProcessResult } from "../utils/types.ts";
+import { portNum } from "../utils/types.ts";
+import { binaryBuildPath } from "../utils/binary-build.ts";
+import { rbuilderConfigFor } from "./rbuilder.ts";
+
+// reth + rbuilder as a single host binary: vanilla reth runs as the node with
+// rbuilder spawned in-process (`reth-rbuilder node --rbuilder.config <toml>`).
+// Built from the rbuilder reth 2.2 migration fork — decker clones and compiles
+// it on first `up`, then caches the binary (see utils/binary-build.ts). Override
+// with `binary: "/path/to/reth-rbuilder"` on the process to skip the build.
+const BUILD: BinaryBuildSpec = {
+  repo: "https://github.com/faheelsattar/rbuilder",
+  ref: "faheel/reth-2.2-migration",
+  // CMAKE_POLICY_VERSION_MINIMUM=3.5: a transitive C dep (runng-sys → nng) pins
+  // an ancient cmake_minimum_required that CMake >= 4 rejects; this lets it
+  // configure anyway. Harmless on older CMake (honored since 3.31).
+  cmd: "CMAKE_POLICY_VERSION_MINIMUM=3.5 cargo build --release --bin reth-rbuilder",
+  artifact: "target/release/reth-rbuilder",
+};
+
+const DEFAULT_HTTP_PORT = 8745;
+
+export const ports: Ports = {
+  // reth node
+  rpc: 8545,
+  authrpc: 8551,
+  metrics: 9090,
+  // rbuilder (telemetry servers: redacted = http+1, full = http+2)
+  http:               { port: DEFAULT_HTTP_PORT,     protocol: "TCP", service: false },
+  redacted_telemetry: { port: DEFAULT_HTTP_PORT + 1, protocol: "TCP", service: false },
+  full_telemetry:     { port: DEFAULT_HTTP_PORT + 2, protocol: "TCP", service: false },
+};
+
+function refs(def: ProcessDef) {
+  const beacon = def.refs?.beacon;
+  const relay = def.refs?.relay;
+  if (!beacon) throw new Error(`reth-rbuilder ${def.name}: missing refs.beacon`);
+  if (!relay) throw new Error(`reth-rbuilder ${def.name}: missing refs.relay`);
+  return { beacon, relay };
+}
+
+export function buildProcess(def: ProcessDef, ctx: HostCtx): ProcessResult {
+  const { beacon, relay } = refs(def);
+  const ps: Ports = { ...ports, ...((def.config?.ports as Ports | undefined) ?? {}) };
+  const dataDir = ctx.dataPath(def.name, "data");
+  const ipcPath = `${dataDir}/reth.ipc`;
+
+  // rbuilder runs in-process, so it reads reth's state from the same datadir/IPC.
+  const toml = rbuilderConfigFor(
+    def.name,
+    `${ctx.artifactsPath}/genesis.json`,
+    dataDir,
+    ipcPath,
+    "0.0.0.0",
+    ctx.url(beacon, "http"),
+    relay,
+    ctx.url(relay, "http"),
+    portNum(ps.http),
+  );
+  const tomlPath = ctx.configPath(def.name, "rbuilder.toml");
+
+  return {
+    process: {
+      command: [
+        def.binary ?? binaryBuildPath(BUILD),
+        "node",
+        "--chain", `${ctx.artifactsPath}/genesis.json`,
+        "--datadir", dataDir,
+        "--color", "never",
+        "--addr", "0.0.0.0",
+        "--port", "30303",
+        "--ipcpath", ipcPath,
+        "--http",
+        "--http.addr", "0.0.0.0",
+        "--http.api", "admin,eth,web3,net,rpc,mev,flashbots",
+        "--http.port", String(portNum(ps.rpc)),
+        "--authrpc.port", String(portNum(ps.authrpc)),
+        "--authrpc.addr", "0.0.0.0",
+        "--authrpc.jwtsecret", `${ctx.artifactsPath}/jwtsecret`,
+        "--metrics", `0.0.0.0:${portNum(ps.metrics)}`,
+        "--engine.persistence-threshold", "0",
+        "--engine.memory-block-buffer-target", "0",
+        "-vvv",
+        "--disable-discovery",
+        "--rbuilder.config", tomlPath,
+      ],
+    },
+    configs: [{ filename: "rbuilder.toml", content: toml }],
+    binaryBuild: def.binary ? undefined : BUILD,
+  };
+}

e2e/helpers.ts

@@ -44,11 +44,29 @@ export async function withTmp(fn: (dir: string) => Promise<void>): Promise<void>
   }
 }
 
+// Ref to pin in test manifests. CI isn't always on `main` (feature-branch and
+// detached-HEAD PR builds are common), so resolve REPO_ROOT's real state: its
+// branch if checked out on one, else the HEAD commit. Both resolve in the clone
+// — a branch via origin/<branch>, a SHA directly (a local-path clone copies the
+// full object store, so even a detached commit is present).
+async function repoRef(): Promise<string> {
+  const git = async (...args: string[]): Promise<string> => {
+    const { code, stdout } = await new Deno.Command("git", {
+      args: ["-C", REPO_ROOT, ...args],
+      stdout: "piped",
+      stderr: "null",
+    }).output();
+    return code === 0 ? new TextDecoder().decode(stdout).trim() : "";
+  };
+  return (await git("branch", "--show-current")) || (await git("rev-parse", "HEAD")) || "main";
+}
+
 // A decker.ts manifest pinned at the local checkout, so pull/auto-pull e2e runs
 // offline and fast instead of hitting GitHub.
 export async function writeLocalManifest(dir: string, recipe = "l1"): Promise<void> {
+  const ref = await repoRef();
   const body = `type P = { decker: { source: string; ref: string; into?: string }; recipe: string };
-export const project: P = { decker: { source: ${JSON.stringify(REPO_ROOT)}, ref: "main", into: ".decker" }, recipe: ${JSON.stringify(recipe)} };
+export const project: P = { decker: { source: ${JSON.stringify(REPO_ROOT)}, ref: ${JSON.stringify(ref)}, into: ".decker" }, recipe: ${JSON.stringify(recipe)} };
 `;
   await Deno.writeTextFile(join(dir, "decker.ts"), body);
 }