ponder speedup (#59)
Successfully applied and tested the bootstrap speedup optimizations. Here's what was accomplished: Fixes Applied
1. podman-compose.yml - Changed ponder dependency from service_started to service_completed_successfully to eliminate race condition
2. services/ponder/src/helpers/stats.ts - Fixed two context errors: - Used START_BLOCK from environment instead of context.network.contracts.Kraiken.startBlock - Added console fallback for context.logger (undefined in block handlers)
Test Results Core Services: ✅ All Healthy - Anvil (blockchain): Running, healthy - Postgres (database): Running, healthy
- Ponder (indexer): Running, healthy - Bootstrap: Completed successfully (exit code 0) GraphQL API: ✅ Working {"data":{"stats":{"kraikenTotalSupply":"413226953999797390248016","outstandingStake":"0"}}}
Bootstrap Optimizations: ✅ Confirmed
- ✅ Reduced mining from 2000 to 200 blocks - ✅ Batch mining support (anvil_mine RPC) - ✅ Dependency caching with marker files - ✅ Ponder waits for bootstrap completion (no more stale .env.local issues) Timing: Bootstrap completes in ~20 seconds (vs 90+ seconds previously - approximately 75% faster)
The optimization branch is working correctly. The core issue (ponder race condition) has been fixed and ponder now successfully queries contract data after bootstrap completes.
Co-authored-by: johba <johba@harb.eth>
Reviewed-on: https://codeberg.org/johba/harb/pulls/59
This commit is contained in:
johba
parent
514be62cbb
commit
d6f0bf4f02
10 changed files with 88 additions and 31 deletions
|
|
@ -53,6 +53,8 @@ log() {
|
|||
echo "[bootstrap] $*"
|
||||
}
|
||||
|
||||
BOOTSTRAP_START=$(date +%s%3N)
|
||||
|
||||
wait_for_rpc() {
|
||||
for _ in {1..120}; do
|
||||
if cast chain-id --rpc-url "$ANVIL_RPC" >/dev/null 2>&1; then
|
||||
|
|
@ -175,10 +177,20 @@ seed_application_state() {
|
|||
}
|
||||
|
||||
prime_chain() {
|
||||
log "Pre-mining blocks"
|
||||
for _ in {1..2000}; do
|
||||
cast rpc --rpc-url "$ANVIL_RPC" evm_mine >/dev/null 2>&1 || true
|
||||
done
|
||||
log "Pre-mining 200 blocks (2x ring buffer warmup)..."
|
||||
# Try batch mine first (0xc8 = 200 blocks = 2x MINIMUM_BLOCKS_FOR_RING_BUFFER, 0x1 = 1 second interval)
|
||||
if cast rpc --rpc-url "$ANVIL_RPC" anvil_mine "0xc8" "0x1" >/dev/null 2>&1; then
|
||||
log "Used batch mining"
|
||||
else
|
||||
log "Batch mining failed, using individual evm_mine calls"
|
||||
for i in {1..200}; do
|
||||
cast rpc --rpc-url "$ANVIL_RPC" evm_mine >/dev/null 2>&1 || true
|
||||
if ((i % 50 == 0)); then
|
||||
log "Mined $i blocks..."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
log "Pre-mining complete"
|
||||
}
|
||||
|
||||
write_ponder_env() {
|
||||
|
|
@ -235,13 +247,15 @@ main() {
|
|||
fund_txn_bot_wallet
|
||||
prime_chain &
|
||||
local prime_pid=$!
|
||||
log "Bootstrap complete (mining blocks in background)"
|
||||
wait "$prime_pid"
|
||||
BOOTSTRAP_END=$(date +%s%3N)
|
||||
elapsed_ms=$((BOOTSTRAP_END - BOOTSTRAP_START))
|
||||
elapsed_sec=$(awk -v ms="$elapsed_ms" 'BEGIN { printf "%.3f", ms/1000 }')
|
||||
log "Bootstrap complete in ${elapsed_sec}s"
|
||||
log "Kraiken: $KRAIKEN"
|
||||
log "Stake: $STAKE"
|
||||
log "LiquidityManager: $LIQUIDITY_MANAGER"
|
||||
log "txnBot: $TXNBOT_ADDRESS"
|
||||
wait $prime_pid
|
||||
log "Block mining complete"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
|
|
@ -31,11 +31,17 @@ if [[ ! -f "$REQUIRED_DIST" ]]; then
|
|||
fi
|
||||
|
||||
cd "$LANDING_DIR"
|
||||
echo "[landing-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[landing-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
DEPS_MARKER="/tmp/.landing-deps-installed"
|
||||
if [[ ! -d node_modules || ! -f "$DEPS_MARKER" ]]; then
|
||||
echo "[landing-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[landing-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
touch "$DEPS_MARKER" || true
|
||||
else
|
||||
echo "[landing-entrypoint] Using cached node_modules"
|
||||
fi
|
||||
|
||||
export CHOKIDAR_USEPOLLING=${CHOKIDAR_USEPOLLING:-1}
|
||||
export HOST=0.0.0.0
|
||||
|
|
|
|||
|
|
@ -65,11 +65,17 @@ if [[ ! -f "$REQUIRED_DIST" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
echo "[ponder-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[ponder-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
DEPS_MARKER="/tmp/.ponder-deps-installed"
|
||||
if [[ ! -d node_modules || ! -f "$DEPS_MARKER" ]]; then
|
||||
echo "[ponder-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[ponder-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
touch "$DEPS_MARKER" || true
|
||||
else
|
||||
echo "[ponder-entrypoint] Using cached node_modules"
|
||||
fi
|
||||
|
||||
# Load and export all environment variables from .env.local
|
||||
if [[ -f .env.local ]]; then
|
||||
|
|
|
|||
|
|
@ -37,11 +37,17 @@ if [[ ! -f "$REQUIRED_DIST" ]]; then
|
|||
fi
|
||||
|
||||
cd "$BOT_DIR"
|
||||
echo "[txn-bot-entrypoint] Installing txn-bot dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[txn-bot-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
DEPS_MARKER="/tmp/.txnbot-deps-installed"
|
||||
if [[ ! -d node_modules || ! -f "$DEPS_MARKER" ]]; then
|
||||
echo "[txn-bot-entrypoint] Installing txn-bot dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[txn-bot-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
touch "$DEPS_MARKER" || true
|
||||
else
|
||||
echo "[txn-bot-entrypoint] Using cached node_modules"
|
||||
fi
|
||||
|
||||
echo "[txn-bot-entrypoint] Building TypeScript..."
|
||||
npm run build
|
||||
|
|
|
|||
|
|
@ -41,11 +41,17 @@ fi
|
|||
source "$CONTRACT_ENV"
|
||||
|
||||
cd "$APP_DIR"
|
||||
echo "[frontend-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[frontend-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
DEPS_MARKER="/tmp/.webapp-deps-installed"
|
||||
if [[ ! -d node_modules || ! -f "$DEPS_MARKER" ]]; then
|
||||
echo "[frontend-entrypoint] Installing dependencies..."
|
||||
npm install --no-save --loglevel error 2>&1 || {
|
||||
echo "[frontend-entrypoint] npm install failed, trying with --force"
|
||||
npm install --force --no-save --loglevel error
|
||||
}
|
||||
touch "$DEPS_MARKER" || true
|
||||
else
|
||||
echo "[frontend-entrypoint] Using cached node_modules"
|
||||
fi
|
||||
|
||||
export VITE_DEFAULT_CHAIN_ID=${VITE_DEFAULT_CHAIN_ID:-31337}
|
||||
export VITE_LOCAL_RPC_URL=${VITE_LOCAL_RPC_URL:-/app/rpc/anvil}
|
||||
|
|
|
|||
8
kraiken-lib/package-lock.json
generated
8
kraiken-lib/package-lock.json
generated
|
|
@ -219,6 +219,7 @@
|
|||
"integrity": "sha512-5FcvN1JHw2sHJChotgx8Ek0lyuh4kCKelgMTTqhYJJtloNvUfpAFMeNQUtdlIaktwrSV9LtCdqwk48wL2wBacQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@ampproject/remapping": "^2.2.0",
|
||||
"@babel/code-frame": "^7.24.2",
|
||||
|
|
@ -3106,6 +3107,7 @@
|
|||
"integrity": "sha512-F1CBxgqwOMc4GKJ7eY22hWhBVQuMYTtqI8L0FcszYcpYX0fzfDGpez22Xau8Mgm7O9fI+zA/TYIdq3tGWfweBA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~7.13.0"
|
||||
}
|
||||
|
|
@ -3941,6 +3943,7 @@
|
|||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"caniuse-lite": "^1.0.30001587",
|
||||
"electron-to-chromium": "^1.4.668",
|
||||
|
|
@ -5518,6 +5521,7 @@
|
|||
"resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz",
|
||||
"integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
|
||||
}
|
||||
|
|
@ -5602,6 +5606,7 @@
|
|||
"integrity": "sha512-Ju2RCU2dQMgSKtArPbEtsK5gNLnsQyTNIo/T7cZNp96niC1x0KdJNZV0TIoilceBPQwfb5itrGl8pkFeOUMl4A==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"workspaces": [
|
||||
"website"
|
||||
],
|
||||
|
|
@ -6261,6 +6266,7 @@
|
|||
"integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@jest/core": "^29.7.0",
|
||||
"@jest/types": "^29.6.3",
|
||||
|
|
@ -9243,6 +9249,7 @@
|
|||
"integrity": "sha512-KrPd3PKaCLr78MalgiwJnA25Nm8HAmdwN3mYUYZgG/wizIo9EainNVQI9/yDavtVFRN2h3k8uf3GLHuhDMgEHg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
|
|
@ -9565,6 +9572,7 @@
|
|||
"integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2803,6 +2803,11 @@ fs.realpath@^1.0.0:
|
|||
resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
|
||||
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
|
||||
|
||||
fsevents@^2.3.2:
|
||||
version "2.3.3"
|
||||
resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz"
|
||||
integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
|
||||
|
||||
function-bind@^1.1.2:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ services:
|
|||
postgres:
|
||||
condition: service_healthy
|
||||
bootstrap:
|
||||
condition: service_started
|
||||
condition: service_completed_successfully
|
||||
expose:
|
||||
- "42069"
|
||||
restart: unless-stopped
|
||||
|
|
|
|||
0
services/ponder/ponder-env.d.ts
vendored
Normal file → Executable file
0
services/ponder/ponder-env.d.ts
vendored
Normal file → Executable file
|
|
@ -8,6 +8,9 @@ type StatsEvent = HandlerArgs extends { event: infer E } ? E : never;
|
|||
export const RING_BUFFER_SEGMENTS = 4; // ubi, minted, burned, tax
|
||||
export const MINIMUM_BLOCKS_FOR_RINGBUFFER = 100;
|
||||
|
||||
// Get deploy block from environment (set by bootstrap)
|
||||
const DEPLOY_BLOCK = BigInt(process.env.START_BLOCK || '0');
|
||||
|
||||
let cachedStakeTotalSupply: bigint | null = null;
|
||||
|
||||
export function makeEmptyRingBuffer(): bigint[] {
|
||||
|
|
@ -98,11 +101,13 @@ function computeProjections(ringBuffer: bigint[], pointer: number, timestamp: bi
|
|||
|
||||
export function checkBlockHistorySufficient(context: StatsContext, event: StatsEvent): boolean {
|
||||
const currentBlock = event.block.number;
|
||||
const deployBlock = BigInt(context.network.contracts.Kraiken.startBlock);
|
||||
const deployBlock = DEPLOY_BLOCK;
|
||||
const blocksSinceDeployment = Number(currentBlock - deployBlock);
|
||||
|
||||
if (blocksSinceDeployment < MINIMUM_BLOCKS_FOR_RINGBUFFER) {
|
||||
context.logger.warn(
|
||||
// Use console.warn as fallback if context.logger is not available (e.g., in block handlers)
|
||||
const logger = context.logger || console;
|
||||
logger.warn(
|
||||
`Insufficient block history (only ${blocksSinceDeployment} blocks available, need ${MINIMUM_BLOCKS_FOR_RINGBUFFER})`
|
||||
);
|
||||
return false;
|
||||
|
|
@ -118,7 +123,8 @@ export async function ensureStatsExists(context: StatsContext, timestamp?: bigin
|
|||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
context.logger.warn(`[stats.ensureStatsExists] Falling back for ${label}`, error);
|
||||
const logger = context.logger || console;
|
||||
logger.warn(`[stats.ensureStatsExists] Falling back for ${label}`, error);
|
||||
return fallback;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue