30 lines
934 B
Bash
Executable File
30 lines
934 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT=${REPO_ROOT:-/ssd/workspace/yi/laoyao_2b_moe}
|
|
CHECKPOINT=${CHECKPOINT:-${REPO_ROOT}/runs/pretrain_8192_8gpu_dp8_mbs14_full_recompute_weighted_heldoutval_resume10000/checkpoints}
|
|
SERVER_PORT=${SERVER_PORT:-5000}
|
|
|
|
cd "${REPO_ROOT}"
|
|
|
|
echo "== checkpoint =="
|
|
test -f "${CHECKPOINT}/latest_checkpointed_iteration.txt"
|
|
cat "${CHECKPOINT}/latest_checkpointed_iteration.txt"
|
|
ls -ld "${CHECKPOINT}"/iter_* | tail -5
|
|
|
|
echo
|
|
echo "== port ${SERVER_PORT} =="
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -ltnp | grep ":${SERVER_PORT} " || true
|
|
else
|
|
netstat -ltnp 2>/dev/null | grep ":${SERVER_PORT} " || true
|
|
fi
|
|
|
|
echo
|
|
echo "== gpu =="
|
|
nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu,power.draw --format=csv,noheader,nounits
|
|
|
|
echo
|
|
echo "== existing containers =="
|
|
docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}' | grep -E 'laoyao|sglang|vllm|NAME' || true
|