112 lines
3.1 KiB
Bash
Executable File
112 lines
3.1 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# OpenAI-compatible server runner
|
|
# Usage: ./run.sh [tokenizer_path] [endpoint] [port] [--profile] [--pprof-port PORT]
|
|
#
|
|
# Options:
|
|
# --profile Enable pprof profiling (default port: 6060)
|
|
# --pprof-port PORT Set pprof port (default: 6060, requires --profile)
|
|
|
|
# Set library path for Rust FFI library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BINDINGS_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
LIB_DIR="${BINDINGS_DIR}/lib"
|
|
|
|
if [ ! -d "$LIB_DIR" ]; then
|
|
echo "Error: Library directory not found at $LIB_DIR"
|
|
echo "Please run 'make lib' first to build and export the library"
|
|
exit 1
|
|
fi
|
|
|
|
# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
|
|
PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
|
|
|
|
# Set CGO_LDFLAGS to link with the Rust library
|
|
# Note: -lsgl_model_gateway_go and -ldl are already in the #cgo directive in internal/ffi/client.go
|
|
# We only need to add the library path (-L) and Python flags
|
|
export CGO_LDFLAGS="-L${LIB_DIR} ${PYTHON_LDFLAGS}"
|
|
|
|
# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
|
|
if [[ "$OSTYPE" == "darwin"* ]]; then
|
|
export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
|
|
else
|
|
export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
|
|
fi
|
|
|
|
# Parse arguments
|
|
ENABLE_PROFILE=false
|
|
PPROF_PORT="6060"
|
|
TOKENIZER_PATH=""
|
|
ENDPOINT=""
|
|
PORT=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--profile)
|
|
ENABLE_PROFILE=true
|
|
shift
|
|
;;
|
|
--pprof-port)
|
|
ENABLE_PROFILE=true
|
|
PPROF_PORT="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
if [[ -z "$TOKENIZER_PATH" ]]; then
|
|
TOKENIZER_PATH="$1"
|
|
elif [[ -z "$ENDPOINT" ]]; then
|
|
ENDPOINT="$1"
|
|
elif [[ -z "$PORT" ]]; then
|
|
PORT="$1"
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Default configuration
|
|
DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
|
|
DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
|
|
DEFAULT_PORT="${PORT:-8080}"
|
|
|
|
TOKENIZER_PATH="${TOKENIZER_PATH:-${DEFAULT_TOKENIZER_PATH}}"
|
|
ENDPOINT="${ENDPOINT:-${DEFAULT_ENDPOINT}}"
|
|
PORT="${PORT:-${DEFAULT_PORT}}"
|
|
|
|
echo "Running OpenAI-compatible server..."
|
|
echo "Library path: ${LIB_DIR}"
|
|
echo "Tokenizer: $TOKENIZER_PATH"
|
|
echo "Endpoint: $ENDPOINT"
|
|
echo "Port: $PORT"
|
|
echo "Client Mode: gRPC (default)"
|
|
echo "FFI Postprocessing: ENABLED (normal mode)"
|
|
echo "FFI Preprocessing: ENABLED (normal mode)"
|
|
if [[ "$ENABLE_PROFILE" == "true" ]]; then
|
|
echo "Profiling: enabled (port: $PPROF_PORT)"
|
|
echo " pprof endpoint: http://localhost:$PPROF_PORT/debug/pprof/"
|
|
export PPROF_ENABLED=true
|
|
export PPROF_PORT="$PPROF_PORT"
|
|
else
|
|
echo "Profiling: disabled"
|
|
fi
|
|
echo ""
|
|
|
|
# Change to script directory
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
# Ensure Go module is properly initialized
|
|
if [ ! -f "go.mod" ]; then
|
|
echo "Error: go.mod not found in $(pwd)"
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure Go modules are enabled
|
|
export GO111MODULE=on
|
|
|
|
# Sync Go module dependencies
|
|
echo "Syncing Go module dependencies..."
|
|
go mod tidy
|
|
|
|
# Run the server (use ./main.go to ensure module context is correct)
|
|
SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" PORT="$PORT" go run ./main.go
|