#!/bin/bash set -e # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Get script directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$SCRIPT_DIR" # Build directories CPP_DIR="$PROJECT_ROOT/internal/binding/cpp" BUILD_DIR="$CPP_DIR/cmake-build-release" RAGFLOW_SERVER_BINARY="$PROJECT_ROOT/bin/ragflow_server" RAGFLOW_CLI_BINARY="$PROJECT_ROOT/bin/ragflow-cli" # Strip symbols from Go binaries (set via --strip / -s) STRIP_SYMBOLS="" # Native static library settings. These are the user-cache paths (~/ragflow-native-libs/). # If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority # and skips the network (download_deps.py) fallback. SYSTEM_DEPS="/opt/ragflow-native-libs" # office_oxide native library settings — static linking OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide" OFFICE_OXIDE_VERSION="0.1.12" # pdfium native library settings — static linking (kognitos/pdfium-static) PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static" PDFIUM_STATIC_VERSION="7809" # pdf_oxide native library settings — static linking (go-ffi tarball) PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide" PDF_OXIDE_VERSION="0.3.73" # onnxruntime native library settings — static linking for the in-process # (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary # (--undefined=OrtGetApiBase + --dynamic-list, no --whole-archive); OrtGetApiBase # is then resolved via dlopen(NULL) (the process-global symbol table, not the # executable's own path), so no libonnxruntime.so is needed at runtime. # Downloaded by ragflow_deps/download_deps.py into onnxruntime/static_lib. ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib" # Copy a dependency from the system pre-seed directory to the user cache. # Returns 0 if the dep was copied or already exists in cache, 1 otherwise. _seed_from_system() { local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide" local dep_dir="${HOME}/ragflow-native-libs/${dep_name}" local sys_dir="${SYSTEM_DEPS}/${dep_name}" echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}" if [ -d "$dep_dir" ]; then echo " ${dep_name} → ${dep_dir} (user cache)" return 0 # already cached fi if [ -d "$sys_dir" ]; then echo " ${dep_name} → ${sys_dir} (system)" mkdir -p "$(dirname "$dep_dir")" cp -r "$sys_dir" "$dep_dir" return 0 fi echo " ${dep_name} not found in system or user cache" return 1 } # ── cl100k_base BPE table ───────────────────────────────────────────── # internal/tokenizer loads this table from disk only (no network) and PANICS in # NumTokensFromString / TrimContentToTokenLimit when it is absent: a deployment # that forgot the file must not silently zero every token count (see # internal/tokenizer/bpe_loader.go and failfast_test.go). CI provisions it in a # workflow step before running the Go tests; build.sh does the same so the # documented local test commands are self-contained. Order mirrors CI: system # pre-seed first, then the network. A failure here is deliberately NOT fatal — # the loader stays the final gate, and its panic message names the file. CL100K_TABLE_URL="https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" CL100K_TABLE_RELPATH="ragflow_deps/cl100k_base.tiktoken" SYSTEM_DEPS_TOKENIZER="/opt/ragflow_deps" # Provision the cl100k_base BPE table into ragflow_deps/ (no-op if present). # Returns 0 when the table is in place, 1 when it could not be provisioned. _ensure_cl100k_table() { local dest="${PROJECT_ROOT}/${CL100K_TABLE_RELPATH}" local sys_copy="${SYSTEM_DEPS_TOKENIZER}/cl100k_base.tiktoken" echo "check if the cl100k_base BPE table exists at ${dest}" if [ -s "$dest" ]; then echo " cl100k_base.tiktoken → ${dest} (already present)" return 0 fi if [ -s "$sys_copy" ]; then mkdir -p "$(dirname "$dest")" cp "$sys_copy" "$dest" echo " cl100k_base.tiktoken → ${dest} (system)" return 0 fi # Download to a temp file and move it into place only after a successful # transfer: an interrupted download would otherwise leave a truncated table # behind, which the loader rejects on digest. local tmp="${dest}.download" mkdir -p "$(dirname "$dest")" if command -v curl >/dev/null 2>&1; then if curl -fsSL -o "$tmp" "$CL100K_TABLE_URL"; then mv "$tmp" "$dest" echo " cl100k_base.tiktoken → ${dest} (downloaded)" return 0 fi elif command -v wget >/dev/null 2>&1; then if wget -q -O "$tmp" "$CL100K_TABLE_URL"; then mv "$tmp" "$dest" echo " cl100k_base.tiktoken → ${dest} (downloaded)" return 0 fi fi rm -f "$tmp" echo -e " ${YELLOW}cl100k_base BPE table NOT provisioned${NC} (offline, or the network is blocked)" echo " internal/tokenizer panics on a missing table by design, so every test that" echo " counts tokens will fail loudly. Recover with any of:" echo " - re-run this command (a transient network error usually clears);" echo " - curl -fsSL -o ${CL100K_TABLE_RELPATH} ${CL100K_TABLE_URL}" echo " - or point TIKTOKEN_CACHE_DIR at a directory holding the table." return 1 } echo -e "${GREEN}=== RAGFlow Go Server Build Script ===${NC}" # Function to print section headers print_section() { echo -e "\n${YELLOW}>>> $1${NC}" } # Detect the package-install command for pcre2 development files. # Outputs the command on stdout; empty string if no supported manager is found. detect_pcre2_install_cmd() { if [ "$(uname)" = "Darwin" ]; then echo "brew install pcre2" elif command -v apt-get >/dev/null 2>&1; then echo "sudo apt-get install -y libpcre2-dev" elif command -v zypper >/dev/null 2>&1; then echo "sudo zypper install -y pcre2-devel" elif command -v dnf >/dev/null 2>&1; then echo "sudo dnf install -y pcre2-devel" elif command -v pacman >/dev/null 2>&1; then echo "sudo pacman -S --noconfirm pcre2" else echo "" fi } # Check whether libpcre2-8 is available (static or shared). check_pcre2() { # Prefer pkg-config when available — works across distros. if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libpcre2-8; then return 0 fi # Fall back to known library paths: # Debian/Ubuntu -> /usr/lib/x86_64-linux-gnu # openSUSE/RHEL -> /usr/lib64 # generic Linux -> /usr/lib, /usr/local/lib # macOS Homebrew -> /opt/homebrew/lib (Apple Silicon), /usr/local/lib (Intel) for p in \ /usr/lib/x86_64-linux-gnu/libpcre2-8.a \ /usr/lib/x86_64-linux-gnu/libpcre2-8.so \ /usr/lib64/libpcre2-8.a \ /usr/lib64/libpcre2-8.so \ /usr/lib/libpcre2-8.a \ /usr/lib/libpcre2-8.so \ /usr/local/lib/libpcre2-8.a \ /usr/local/lib/libpcre2-8.so \ /usr/local/lib/libpcre2-8.dylib \ /opt/homebrew/lib/libpcre2-8.a \ /opt/homebrew/lib/libpcre2-8.dylib; do [ -f "$p" ] && return 0 done return 1 } # Check dependencies check_cpp_deps() { print_section "Checking c++ dependencies" command -v cmake >/dev/null 2>&1 || { echo -e "${RED}Error: cmake is required but not installed.${NC}"; exit 1; } command -v clang++ >/dev/null 2>&1 || { echo -e "${RED}Error: clang++ is required but not installed.${NC}"; exit 1; } if check_pcre2; then echo "✓ pcre2 library found" else install_cmd="$(detect_pcre2_install_cmd)" echo -e "${YELLOW}Warning: libpcre2-8 not found. You may need to install it:${NC}" if [ -n "$install_cmd" ]; then echo " $install_cmd" else echo " (No supported package manager detected — install pcre2 development files manually)" fi fi echo "✓ Required tools are available" } check_go_deps() { print_section "Checking go dependencies" command -v go >/dev/null 2>&1 || { echo -e "${RED}Error: go is required but not installed.${NC}"; exit 1; } echo "✓ Required tools are available" check_ort_version_consistency } # Fail fast when the ONNX Runtime native version is declared inconsistently. # The in-process (Go) DeepDoc backend statically links libonnxruntime.a built # from ONE exact ORT release; a mismatch links a wrong/missing .a and only fails # at runtime (dlopen(NULL) can't find OrtGetApiBase). The version is pinned in # three Go-side locations that must all agree. check_ort_version_consistency() { print_section "Checking ONNX Runtime version consistency" local env_go deps dockerfile env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')" deps="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')" if [ -z "$env_go" ] || [ -z "$deps" ] || [ -z "$dockerfile" ]; then echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2 exit 1 fi if [ "$env_go" != "$deps" ] || [ "$env_go" != "$dockerfile" ]; then echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2 printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion" printf ' %-10s %s\n' "$deps" "ragflow_deps/download_deps.py:ORT_VERSION" printf ' %-10s %s\n' "$dockerfile" "dockerfile:ARG ORT_VERSION" exit 1 fi echo -e "${GREEN}✓ ONNX Runtime native version consistent: ${env_go}${NC}" } # Check the ONNX Runtime static archive the Go DeepDoc backend links. # # This mirrors check_office_oxide_deps: it verifies the on-disk archive # actually matches the released artifact, so a stale/incompatible .a — one # re-issued under the SAME release tag and asset name, or an older download — # fails fast HERE with an actionable message, instead of producing a binary # that links fine but crashes at runtime when the shared-initializer feature # (SessionGetInitializer*, added by infiniflow/ragflow-build) runs. # # The onnxruntime_go binding reaches ORT purely through the OrtApi # function-pointer table (onnxruntime_go/shared_initializer.go -> # onnxruntime_wrapper.c -> ort_api->SessionGetInitializer*), so a stale .a # that lacks those custom slots links successfully and only fails at runtime. # The check below catches that by comparing the local release ZIP against the # published {asset}.sha256 sidecar (same source of truth the download scripts # use). Devs who ran a download script have the zip under ragflow_deps/; CI # seeds from /opt and has no zip, where the bake is authoritative. check_onnxruntime_deps() { local ort_version ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \ "${PROJECT_ROOT}/internal/common/environments.go" \ | sed -E 's/.*"([^"]+)".*/\1/')" if [ -z "$ort_version" ]; then echo -e "${RED}Error: cannot parse DeepDocORTVersion from internal/common/environments.go${NC}" >&2 return 1 fi # Locate the ORT static archives under the version dir. Populate the # globals ORT_A_FILES (space-separated .a paths) and ORT_A_SHA256 (sha256 # of the primary libonnxruntime.a) for the caller's version-stamped # cache path. ORT_A_FILES="" local primary="" local f while IFS= read -r f; do case "$(basename "$f")" in *cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;; esac case "$f" in */onnxruntime-linux-x64-static_lib-"${ort_version}"*/lib/*.a) ORT_A_FILES="$ORT_A_FILES $f" [ -z "$primary" ] && primary="$f" case "$(basename "$f")" in libonnxruntime.a) primary="$f" ;; esac ;; esac done < <(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null) if [ -z "$ORT_A_FILES" ]; then local avail avail="$(find "$ONNXRUNTIME_STATIC_PREFIX" -maxdepth 1 -type d \ -name 'onnxruntime-linux-x64-static_lib-*' -exec basename {} \; 2>/dev/null | tr '\n' ' ')" echo -e "${RED}Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX${NC}" >&2 echo " available: ${avail:-}" >&2 echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2 echo " uv run python3 ragflow_deps/download_deps.py" >&2 echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 return 1 fi # Pinned sha256 of the published onnxruntime-v${ort_version} archive. # # Upstream does not re-issue this archive, so we pin the digest here and # verify the LOCAL copy against it — no network access at build time. This # replaces the previous check that fetched the .sha256 sidecar over the # network on every build, which hung the build when the machine was offline # or GitHub was unreachable (curl had no connect timeout). # # A mismatch means the local copy is corrupt or stale; we only WARN (not # fail) so a one-off re-issue cannot break the build. The downloader # ragflow_deps/download_deps.py still verifies against # the published sidecar at download time, so a re-issued archive is picked # up on the next download. local asset="onnxruntime-v${ort_version}-linux-x86_64.zip" local zip_path="${PROJECT_ROOT}/ragflow_deps/${asset}" local expected="439308c93822d26cf04341cab3e77c595bbe88a12b54043210d14a0ca5574bd1" if [ -f "$zip_path" ]; then local actual actual="$(sha256sum "$zip_path" | awk '{print $1}')" if [ "$actual" != "$expected" ]; then echo -e "${YELLOW}Warning: ONNX Runtime archive ${asset} sha256 mismatch${NC}" >&2 echo " expected sha256: $expected" >&2 echo " local sha256: $actual" >&2 echo " The local copy may be corrupt or stale. Refresh it with:" >&2 echo " rm -f ${zip_path}" >&2 echo " uv run python3 ragflow_deps/download_deps.py" >&2 else echo " ✓ ${asset} sha256 matches pinned digest" fi else echo " (no local ${asset}; skipping sha256 check — CI seed assumed authoritative)" fi # Content hash of the primary archive for the version-stamped cache path. ORT_A_SHA256="$(sha256sum "$primary" | awk '{print $1}')" echo "✓ onnxruntime v${ort_version} static lib verified under $ONNXRUNTIME_STATIC_PREFIX" } # Check office_oxide native library check_office_oxide_deps() { print_section "Checking office_oxide native library" _seed_from_system "office_oxide" || true local lib_file="liboffice_oxide.a" local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}" local header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then echo -e "${RED}Error: office_oxide native library not found${NC}" echo " Expected: ${lib_path}" echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or manually download: https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/native-linux-x86_64.tar.gz" exit 1 fi # Verify the on-disk lib matches the pinned version. A stale older lib # (e.g. v0.1.7) silently reintroduces the PPT97 content-loss bug # (github.com/yfedoseev/office_oxide#85, fixed in v0.1.8): PlainText() # on a legacy .ppt returns stale metadata instead of slide text. if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then local found_version found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1) echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}" rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz" if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_deps.py); then echo -e "${RED}Error: office_oxide native lib version mismatch${NC}" echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:" echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz" echo " uv run python3 ragflow_deps/download_deps.py" exit 1 fi lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}" header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then echo -e "${RED}Error: office_oxide native library not found after refresh${NC}" exit 1 fi if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1) echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}" echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}" exit 1 fi fi echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}" return 0 } # Check pdfium static library. check_pdfium_deps() { _seed_from_system "pdfium-static" || true local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" if [ -f "$lib_path" ]; then echo " pdfium (static) → ${PDFIUM_STATIC_PREFIX}" return 0 fi echo " pdfium (static) not found" echo " Expected: ${lib_path}" echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or: curl -fsSL https://github.com/kognitos/pdfium-static/releases/download/chromium%2F${PDFIUM_STATIC_VERSION}/pdfium-linux-x64-static.tgz | tar xz -C ${PDFIUM_STATIC_PREFIX}" return 1 } # Check pdf_oxide static library. # pdf_oxide_validate_version # # Validates the pdf_oxide version embedded in a lib's string constant pool # against the pinned . The marker "pdf_oxide " is # merged into Rust's constant pool and has no fixed length, so a suffixed build # (e.g. 0.3.73.1) and a clean 0.3.73 whose next pooled constant begins with a # digit are byte-identical and cannot be told apart. # # Capture exactly three dotted segments and reject any trailing [0-9.] as # ambiguous, rather than silently truncating it into a (wrong) match. # # Returns: # 0 exact match (prints the found version) # 1 version missing or mismatched (prints the found version, or empty) # 2 ambiguous: the marker is followed by extra digits/dots in the pool pdf_oxide_validate_version() { local pool_text="$1" required="$2" local found esc found=$(printf '%s\n' "$pool_text" \ | grep -oE "pdf_oxide [0-9]+\.[0-9]+\.[0-9]+" | head -1 | cut -d' ' -f2) if [ -z "$found" ]; then return 1 fi # A digit or dot immediately after the captured "pdf_oxide X.Y.Z" marker # means a suffixed build (0.3.73.1) or a constant concatenated on to the # version. Treat it as ambiguous rather than as a wrong version. # Anchor to the exact captured version (dots escaped) so the third segment # cannot backtrack and swallow the trailing digit, which would otherwise # flag a clean 0.3.73 marker as ambiguous. esc="${found//./\\.}" if printf '%s\n' "$pool_text" | grep -qE "pdf_oxide ${esc}[0-9.]"; then return 2 fi if [ "$found" != "$required" ]; then printf '%s\n' "$found" return 1 fi printf '%s\n' "$found" return 0 } check_pdf_oxide_deps() { _seed_from_system "pdf_oxide" || true # Map platform to tarball-internal subdirectory. local platform_subdir case "$(uname -s)" in Linux) case "$(uname -m)" in x86_64) platform_subdir="linux_amd64" ;; aarch64|arm64) platform_subdir="linux_arm64" ;; *) echo " pdf_oxide (static) → unsupported arch"; return 1 ;; esac ;; Darwin) case "$(uname -m)" in x86_64) platform_subdir="darwin_amd64" ;; arm64) platform_subdir="darwin_arm64" ;; *) echo " pdf_oxide (static) → unsupported arch"; return 1 ;; esac ;; *) echo " pdf_oxide (static) → unsupported OS"; return 1 ;; esac local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a" if [ -f "$lib_path" ]; then # Verify the on-disk lib matches the pinned version. _seed_from_system # accepts an existing user-cache directory without inspecting it, so a # lib left over from an earlier pin is reused silently and the upgrade # becomes a no-op. # # The version marker "pdf_oxide " is merged into Rust's string # constant pool (unlike office_oxide's standalone "0.1.9" line), so we # capture exactly three dotted segments and reject any trailing [0-9.] # as ambiguous — a suffixed build (0.3.73.1) or a constant that is # byte-identical to one must not be silently accepted. The "pdf_oxide " # prefix keeps bare version numbers of vendored dependencies out of the # match. local pool_text found_version rc pool_text=$(strings "$lib_path" 2>/dev/null) found_version=$(pdf_oxide_validate_version "$pool_text" "$PDF_OXIDE_VERSION"); rc=$? case "$rc" in 0) echo " pdf_oxide (static) → ${PDF_OXIDE_PREFIX}" return 0 ;; 2) echo -e "${RED}Error: pdf_oxide native lib version ambiguous${NC}" echo " The version marker is followed by extra digits/dots in the" echo " string pool; this is usually a suffixed build (e.g. ${PDF_OXIDE_VERSION}.1)" echo " or a concatenated constant byte-identical to one. Refresh the lib" echo " to a clean pin." echo " Required: v${PDF_OXIDE_VERSION}" return 1 ;; *) echo -e "${RED}Error: pdf_oxide native lib version mismatch${NC}" echo " Required: v${PDF_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently reverts PDF parsing fixes. Refresh:" echo " rm -rf ${PDF_OXIDE_PREFIX} ragflow_deps/pdf_oxide-go-ffi-linux-amd64.tar.gz" echo " uv run python3 ragflow_deps/download_deps.py" return 1 ;; esac fi echo " pdf_oxide (static) not found" echo " Expected: ${lib_path}" echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or: curl -fsSL https://github.com/yfedoseev/pdf_oxide/releases/download/v${PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz | tar xz -C ${PDF_OXIDE_PREFIX}" return 1 } # Build C++ static library build_cpp() { print_section "Building C++ static library" mkdir -p "$BUILD_DIR" cd "$BUILD_DIR" echo "Running cmake..." cmake .. -DCMAKE_BUILD_TYPE=Release echo "Building librag_tokenizer_c_api.a..." local jobs jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)" make rag_tokenizer_c_api -j"$jobs" if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then echo -e "${RED}Error: Failed to build C++ static library${NC}" exit 1 fi # Rename the tokenizer's bundled re2 symbols into a private namespace so they # never collide with onnxruntime's copy of re2 at final link time. # # Why this is needed: onnxruntime.a and librag_tokenizer_c_api.a both embed a # copy of re2, but they are built against different toolchains/libstdc++ and # are ABI-incompatible. Symbol-localization approaches (--exclude-libs, linker # version scripts) cannot fix it: both re2 copies must live in the same global # symbol table, and the linker resolves the tokenizer's re2 references to # whichever copy was archived first (here, onnxruntime's), regardless of which # copy is localized. SIGSEGV in re2::DFA::InlinedSearchLoop is the symptom. # # Renaming the tokenizer's re2 symbols into a private prefix gives each copy # its own namespace, so the tokenizer calls its own ABI-compatible re2 and # onnxruntime keeps calling its own. Verified to eliminate the crash. local tok_a="$BUILD_DIR/librag_tokenizer_c_api.a" local rename_map="$BUILD_DIR/re2_rename.map" if command -v objcopy >/dev/null 2>&1; then nm "$tok_a" \ | awk '$2 ~ /^[TDBRWtdbrwiIVv]$/ && $3 ~ /^_ZN3re2|_ZNK3re2|_ZTVN3re2|_ZTIN3re2|_ZTSN3re2/ { print $3" ragtokre2_"$3 }' \ | sort -u > "$rename_map" if [ -s "$rename_map" ]; then objcopy --redefine-syms="$rename_map" "$tok_a" echo -e "${GREEN}✓ Renamed $(wc -l < "$rename_map") tokenizer re2 symbols into private namespace (ragtokre2_)${NC}" fi else echo -e "${YELLOW}Warning: objcopy not found, skipping re2 symbol rename (re2 collision with onnxruntime may cause SIGSEGV)${NC}" fi echo -e "${GREEN}✓ C++ static library built successfully${NC}" } # Build C++ test executable build_cpp_test() { print_section "Building C++ test executable" if [ ! -d "$BUILD_DIR" ]; then echo "Build directory not found, running cmake first..." mkdir -p "$BUILD_DIR" cd "$BUILD_DIR" cmake .. -DCMAKE_BUILD_TYPE=Release else cd "$BUILD_DIR" fi echo "Building rag_analyzer_c_test..." local jobs jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)" make rag_analyzer_c_test -j"$jobs" if [ ! -f "$BUILD_DIR/rag_analyzer_c_test" ]; then echo -e "${RED}Error: Failed to build rag_analyzer_c_test${NC}" exit 1 fi echo -e "${GREEN}✓ C++ test executable built successfully: $BUILD_DIR/rag_analyzer_c_test${NC}" } # Build Go server build_go() { print_section "Building RAGFlow go" cd "$PROJECT_ROOT" # Check if C++ library exists if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then echo -e "${RED}Error: C++ static library not found. Run with --cpp first.${NC}" exit 1 fi if check_pcre2; then echo "✓ pcre2 library found" else install_cmd="$(detect_pcre2_install_cmd)" if [ -z "$install_cmd" ]; then echo -e "${RED}Error: libpcre2-8 not found and no supported package manager detected.${NC}" echo "Please install pcre2 development files manually." exit 1 fi if [ "$(uname)" = "Darwin" ]; then echo -e "${RED}Error: libpcre2-8 not found. Install with: $install_cmd${NC}" exit 1 fi echo -e "${YELLOW}Warning: libpcre2-8 not found. Installing with: $install_cmd${NC}" eval "$install_cmd" fi setup_cgo_env # The in-process (Go) DeepDoc backend is statically linked against ONNX # Runtime (--undefined=OrtGetApiBase + -Wl,--dynamic-list, see # setup_cgo_env). The forked onnxruntime_go binding resolves OrtGetApiBase # only at RUNTIME via dlopen(NULL), so a missing ORT archive still lets # `go build` SUCCEED and yields a server binary that FAILS AT STARTUP with a # fatal "no in-process DeepDoc backend serving". That is exactly the # breakage colleagues hit when ORT was not present at build time and # setup_cgo_env silently skipped it. Fail the build HERE instead of # deferring the breakage to runtime: a server binary without ORT is unusable, # not a degraded one. if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2 echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2 echo " statically linked into ragflow_server (--undefined=OrtGetApiBase +" >&2 echo " -Wl,--dynamic-list). Without it the binary compiles but dies" >&2 echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2 echo " Fetch the static libs with:" >&2 echo " uv run python3 ragflow_deps/download_deps.py" >&2 echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 echo " This production binary must statically include ORT — there is no" >&2 echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2 echo " - the ORT static_lib dir was never seeded: run" >&2 echo " 'uv run python3 ragflow_deps/download_deps.py', or pre-seed" >&2 echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2 echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2 return 1 fi local strip_flags=() [ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w") echo "Building RAGFlow binary: $RAGFLOW_CLI_BINARY, $RAGFLOW_SERVER_BINARY" set -x GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ go build -tags cgo,static,sonic "${strip_flags[@]}" -o "$RAGFLOW_CLI_BINARY" cmd/ragflow-cli.go GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go build -tags cgo,static,sonic "${strip_flags[@]}" -o "$RAGFLOW_SERVER_BINARY" \ cmd/ragflow_server.go cmd/deepdoc_server_ee.go set +x if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then echo -e "${RED}Error: Failed to build RAGFlow main binary${NC}" exit 1 fi echo -e "${GREEN}✓ Go ragflow-cli built successfully: $RAGFLOW_CLI_BINARY${NC}" echo -e "${GREEN}✓ Go ragflow_server built successfully: $RAGFLOW_SERVER_BINARY${NC}" } # Configure CGO flags for native libraries (office_oxide, pdfium, pdf_oxide). # All three are statically linked — no LD_LIBRARY_PATH or -Wl,-rpath needed. setup_cgo_env() { # ── office_oxide ────────────────────────────────────────────────── check_office_oxide_deps # Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the # file content of referenced .a archives. So swapping the .a in-place # (same path) does NOT invalidate the cache, and `go build` silently # reuses a stale binary linked against the old .a. # # Create a version-stamped symlink directory so the flag string # includes the actual linked version. When the .a is upgraded, the # path changes → Go cache key changes → automatic relink. local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib" local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}" mkdir -p "$versioned_lib_dir" ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \ "${versioned_lib_dir}/liboffice_oxide.a" export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}" export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a" # ── pdfium ──────────────────────────────────────────────────────── check_pdfium_deps || return 1 export CGO_LDFLAGS="$CGO_LDFLAGS ${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" # Linux: Chromium-built objects use Clang's .eh_frame format which GNU ld # cannot merge. Use lld (LLVM linker) which handles them correctly. # --allow-multiple-definition: pdf_oxide and office_oxide are both Rust # staticlibs that embed the Rust runtime; linking them together produces # duplicate rust_eh_personality / compiler-rt builtins. Those duplicates are # ABI-IDENTICAL, so merging them is benign. if [ "$(uname -s)" = "Linux" ]; then if ! command -v ld.lld >/dev/null 2>&1; then echo -e "${RED}Error: ld.lld not found. Install with: sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld${NC}" echo " lld is required to static-link Chromium-built pdfium (.eh_frame format)" return 1 fi export CGO_LDFLAGS="$CGO_LDFLAGS \ ${PDFIUM_STATIC_PREFIX}/lib/libc++.a \ ${PDFIUM_STATIC_PREFIX}/lib/libc++abi.a \ -fuse-ld=lld -Wl,--allow-multiple-definition" # The re2 regex-library collision between onnxruntime.a and # librag_tokenizer_c_api.a is fixed at the .a level in build_cpp(): # the tokenizer's bundled re2 symbols are renamed into a private # namespace (ragtokre2_) so the two re2 copies never share a symbol. # See build_cpp() for details. fi # ── pdf_oxide ───────────────────────────────────────────────────── check_pdf_oxide_deps || return 1 # The go-ffi tarball places the .a under lib//. local pdf_oxide_subdir case "$(uname -s)" in Linux) case "$(uname -m)" in x86_64) pdf_oxide_subdir="linux_amd64" ;; aarch64|arm64) pdf_oxide_subdir="linux_arm64" ;; *) echo "pdf_oxide: unsupported arch"; return 1 ;; esac ;; Darwin) case "$(uname -m)" in x86_64) pdf_oxide_subdir="darwin_amd64" ;; arm64) pdf_oxide_subdir="darwin_arm64" ;; *) echo "pdf_oxide: unsupported arch"; return 1 ;; esac ;; esac # Version-stamp the archive path so an in-place .a upgrade invalidates # Go's build cache. See the office_oxide block above for why the raw path # is not enough: the cache key hashes CGO_LDFLAGS as a string, not the # contents of the archives it names. local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}" local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}" mkdir -p "$pdf_oxide_versioned_dir" ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \ "${pdf_oxide_versioned_dir}/libpdf_oxide.a" export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a" # ── onnxruntime (static, resolved via dlopen(NULL)) ──────────────── # macOS native builds of the in-process DeepDoc backend are not supported: # ONNX Runtime is statically linked with GNU ld flags # (-Wl,--undefined=OrtGetApiBase + -Wl,--dynamic-list) and resolved at # runtime via dlopen(NULL); Apple's ld64 does not understand these flags. # Build on Linux or cross-compile there. case "$(uname -s)" in Darwin) echo "Error: macOS native build of the in-process DeepDoc backend is not supported." >&2 echo " ONNX Runtime is linked with GNU ld flags (-Wl,--undefined=OrtGetApiBase + -Wl,--dynamic-list)" >&2 echo " and resolved via dlopen(NULL); Apple's ld64 does not support them. Build on Linux." >&2 return 1 ;; esac # Statically link libonnxruntime*.a into the binary. The org Go binding # (onnxruntime_go, github.com/infiniflow/onnxruntime_go) resolves OrtGetApiBase # with dlopen(NULL) + dlsym(handle, "OrtGetApiBase"), so the ONLY symbol that # must be visible in the process-global table is OrtGetApiBase. # # We therefore do NOT use --whole-archive: that flag force-pulls every .o in # the archive (including unregistered "dead" kernels we never call), which # is why every ORT size-trimming build flag had near-zero effect before. # Instead we link the archives normally and let GNU ld's archive-level GC # drop any kernel/EP object that nothing references. The onnxruntime_go # binding reaches ORT purely through the OrtApi function-pointer table # returned by OrtGetApiBase, and a minimal/reduced-ops build registers only # the operators the deepdoc models actually use, so the reachable closure is # small. --dynamic-list exports just OrtGetApiBase (the only symbol dlopen # needs) and leaves Go's runtime symbols untouched, unlike a "local: *" # version script which breaks PIE absolute relocations. No libonnxruntime.so # is required or supported at runtime; there is no dynamic .so fallback. # # Seed the static ORT archives from the system pre-bake (/opt, laid down # by the CI runner image) into the user cache before the link check # below. Mirrors the existing _seed_from_system calls for the other # native libs so CI never downloads ORT at build/test time. _seed_from_system "onnxruntime" || true check_onnxruntime_deps || return 1 # Version-stamp the archive paths so an in-place .a upgrade invalidates # Go's build cache. Go's cache keys CGO_LDFLAGS as a string and does NOT # hash the referenced .a contents, so swapping the .a in place (same path, # e.g. a re-issue under the same tag/asset name) silently reuses a stale # linked binary. Stamp with the primary archive's sha256 so the flag string # changes when the content changes → automatic relink. local ort_version ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \ "${PROJECT_ROOT}/internal/common/environments.go" \ | sed -E 's/.*"([^"]+)".*/\1/')" local ort_versioned_dir="${ONNXRUNTIME_STATIC_PREFIX}/v${ort_version}-${ORT_A_SHA256:0:16}" mkdir -p "$ort_versioned_dir" local ort_a_versioned="" local f for f in $ORT_A_FILES; do ln -sf "$f" "$ort_versioned_dir/$(basename "$f")" ort_a_versioned="$ort_a_versioned $ort_versioned_dir/$(basename "$f")" done # Export exactly one symbol (OrtGetApiBase) for the binding's # dlopen(NULL)+dlsym lookup via --dynamic-list (NOT --version-script # with "local: *", which hides Go's runtime type symbols and breaks # the PIE absolute relocations). No --whole-archive, so GNU ld's # archive-level GC drops any ORT kernel/EP object nothing references. # # The dynamic list is written to a STABLE, project-scoped path (not # mktemp) so CGO_LDFLAGS is reproducible across builds and survives # across invocations; its content never changes, so overwriting is # safe and nothing leaks in /tmp. .cache/ is gitignored. local ort_dynamic_list="${PROJECT_ROOT}/.cache/ort_dynamic_list.txt" mkdir -p "$(dirname "$ort_dynamic_list")" printf '{\n OrtGetApiBase;\n};\n' > "$ort_dynamic_list" # --undefined=OrtGetApiBase force-pulls the archive member that # defines OrtGetApiBase (the Go binding reaches ORT only via # dlsym("OrtGetApiBase"), so nothing references it at link time and # it would otherwise be GC'd). From there the minimal build's CPU-EP # registration call chain pulls in the operators the models use. # The explicit space between $ort_dynamic_list and the versioned .a # list keeps the two as separate linker arguments regardless of the # list's leading space. export CGO_LDFLAGS="$CGO_LDFLAGS -Wl,--undefined=OrtGetApiBase -Wl,--dynamic-list=$ort_dynamic_list $ort_a_versioned -lstdc++" echo " onnxruntime (static) → $ONNXRUNTIME_STATIC_PREFIX" # The re2 regex-library collision between onnxruntime.a and # librag_tokenizer_c_api.a is fixed at the .a level in build_cpp(): # the tokenizer's bundled re2 symbols are renamed into a private # namespace (ragtokre2_) so the two re2 copies never share a symbol # name. --dynamic-list (above) exports OrtGetApiBase into the process # dynamic symbol table, which is what the binding's dlopen(NULL)+dlsym # lookup needs at runtime (no --export-dynamic required). # ── platform-specific system libraries ──────────────────────────── case "$(uname -s)" in Linux) export CGO_LDFLAGS="$CGO_LDFLAGS -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc" ;; Darwin) export CGO_LDFLAGS="$CGO_LDFLAGS \ -framework CoreGraphics -framework CoreFoundation \ -framework Security -framework SystemConfiguration \ -liconv -lresolv -lc++" ;; esac echo "CGO_CFLAGS: $CGO_CFLAGS" echo "CGO_LDFLAGS: $CGO_LDFLAGS" } # Run Go unit tests with the same CGO env as `build_go`. Any extra args are # forwarded to `go test`, e.g. `./build.sh --test -run TestFoo ./internal/admin/...`. run_go_tests() { print_section "Running Go tests" # The tokenizer panics instead of counting 0 when the cl100k table is # missing, so provision it before the tests, as CI does. Non-fatal: without # it only the token-counting tests fail, and they say why. _ensure_cl100k_table || true cd "$PROJECT_ROOT" setup_cgo_env if [ "$#" -eq 0 ]; then set -- ./... fi GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go test -tags cgo,static -count=1 "$@" run_native_tests } # Run the unit tests of the native package (DeepDoc det/DLA/TSR/OCR-rec Go # ports), now a regular package under the MAIN module gated by the `cgo` build # tag (same isolation as office_oxide/pdfium). It used to be a nested Go module # (own go.mod) that the root `./...` never descended into; after the merge it is # covered by the normal module tests. The build is pure-Go geometry plus the # cgo onnxruntime binding, so it runs whenever CGO_ENABLED=1 (the same gate the # rest of the native C libs use). Model-backed integration tests are handled by # run_native_integration_tests. run_native_tests() { print_section "Running native unit tests" ( cd "$PROJECT_ROOT" && \ GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ CGO_ENABLED=1 \ go test -tags cgo,static -count=1 ./internal/deepdoc/native/... ) } # Run the model-backed integration tests of the native package and the # native_analyzer package (the DocAnalyzer the PDF parser consumes). These # require MODEL_DIR at runtime; ONNX Runtime is statically linked and resolved # via dlopen(NULL), so no ORT_LIB is needed. The tests self-skip when MODEL_DIR # is unset or the binary lacks static ORT (see native_integration_test.go # skipIfNoModels / native_analyzer_test.go analyzerWithModels). Run under the # `cgo integration` tier. # # Why -race is split instead of applied to the whole suite: # - The DLA/TSR/OCR-rec/Det golden-comparison tests are single-threaded and # deterministic. -race multiplies their (model-heavy) heap ~10x for ZERO # race-detection value, and that is what used to OOM-kill the runner # (SIGTERM) under `cgo integration`. So they run WITHOUT -race. # - Only the concurrency-correctness tests (TestInferenceConcurrency* in the # native package, and native_analyzer's TestAnalyzerConcurrentBatchAndSingle) # actually benefit from -race: they hammer the shared session pools from many # goroutines, and the detector catches data races on the pools / per-session # in-out tensors / cross-call batch state. Those run WITH -race. # ORT's C internals are not instrumented, but all our shared mutable state lives # in Go, which the detector covers. CGO_ENABLED=1 is required for both the build # and the race runtime. run_native_integration_tests() { # Optional isolation: NATIVE_TEST_RUN forwards a -run filter and # NATIVE_TEST_V adds -v so a single native test can be exercised in # isolation (e.g. `NATIVE_TEST_RUN='TestNativeLoadsOrtModels$' NATIVE_TEST_V=1`). local native_run_filter=() if [ -n "${NATIVE_TEST_RUN:-}" ]; then native_run_filter+=(-run "${NATIVE_TEST_RUN}") fi if [ -n "${NATIVE_TEST_V:-}" ]; then native_run_filter+=(-v) fi print_section "Running native integration tests (golden/comparison, no race)" ( cd "$PROJECT_ROOT" && \ GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ CGO_ENABLED=1 \ go test -tags "cgo static integration fetch_testdata" -count=1 "${native_run_filter[@]}" ./internal/deepdoc/native/... ) print_section "Running native integration concurrency tests (race detector on)" ( cd "$PROJECT_ROOT" && \ GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ CGO_ENABLED=1 \ go test -tags "cgo static integration fetch_testdata" -race -count=1 \ -run 'TestInferenceConcurrency' ./internal/deepdoc/native/... ) print_section "Running native_analyzer race tests (race detector on)" ( cd "$PROJECT_ROOT" && \ GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ CGO_ENABLED=1 \ go test -tags "cgo static integration fetch_testdata" -race -count=1 \ ./internal/deepdoc/parser/pdf/inference/native_analyzer/... ) } # Run Go tests gated behind a build tag (or space-separated tag list), e.g. # `./build.sh --test-integration -run TestFoo ./internal/engine/...`. # See "Go Test Tiers" in AGENTS.md for the tier definitions. run_go_tests_tagged() { local tags="$1"; shift print_section "Running Go tests (tags: ${tags})" # Same cl100k table requirement as the unit tier (integration/e2e runs count # tokens too); non-fatal, see _ensure_cl100k_table. _ensure_cl100k_table || true cd "$PROJECT_ROOT" setup_cgo_env if [ "$#" -eq 0 ]; then set -- ./... fi GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go test -tags "${tags} static" -count=1 "$@" } # Clean build artifacts clean() { print_section "Cleaning build artifacts" rm -rf "$BUILD_DIR" rm -f "$RAGFLOW_SERVER_BINARY" rm -f "$RAGFLOW_CLI_BINARY" echo -e "${GREEN}✓ Build artifacts cleaned${NC}" } # Run the server run() { if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then echo -e "${RED}Error: $RAGFLOW_SERVER_BINARY not found. Build first with --all or --go${NC}" exit 1 fi cd "$PROJECT_ROOT" # admin_server must be running before ragflow_server, otherwise ragflow_server's # heartbeats to admin will error out (see internal/development.md). print_section "Starting admin server (background)" "$RAGFLOW_SERVER_BINARY" --admin & ADMIN_PID=$! # One trap for both background services: a second `trap ... EXIT INT TERM` # would replace this one rather than add to it, leaving admin_server holding # port 9381 after the foreground server exits. INGESTOR_PID is cleared first # so a value inherited from the environment cannot be signalled during the # window before the ingestor starts. INGESTOR_PID="" trap 'kill "$ADMIN_PID" ${INGESTOR_PID:+"$INGESTOR_PID"} 2>/dev/null || true' EXIT INT TERM # Give admin_server a moment to bind its listening port (9381) before # ragflow_server starts sending heartbeats to it. sleep 1 print_section "Starting ingestor (background)" "$RAGFLOW_SERVER_BINARY" --ingestor & INGESTOR_PID=$! sleep 1 print_section "Starting RAGFlow server (foreground)" "$RAGFLOW_SERVER_BINARY" --api } # Show help show_help() { # Quoted delimiter so backticks, `$var`, and `\$` in the help text are # printed literally instead of being interpreted as command substitution. cat << 'EOF' Usage: $0 [OPTIONS] Build script for RAGFlow Go server with C++ bindings. OPTIONS: --all, -a Build everything (C++ library + Go server) [default] --cpp, -c Build only C++ static library --cpp-test Build C++ test executable (builds the C++ library if needed) --go, -g Build only Go server (requires C++ library to be built) --test, -t Run Go unit tests (no build tag). Sets up the CGO env and native static libs (office_oxide/pdfium/pdf_oxide) needed to build (same contract as the Go tier table in AGENTS.md). Extra args are forwarded to `go test`, e.g. `$0 --test -run TestFoo ./internal/admin/...` --test-integration Run Go tests tagged 'integration' (need real services, e.g. MySQL/MinIO/ES/Infinity/LLM). e.g. `$0 --test-integration ./internal/engine/...` --test-integration-go Run ONLY the generic 'integration' tier (MySQL/MinIO/ Redis/NATS/Infinity/ES) without the model-backed native DeepDoc tests. Use this for the CI integration job, which brings up the service stack separately from the native backend job. e.g. `$0 --test-integration-go ./internal/storage/...` --test-e2e Run Go tests tagged 'e2e' (full-pipeline, heavy). --test-manual Run Go tests tagged 'manual' (very slow; local opt-in ONLY, never run in CI). --test-all Run 'integration' + 'e2e' tests (excludes 'manual'). --test-native Run the in-process (Go) DeepDoc backend tests tagged 'cgo integration' (needs libonnxruntime + the InfiniFlow/deepdoc model snapshot; self-skip otherwise). e.g. `$0 --test-native` --clean, -C Clean all build artifacts --check-ort-version Verify the ONNX Runtime native version is declared consistently across all sources (exit 1 on mismatch). --run, -r Build and run the server --strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w") (disabled by default, useful for smaller production binaries) --help, -h Show this help message EXAMPLES: $0 # Build everything $0 --cpp # Build only C++ library $0 --go # Build only Go server $0 --cpp-test # Build C++ test executable $0 --test # Run all Go tests (unit tier, no build tag) $0 --test -run TestFoo ./internal/admin/... # Targeted Go tests $0 --test-integration ./internal/engine/... # integration + native tiers $0 --test-integration-go ./internal/storage/... # generic integration tier only $0 --test-e2e # e2e tier $0 --test-manual # manual tier (very slow) $0 --test-all # integration + e2e (no manual) $0 --run # Build and run $0 --clean # Clean build artifacts DEPENDENCIES: - cmake >= 4.0 - go >= 1.26.4 - clang++ with C++20 support - office_oxide native library (download with: uv run python3 ragflow_deps/download_deps.py) - lld (Linux only): sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld - pcre2 development files - Debian/Ubuntu: libpcre2-dev - openSUSE/RHEL/Fedora: pcre2-devel - macOS (Homebrew): pcre2 EOF } # Main function main() { # Parse --strip / -s before other arguments local args=() for arg in "$@"; do case "$arg" in --strip|-s) STRIP_SYMBOLS="1" ;; *) args+=("$arg") ;; esac done case "${args[0]:-}" in --cpp|-c) check_cpp_deps build_cpp ;; --cpp-test) check_cpp_deps build_cpp_test ;; --go|-g) check_go_deps build_go ;; --test|-t) check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests "${args[@]:2}" else run_go_tests "${args[@]:1}" fi ;; --test-integration) check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests_tagged integration "${args[@]:2}" else run_go_tests_tagged integration "${args[@]:1}" fi run_native_integration_tests ;; --test-integration-go) # Runs only the generic `integration` test tier (MySQL/MinIO/Redis/ # NATS/Infinity/ES/etc.) WITHOUT the model-backed native DeepDoc # tests. The native tests need MODEL_DIR + the InfiniFlow/deepdoc # snapshot and are covered separately by `--test-native` / # `ragflow_native_backend` in CI; running them here would also force # the `fetch_testdata` tag's init-time network fetch of testdata, # which is undesirable in a generic CI run. Use this target for the # CI integration job. e.g. `$0 --test-integration-go ./internal/storage/...` check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests_tagged integration "${args[@]:2}" else run_go_tests_tagged integration "${args[@]:1}" fi ;; --test-e2e) check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests_tagged e2e "${args[@]:2}" else run_go_tests_tagged e2e "${args[@]:1}" fi ;; --test-manual) check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests_tagged manual "${args[@]:2}" else run_go_tests_tagged manual "${args[@]:1}" fi ;; --test-all) check_go_deps if [ "${args[1]:-}" = "--" ]; then run_go_tests_tagged "integration e2e" "${args[@]:2}" else run_go_tests_tagged "integration e2e" "${args[@]:1}" fi ;; --test-native) check_go_deps if [ "${args[1]:-}" = "--" ]; then pkgs=("${args[@]:2}") else pkgs=("${args[@]:1}") fi if [ "${#pkgs[@]}" -eq 0 ]; then pkgs=(./internal/deepdoc/parser/pdf/inference/native_analyzer/...) fi # The in-process (Go) DeepDoc backend needs the .ort weights. Default # MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so a # local run needs no MODEL_DIR export after `download_deps.py`. REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}" run_go_tests_tagged "cgo integration" "${pkgs[@]}" run_native_integration_tests ;; --clean|-C) clean ;; --check-ort-version) check_ort_version_consistency ;; --run|-r) check_cpp_deps check_go_deps build_cpp build_go run ;; --help|-h) show_help ;; --all|-a|"") check_cpp_deps check_go_deps build_cpp build_go echo -e "\n${GREEN}=== Build completed successfully! ===${NC}" echo "Binary: $RAGFLOW_SERVER_BINARY, $RAGFLOW_CLI_BINARY" ;; *) echo -e "${RED}Unknown option: ${args[0]}${NC}" show_help exit 1 ;; esac } # Only run the build when executed directly. When sourced (e.g. by tests), # skip main so the version-gate helpers can be unit-tested in isolation. if [ "${BASH_SOURCE[0]}" = "$0" ]; then main "$@" fi