diff --git a/.github/workflows/apache-rat-audit.yml b/.github/workflows/apache-rat-audit.yml index 4826fc89228..215fe40185b 100644 --- a/.github/workflows/apache-rat-audit.yml +++ b/.github/workflows/apache-rat-audit.yml @@ -52,12 +52,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 - name: Set up Java and Maven - uses: actions/setup-java@v3 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: '11' @@ -195,7 +195,7 @@ jobs: - name: Upload Rat check results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: rat-check-results path: rat-output.log diff --git a/.github/workflows/binary-swap-check.yml b/.github/workflows/binary-swap-check.yml index 282fc9af877..fb0abf42e81 100644 --- a/.github/workflows/binary-swap-check.yml +++ b/.github/workflows/binary-swap-check.yml @@ -244,14 +244,14 @@ jobs: echo "Building RPM with Version: ${SAFE_VERSION}" "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${SAFE_VERSION}" --release "1" - os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) + os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release) RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${SAFE_VERSION}"-"1".el"${os_version}".x86_64.rpm # Verify RPM echo "Verifying RPM..." rpm -qip "${RPM_FILE}" for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -388,14 +388,14 @@ jobs: SAFE_VERSION=$(echo "99.0.0" | tr '-' '_') "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${SAFE_VERSION}" --release "current" - os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) + os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release) RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${SAFE_VERSION}"-"current".el"${os_version}".x86_64.rpm # Verify RPM echo "Verifying RPM..." rpm -qip "${RPM_FILE}" for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -523,10 +523,15 @@ jobs: echo "Installing baseline RPM: ${BASELINE_RPM}" dnf install -y "${BASELINE_RPM}" - # Check installed location based on where bin/postgres ended up - INSTALLED_PG=$(rpm -ql apache-cloudberry-db-incubating | grep "bin/postgres$" | head -1) + # Check installed location based on where bin/postgres ended up. + # Ask the RPM for its own Name: the package name now carries the + # major version (apache-cloudberry-db-incubating-), and the + # baseline may still use the historical unversioned name. + PKG_NAME=$(rpm -qp --queryformat '%{NAME}\n' "${BASELINE_RPM}") + echo "Querying installed package: ${PKG_NAME}" + INSTALLED_PG=$(rpm -ql "${PKG_NAME}" | grep "bin/postgres$" | head -1) if [ -z "$INSTALLED_PG" ]; then - echo "::error::Could not find bin/postgres in installed RPM" + echo "::error::Could not find bin/postgres in installed RPM (${PKG_NAME})" exit 1 fi @@ -580,8 +585,15 @@ jobs: dnf install -y "${CURRENT_RPM}" || dnf upgrade -y "${CURRENT_RPM}" - # Check installed location - INSTALLED_PG=$(rpm -ql apache-cloudberry-db-incubating | grep "bin/postgres$" | head -1) + # Check installed location (see the baseline step for why the + # package name is read from the RPM instead of hardcoded). + PKG_NAME=$(rpm -qp --queryformat '%{NAME}\n' "${CURRENT_RPM}") + echo "Querying installed package: ${PKG_NAME}" + INSTALLED_PG=$(rpm -ql "${PKG_NAME}" | grep "bin/postgres$" | head -1) + if [ -z "$INSTALLED_PG" ]; then + echo "::error::Could not find bin/postgres in installed RPM (${PKG_NAME})" + exit 1 + fi INSTALLED_DIR=$(dirname $(dirname "$INSTALLED_PG")) echo "Detected installation at: ${INSTALLED_DIR}" diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml deleted file mode 100644 index 7ae0da0aa82..00000000000 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ /dev/null @@ -1,1944 +0,0 @@ -# -------------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed -# with this work for additional information regarding copyright -# ownership. The ASF licenses this file to You under the Apache -# License, Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. -# -# -------------------------------------------------------------------- -# GitHub Actions Workflow: Apache Cloudberry Build Pipeline (Rocky 8) -# -------------------------------------------------------------------- -# Description: -# -# This workflow builds, tests, and packages Apache Cloudberry on -# Rocky Linux 8. It ensures artifact integrity, performs installation -# tests, validates key operations, and provides detailed test reports, -# including handling for ignored test cases. -# -# Workflow Overview: -# 1. **Check Skip**: -# - Dynamically determines if the workflow should run based on CI skip flags. -# - Evaluates the following fields for skip flags: -# - **Pull Request Events**: PR title and PR body. -# - **Push Events**: Commit message of the head commit. -# - Supports the following skip patterns (case-insensitive): -# - `[skip ci]` -# - `[ci skip]` -# - `[no ci]` -# - **Example Usage**: -# - Add `[skip ci]` to a commit message, PR title, or body to skip the workflow. -# -# 2. **Build Job**: -# - Configures and builds Apache Cloudberry. -# - Supports debug build configuration via ENABLE_DEBUG flag. -# - Runs unit tests and verifies build artifacts. -# - Creates RPM packages (regular or debug), source tarballs, and logs. -# - **Key Artifacts**: RPM package, source tarball, build logs. -# -# 3. **RPM Install Test Job**: -# - Verifies RPM integrity and installs Cloudberry. -# - Validates successful installation. -# - **Key Artifacts**: Installation logs, verification results. -# -# 4. **Test Job (Matrix)**: -# - Executes a test matrix to validate different scenarios. -# - Creates a demo cluster and runs installcheck tests. -# - Parses and reports test results, including failed and ignored tests. -# - Detects and analyzes any core dumps generated during tests. -# - **Key Features**: -# - Regression diffs are displayed if found, aiding quick debugging. -# - Both failed and ignored test names are logged and reported. -# - Core dumps are analyzed using GDB for stack traces. -# - **Key Artifacts**: Test logs, regression files, test summaries, core analyses. -# -# 5. **Report Job**: -# - Aggregates job results into a final report. -# - Sends failure notifications if any step fails. -# -# Execution Environment: -# - **Runs On**: ubuntu-22.04 with Rocky Linux 8 containers. -# - **Resource Requirements**: -# - Disk: Minimum 20GB free space. -# - Memory: Minimum 8GB RAM. -# - CPU: Recommended 4+ cores. -# -# Triggers: -# - Push to `main` branch. -# - Pull request that modifies this workflow file. -# - Scheduled: Every Monday at 02:00 UTC. -# - Manual workflow dispatch. -# -# Container Images: -# - **Build**: `apache/incubator-cloudberry:cbdb-build-rocky8-latest` -# - **Test**: `apache/incubator-cloudberry:cbdb-test-rocky8-latest` -# -# Artifacts: -# - RPM Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Regression Diffs (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Core Dump Analyses (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# -# Notes: -# - Supports concurrent job execution. -# - Includes robust skip logic for pull requests and pushes. -# - Handles ignored test cases, ensuring results are comprehensive. -# - Provides detailed logs and error handling for failed and ignored tests. -# - Analyzes core dumps generated during test execution. -# - Supports debug builds with preserved symbols. -# -------------------------------------------------------------------- - -name: Apache Cloudberry Build (Rocky 8) - -on: - push: - branches: [main, REL_2_STABLE] - pull_request: - paths: - - '.github/workflows/build-cloudberry-rocky8.yml' - # We can enable the PR test when needed - # branches: [main, REL_2_STABLE] - # types: [opened, synchronize, reopened, edited] - schedule: - # Run every Monday at 02:00 UTC - - cron: '0 2 * * 1' - workflow_dispatch: - inputs: - test_selection: - description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' - required: false - default: 'all' - type: string - reuse_artifacts_from_run_id: - description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' - required: false - default: '' - type: string - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -# Note: Step details, logs, and artifacts require users to be logged into GitHub -# even for public repositories. This is a GitHub security feature and cannot -# be overridden by permissions. - -permissions: - # READ permissions allow viewing repository contents - contents: read # Required for checking out code and reading repository files - - # READ permissions for packages (Container registry, etc) - packages: read # Allows reading from GitHub package registry - - # WRITE permissions for actions includes read access to: - # - Workflow runs - # - Artifacts (requires GitHub login) - # - Logs (requires GitHub login) - actions: write - - # READ permissions for checks API: - # - Step details visibility (requires GitHub login) - # - Check run status and details - checks: read - - # READ permissions for pull request metadata: - # - PR status - # - Associated checks - # - Review states - pull-requests: read - -env: - LOG_RETENTION_DAYS: 7 - ENABLE_DEBUG: false - -jobs: - - ## ====================================================================== - ## Job: check-skip - ## ====================================================================== - - check-skip: - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip-check.outputs.should_skip }} - steps: - - id: skip-check - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - PR_TITLE: ${{ github.event.pull_request.title || '' }} - PR_BODY: ${{ github.event.pull_request.body || '' }} - run: | - # Default to not skipping - echo "should_skip=false" >> "$GITHUB_OUTPUT" - - # Apply skip logic only for pull_request events - if [[ "$EVENT_NAME" == "pull_request" ]]; then - # Combine PR title and body for skip check - MESSAGE="${PR_TITLE}\n${PR_BODY}" - - # Escape special characters using printf %s - ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") - - echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" - - # Check for skip patterns - if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then - echo "should_skip=true" >> "$GITHUB_OUTPUT" - fi - else - echo "Skip logic is not applied for $EVENT_NAME events." - fi - - - name: Report Skip Status - if: steps.skip-check.outputs.should_skip == 'true' - run: | - echo "CI Skip flag detected in PR - skipping all checks." - exit 0 - - ## ====================================================================== - ## Job: prepare-test-matrix - ## ====================================================================== - - prepare-test-matrix: - runs-on: ubuntu-22.04 - needs: [check-skip] - if: needs.check-skip.outputs.should_skip != 'true' - outputs: - test-matrix: ${{ steps.set-matrix.outputs.matrix }} - - steps: - - id: set-matrix - run: | - echo "=== Matrix Preparation Diagnostics ===" - echo "Event type: ${{ github.event_name }}" - echo "Test selection input: '${{ github.event.inputs.test_selection }}'" - - # Define defaults - DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 - DEFAULT_ENABLE_CGROUPS=false - DEFAULT_ENABLE_CORE_CHECK=true - DEFAULT_PG_SETTINGS_OPTIMIZER="" - - # Define base test configurations - ALL_TESTS='{ - "include": [ - {"test":"ic-good-opt-off", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"off"} - }, - {"test":"ic-good-opt-on", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"on"} - }, - {"test":"pax-ic-good-opt-off", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"off", - "default_table_access_method":"pax" - } - }, - {"test":"pax-ic-good-opt-on", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"on", - "default_table_access_method":"pax" - } - }, - {"test":"pax-ic-isolation2-opt-off", - "make_configs":["contrib/pax_storage/:isolation2_test"], - "pg_settings":{ - "optimizer":"off", - "default_table_access_method":"pax" - }, - "enable_core_check":false - }, - {"test":"pax-ic-isolation2-opt-on", - "make_configs":["contrib/pax_storage/:isolation2_test"], - "pg_settings":{ - "optimizer":"on", - "default_table_access_method":"pax" - }, - "enable_core_check":false - }, - {"test":"ic-expandshrink", - "make_configs":["src/test/isolation2:installcheck-expandshrink"] - }, - {"test":"ic-singlenode", - "make_configs":["src/test/isolation:installcheck-singlenode", - "src/test/singlenode_regress:installcheck-singlenode", - "src/test/singlenode_isolation2:installcheck-singlenode"], - "num_primary_mirror_pairs":0 - }, - {"test":"ic-resgroup-v2", - "make_configs":["src/test/isolation2:installcheck-resgroup-v2"], - "enable_cgroups":true - }, - {"test":"ic-contrib", - "make_configs":["contrib/auto_explain:installcheck", - "contrib/amcheck:installcheck", - "contrib/citext:installcheck", - "contrib/btree_gin:installcheck", - "contrib/btree_gist:installcheck", - "contrib/dblink:installcheck", - "contrib/dict_int:installcheck", - "contrib/dict_xsyn:installcheck", - "contrib/extprotocol:installcheck", - "contrib/file_fdw:installcheck", - "contrib/formatter_fixedwidth:installcheck", - "contrib/hstore:installcheck", - "contrib/indexscan:installcheck", - "contrib/pg_trgm:installcheck", - "contrib/indexscan:installcheck", - "contrib/pgcrypto:installcheck", - "contrib/pgstattuple:installcheck", - "contrib/tablefunc:installcheck", - "contrib/try_convert:installcheck", - "contrib/passwordcheck:installcheck", - "contrib/pg_buffercache:installcheck", - "contrib/sslinfo:installcheck"] - }, - {"test":"ic-gpcontrib", - "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/zstd:installcheck", - "gpcontrib/gp_sparse_vector:installcheck", - "gpcontrib/gp_toolkit:installcheck"] - }, - {"test":"gpcontrib-gp-stats-collector", - "make_configs":["gpcontrib/gp_stats_collector:installcheck"], - "extension":"gp_stats_collector" - }, - {"test":"ic-fixme", - "make_configs":["src/test/regress:installcheck-fixme"], - "enable_core_check":false - }, - {"test":"ic-isolation2", - "make_configs":["src/test/isolation2:installcheck-isolation2"] - }, - {"test":"ic-isolation2-hot-standby", - "make_configs":["src/test/isolation2:installcheck-hot-standby"] - }, - {"test":"ic-isolation2-crash", - "make_configs":["src/test/isolation2:installcheck-isolation2-crash"], - "enable_core_check":false - }, - {"test":"ic-parallel-retrieve-cursor", - "make_configs":["src/test/isolation2:installcheck-parallel-retrieve-cursor"] - }, - {"test":"ic-cbdb-parallel", - "make_configs":["src/test/regress:installcheck-cbdb-parallel"] - }, - {"test":"ic-recovery", - "make_configs":["src/test/recovery:installcheck"], - "enable_core_check":false - } - ] - }' - - # Function to apply defaults - apply_defaults() { - echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ - --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ - --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ - --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ - 'def get_defaults: - { - num_primary_mirror_pairs: ($npm|tonumber), - enable_cgroups: $ec, - enable_core_check: $ecc, - pg_settings: { - optimizer: $opt - } - }; - get_defaults * .' - } - - # Extract all valid test names from ALL_TESTS - VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') - - # Parse input test selection - IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" - - # Default to all tests if selection is empty or 'all' - if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then - mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" - fi - - # Validate and filter selected tests - INVALID_TESTS=() - FILTERED_TESTS=() - for TEST in "${SELECTED_TESTS[@]}"; do - TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace - if echo "$VALID_TESTS" | grep -qw "$TEST"; then - FILTERED_TESTS+=("$TEST") - else - INVALID_TESTS+=("$TEST") - fi - done - - # Handle invalid tests - if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then - echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" - echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" - exit 1 - fi - - # Build result JSON with defaults applied - RESULT='{"include":[' - FIRST=true - for TEST in "${FILTERED_TESTS[@]}"; do - CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") - FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") - if [[ "$FIRST" == true ]]; then - FIRST=false - else - RESULT="${RESULT}," - fi - RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" - done - RESULT="${RESULT}]}" - - # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . - - # Fix: Use block redirection - { - echo "matrix<> "$GITHUB_OUTPUT" - - echo "=== Matrix Preparation Complete ===" - - ## ====================================================================== - ## Job: build - ## ====================================================================== - - build: - name: Build Apache Cloudberry RPM (Rocky 8) - env: - JOB_TYPE: build - needs: [check-skip] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - if: github.event.inputs.reuse_artifacts_from_run_id == '' - outputs: - build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} - - container: - image: apache/incubator-cloudberry:cbdb-build-rocky8-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Set build timestamp - if: needs.check-skip.outputs.should_skip != 'true' - id: set_timestamp # Add an ID to reference this step - run: | - timestamp=$(date +'%Y%m%d_%H%M%S') - echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs - echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable - - - name: Checkout Apache Cloudberry - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/checkout@v4 - with: - fetch-depth: 1 - submodules: true - - - name: Cloudberry Environment Initialization - if: needs.check-skip.outputs.should_skip != 'true' - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Generate Build Job Summary Start - if: needs.check-skip.outputs.should_skip != 'true' - run: | - { - echo "# Build Job Summary" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" - echo "- OS Version: $(cat /etc/redhat-release)" - echo "- GCC Version: $(gcc --version | head -n1)" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Run Apache Cloudberry configure script - if: needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then - echo "::error::Configure script failed" - exit 1 - fi - - - name: Run Apache Cloudberry build script - if: needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then - echo "::error::Build script failed" - exit 1 - fi - - - name: Verify build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - run: | - set -eo pipefail - - echo "Verifying build artifacts..." - { - echo "=== Build Artifacts Verification ===" - echo "Timestamp: $(date -u)" - - if [ ! -d "/usr/local/cloudberry-db" ]; then - echo "::error::Build artifacts directory not found" - exit 1 - fi - - # Verify critical binaries - critical_binaries=( - "/usr/local/cloudberry-db/bin/postgres" - "/usr/local/cloudberry-db/bin/psql" - ) - - echo "Checking critical binaries..." - for binary in "${critical_binaries[@]}"; do - if [ ! -f "$binary" ]; then - echo "::error::Critical binary missing: $binary" - exit 1 - fi - if [ ! -x "$binary" ]; then - echo "::error::Binary not executable: $binary" - exit 1 - fi - echo "Binary verified: $binary" - ls -l "$binary" - done - - # Test binary execution - echo "Testing binary execution..." - if ! /usr/local/cloudberry-db/bin/postgres --version; then - echo "::error::postgres binary verification failed" - exit 1 - fi - if ! /usr/local/cloudberry-db/bin/psql --version; then - echo "::error::psql binary verification failed" - exit 1 - fi - - echo "All build artifacts verified successfully" - } 2>&1 | tee -a build-logs/details/build-verification.log - - - name: Create Source tarball, create RPM and verify artifacts - if: needs.check-skip.outputs.should_skip != 'true' - env: - CBDB_VERSION: 99.0.0 - BUILD_NUMBER: 1 - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Artifact Creation Log ===" - echo "Timestamp: $(date -u)" - - # Create source tarball - echo "Creating source tarball..." - tar czf "${SRC_DIR}"/../apache-cloudberry-incubating-src.tgz -C "${SRC_DIR}"/.. ./cloudberry - mv "${SRC_DIR}"/../apache-cloudberry-incubating-src.tgz "${SRC_DIR}" - - # Verify tarball contents - echo "Verifying source tarball contents..." - if ! tar tzf "${SRC_DIR}"/apache-cloudberry-incubating-src.tgz > /dev/null; then - echo "::error::Source tarball verification failed" - exit 1 - fi - - # Create RPM - echo "Creating RPM package..." - rpmdev-setuptree - ln -s "${SRC_DIR}"/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec "${HOME}"/rpmbuild/SPECS/apache-cloudberry-db-incubating.spec - cp "${SRC_DIR}"/LICENSE /usr/local/cloudberry-db - - DEBUG_RPMBUILD_OPT="" - DEBUG_IDENTIFIER="" - if [ "${{ env.ENABLE_DEBUG }}" = "true" ]; then - DEBUG_RPMBUILD_OPT="--with-debug" - DEBUG_IDENTIFIER=".debug" - fi - - "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${CBDB_VERSION}" --release "${BUILD_NUMBER}" "${DEBUG_RPMBUILD_OPT}" - - # Get OS version and move RPM - os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) - RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm - cp "${RPM_FILE}" "${SRC_DIR}" - RPM_DEBUG="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-debuginfo-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm - cp "${RPM_DEBUG}" "${SRC_DIR}" - - # Get package information - echo "Package Information:" - rpm -qip "${RPM_FILE}" - - # Verify critical files in RPM - echo "Verifying critical files in RPM..." - for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then - echo "::error::Critical binary '${binary}' not found in RPM" - exit 1 - fi - done - - # Record checksums - echo "Calculating checksums..." - sha256sum "${RPM_FILE}" | tee -a build-logs/details/checksums.log - sha256sum "${SRC_DIR}"/apache-cloudberry-incubating-src.tgz | tee -a build-logs/details/checksums.log - - echo "Artifacts created and verified successfully" - - } 2>&1 | tee -a build-logs/details/artifact-creation.log - - - name: Run Apache Cloudberry unittest script - if: needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then - echo "::error::Unittest script failed" - exit 1 - fi - - - name: Generate Build Job Summary End - if: always() - run: | - { - echo "## Build Results" - echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload build logs - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 - with: - name: build-logs-rocky8-${{ env.BUILD_TIMESTAMP }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Cloudberry RPM build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - *.rpm - - - name: Upload Cloudberry source build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-source-build-artifacts-rocky8 - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - apache-cloudberry-incubating-src.tgz - - ## ====================================================================== - ## Job: rpm-install-test - ## ====================================================================== - - rpm-install-test: - name: RPM Install Test Apache Cloudberry (Rocky 8) - needs: [check-skip, build] - if: | - !cancelled() && - (needs.build.result == 'success' || needs.build.result == 'skipped') && - github.event.inputs.reuse_artifacts_from_run_id == '' - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - container: - image: apache/incubator-cloudberry:cbdb-test-rocky8-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "RPM install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Download Cloudberry RPM build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 - path: ${{ github.workspace }}/rpm_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Cloudberry Environment Initialization - if: needs.check-skip.outputs.should_skip != 'true' - env: - LOGS_DIR: install-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Verify RPM artifacts - if: needs.check-skip.outputs.should_skip != 'true' - id: verify-artifacts - run: | - set -eo pipefail - - RPM_FILE=$(ls "${GITHUB_WORKSPACE}"/rpm_build_artifacts/apache-cloudberry-db-incubating-[0-9]*.rpm | grep -v "debuginfo") - if [ ! -f "${RPM_FILE}" ]; then - echo "::error::RPM file not found" - exit 1 - fi - - echo "rpm_file=${RPM_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying RPM artifacts..." - { - echo "=== RPM Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "RPM File: ${RPM_FILE}" - - # Get RPM metadata and verify contents - echo "Package Information:" - rpm -qip "${RPM_FILE}" - - # Get key RPM attributes for verification - RPM_VERSION=$(rpm -qp --queryformat "%{VERSION}" "${RPM_FILE}") - RPM_RELEASE=$(rpm -qp --queryformat "%{RELEASE}" "${RPM_FILE}") - echo "version=${RPM_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${RPM_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the RPM - echo "Verifying critical files in RPM..." - for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then - echo "::error::Critical binary '${binary}' not found in RPM" - exit 1 - fi - done - - echo "RPM Details:" - echo "- Version: ${RPM_VERSION}" - echo "- Release: ${RPM_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${RPM_FILE}" - - } 2>&1 | tee -a install-logs/details/rpm-verification.log - - - name: Install Cloudberry RPM - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - RPM_FILE: ${{ steps.verify-artifacts.outputs.rpm_file }} - RPM_VERSION: ${{ steps.verify-artifacts.outputs.version }} - RPM_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${RPM_FILE}" ]; then - echo "::error::RPM_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== RPM Installation Log ===" - echo "Timestamp: $(date -u)" - echo "RPM File: ${RPM_FILE}" - echo "Version: ${RPM_VERSION}" - echo "Release: ${RPM_RELEASE}" - - # Refresh repository metadata to avoid mirror issues - echo "Refreshing repository metadata..." - dnf clean all - dnf makecache --refresh || dnf makecache - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install RPM with retry logic for mirror issues - # Use --releasever=8 to pin to stable Rocky Linux 8 repos (not bleeding-edge 8.10) - echo "Starting installation..." - if ! time dnf install -y --setopt=retries=10 --releasever=8 "${RPM_FILE}"; then - echo "::error::RPM installation failed" - exit 1 - fi - - echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating - echo "Installed files:" - rpm -ql apache-cloudberry-db-incubating - } 2>&1 | tee -a install-logs/details/rpm-installation.log - - - name: Upload install logs - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 - with: - name: install-logs-rocky8-${{ needs.build.outputs.build_timestamp }} - path: | - install-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Generate Install Test Job Summary End - if: always() - shell: bash {0} - run: | - { - echo "# Installed Package Summary" - echo "\`\`\`" - - rpm -qi apache-cloudberry-db-incubating - echo "\`\`\`" - } >> "$GITHUB_STEP_SUMMARY" || true - - ## ====================================================================== - ## Job: test - ## ====================================================================== - - test: - name: ${{ matrix.test }} (Rocky 8) - needs: [check-skip, build, prepare-test-matrix] - if: | - !cancelled() && - (needs.build.result == 'success' || needs.build.result == 'skipped') - runs-on: ubuntu-22.04 - timeout-minutes: 120 - # actionlint-allow matrix[*].pg_settings - strategy: - fail-fast: false # Continue with other tests if one fails - matrix: ${{ fromJson(needs.prepare-test-matrix.outputs.test-matrix) }} - - container: - image: apache/incubator-cloudberry:cbdb-build-rocky8-latest - options: >- - --privileged - --user root - --hostname cdw - --shm-size=2gb - --ulimit core=-1 - --cgroupns=host - -v /sys/fs/cgroup:/sys/fs/cgroup:rw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Use timestamp from previous job - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "Timestamp from output: ${{ needs.build.outputs.build_timestamp }}" - - - name: Cloudberry Environment Initialization - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Setup cgroups - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - run: | - set -uxo pipefail - - if [ "${{ matrix.enable_cgroups }}" = "true" ]; then - - echo "Current mounts:" - mount | grep cgroup - - CGROUP_BASEDIR=/sys/fs/cgroup - - # 1. Basic setup with permissions - sudo chmod -R 777 ${CGROUP_BASEDIR}/ - sudo mkdir -p ${CGROUP_BASEDIR}/gpdb - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 2. Enable controllers - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true - - # 3. CPU settings - sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true - sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true - sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true - - # 4. Memory settings - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true - - # 5. IO settings - echo "Available block devices:" - lsblk - - sudo bash -c " - if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then - echo 'Detected IO devices:' - cat \${CGROUP_BASEDIR}/gpdb/io.stat - fi - echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true - " - - # 6. Fix permissions again after all writes - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 7. Check required files - echo "Checking required files:" - required_files=( - "cgroup.procs" - "cpu.max" - "cpu.pressure" - "cpu.weight" - "cpu.weight.nice" - "cpu.stat" - "cpuset.cpus" - "cpuset.mems" - "cpuset.cpus.effective" - "cpuset.mems.effective" - "memory.current" - "io.max" - ) - - for file in "${required_files[@]}"; do - if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then - echo "✓ $file exists" - ls -l "${CGROUP_BASEDIR}/gpdb/$file" - else - echo "✗ $file missing" - fi - done - - # 8. Test subdirectory creation - echo "Testing subdirectory creation..." - sudo -u gpadmin bash -c " - TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 - if mkdir -p \$TEST_DIR; then - echo 'Created test directory' - sudo chmod -R 777 \$TEST_DIR - if echo \$\$ > \$TEST_DIR/cgroup.procs; then - echo 'Successfully wrote to cgroup.procs' - cat \$TEST_DIR/cgroup.procs - # Move processes back to parent before cleanup - echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs - else - echo 'Failed to write to cgroup.procs' - ls -la \$TEST_DIR/cgroup.procs - fi - ls -la \$TEST_DIR/ - rmdir \$TEST_DIR || { - echo 'Moving all processes to parent before cleanup' - cat \$TEST_DIR/cgroup.procs | while read pid; do - echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true - done - rmdir \$TEST_DIR - } - else - echo 'Failed to create test directory' - fi - " - - # 9. Verify setup as gpadmin user - echo "Testing cgroup access as gpadmin..." - sudo -u gpadmin bash -c " - echo 'Checking mounts...' - mount | grep cgroup - - echo 'Checking /proc/self/mounts...' - cat /proc/self/mounts | grep cgroup - - if ! grep -q cgroup2 /proc/self/mounts; then - echo 'ERROR: cgroup2 mount NOT visible to gpadmin' - exit 1 - fi - echo 'SUCCESS: cgroup2 mount visible to gpadmin' - - if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then - echo 'ERROR: gpadmin cannot write to gpdb cgroup' - exit 1 - fi - echo 'SUCCESS: gpadmin can write to gpdb cgroup' - - echo 'Verifying key files content:' - echo 'cpu.max:' - cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' - echo 'cpuset.cpus:' - cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' - echo 'cgroup.subtree_control:' - cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' - " - - # 10. Show final state - echo "Final cgroup state:" - ls -la ${CGROUP_BASEDIR}/gpdb/ - echo "Cgroup setup completed successfully" - else - echo "Cgroup setup skipped" - fi - - - name: "Generate Test Job Summary Start: ${{ matrix.test }}" - if: always() - run: | - { - echo "# Test Job Summary: ${{ matrix.test }} (Rocky 8)" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## Skip Status" - echo "✓ Test execution skipped via CI skip flag" - else - echo "- OS Version: $(cat /etc/redhat-release)" - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Download Cloudberry RPM build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky8 - path: ${{ github.workspace }}/rpm_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Download Cloudberry Source build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-source-build-artifacts-rocky8 - path: ${{ github.workspace }}/source_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Verify downloaded artifacts - if: needs.check-skip.outputs.should_skip != 'true' - id: verify-artifacts - run: | - set -eo pipefail - - SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-incubating-src.tgz) - if [ ! -f "${SRC_TARBALL_FILE}" ]; then - echo "::error::SRC TARBALL file not found" - exit 1 - fi - - echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying SRC TARBALL artifacts..." - { - echo "=== SRC TARBALL Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${SRC_TARBALL_FILE}" - - } 2>&1 | tee -a build-logs/details/src-tarball-verification.log - - RPM_FILE=$(ls "${GITHUB_WORKSPACE}"/rpm_build_artifacts/apache-cloudberry-db-incubating-[0-9]*.rpm | grep -v "debuginfo") - if [ ! -f "${RPM_FILE}" ]; then - echo "::error::RPM file not found" - exit 1 - fi - - echo "rpm_file=${RPM_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying RPM artifacts..." - { - echo "=== RPM Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "RPM File: ${RPM_FILE}" - - # Get RPM metadata and verify contents - echo "Package Information:" - rpm -qip "${RPM_FILE}" - - # Get key RPM attributes for verification - RPM_VERSION=$(rpm -qp --queryformat "%{VERSION}" "${RPM_FILE}") - RPM_RELEASE=$(rpm -qp --queryformat "%{RELEASE}" "${RPM_FILE}") - echo "version=${RPM_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${RPM_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the RPM - echo "Verifying critical files in RPM..." - for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then - echo "::error::Critical binary '${binary}' not found in RPM" - exit 1 - fi - done - - echo "RPM Details:" - echo "- Version: ${RPM_VERSION}" - echo "- Release: ${RPM_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${RPM_FILE}" - - } 2>&1 | tee -a build-logs/details/rpm-verification.log - - - name: Install Cloudberry RPM - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - RPM_FILE: ${{ steps.verify-artifacts.outputs.rpm_file }} - RPM_VERSION: ${{ steps.verify-artifacts.outputs.version }} - RPM_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${RPM_FILE}" ]; then - echo "::error::RPM_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== RPM Installation Log ===" - echo "Timestamp: $(date -u)" - echo "RPM File: ${RPM_FILE}" - echo "Version: ${RPM_VERSION}" - echo "Release: ${RPM_RELEASE}" - - # Refresh repository metadata to avoid mirror issues - echo "Refreshing repository metadata..." - dnf clean all - dnf makecache --refresh || dnf makecache - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install RPM with retry logic for mirror issues - # Use --releasever=8 to pin to stable Rocky Linux 8 repos (not bleeding-edge 8.10) - echo "Starting installation..." - if ! time dnf install -y --setopt=retries=10 --releasever=8 "${RPM_FILE}"; then - echo "::error::RPM installation failed" - exit 1 - fi - - echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating - } 2>&1 | tee -a build-logs/details/rpm-installation.log - - # Clean up downloaded RPM artifacts to free disk space - echo "=== Disk space before RPM cleanup ===" - echo "Human readable:" - df -kh / - echo "Exact KB:" - df -k / - echo "RPM artifacts size:" - du -sh "${GITHUB_WORKSPACE}"/rpm_build_artifacts || true - echo "Cleaning up RPM artifacts to free disk space..." - rm -rf "${GITHUB_WORKSPACE}"/rpm_build_artifacts - echo "=== Disk space after RPM cleanup ===" - echo "Human readable:" - df -kh / - echo "Exact KB:" - df -k / - - - name: Extract source tarball - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Source Extraction Log ===" - echo "Timestamp: $(date -u)" - - echo "Starting extraction..." - if ! time tar zxf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then - echo "::error::Source extraction failed" - exit 1 - fi - - echo "Extraction completed successfully" - echo "Extracted contents:" - ls -la "${SRC_DIR}/../cloudberry" - echo "Directory size:" - du -sh "${SRC_DIR}/../cloudberry" - } 2>&1 | tee -a build-logs/details/source-extraction.log - - # Clean up source tarball to free disk space - echo "=== Disk space before source tarball cleanup ===" - echo "Human readable:" - df -kh / - echo "Exact KB:" - df -k / - echo "Source tarball artifacts size:" - du -sh "${GITHUB_WORKSPACE}"/source_build_artifacts || true - echo "Cleaning up source tarball to free disk space..." - rm -rf "${GITHUB_WORKSPACE}"/source_build_artifacts - echo "=== Disk space after source tarball cleanup ===" - echo "Human readable:" - df -kh / - echo "Exact KB:" - df -k / - - - name: Create Apache Cloudberry demo cluster - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then - echo "::error::Demo cluster creation failed" - exit 1 - fi - - } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log - - - name: "Run Tests: ${{ matrix.test }}" - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - BUILD_DESTINATION: /usr/local/cloudberry-db - shell: bash {0} - run: | - set -o pipefail - - # Initialize test status - overall_status=0 - - # Create logs directory structure - mkdir -p build-logs/details - - # Core file config - mkdir -p "/tmp/cloudberry-cores" - chmod 1777 "/tmp/cloudberry-cores" - sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" - sysctl kernel.core_pattern - su - gpadmin -c "ulimit -c" - - # WARNING: PostgreSQL Settings - # When adding new pg_settings key/value pairs: - # 1. Add a new check below for the setting - # 2. Follow the same pattern as optimizer - # 3. Update matrix entries to include the new setting - - # Create extension if required - if [[ "${{ matrix.extension != '' }}" == "true" ]]; then - case "${{ matrix.extension }}" in - gp_stats_collector) - if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ - source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ - gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ - gpstop -ra && \ - echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ - SHOW shared_preload_libraries; \ - TABLE pg_extension;' | \ - psql postgres" - then - echo "Error creating gp_stats_collector extension" - exit 1 - fi - ;; - *) - echo "Unknown extension: ${{ matrix.extension }}" - exit 1 - ;; - esac - fi - - # Set PostgreSQL options if defined - PG_OPTS="" - if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" - fi - - if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" - fi - - # Read configs into array - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting test execution for ${{ matrix.test }} ===" - echo "Number of configurations to execute: ${#configs[@]}" - echo "" - - # Execute each config separately - for ((i=0; i<${#configs[@]}; i++)); do - config="${configs[$i]}" - IFS=':' read -r dir target <<< "$config" - - echo "=== Executing configuration $((i+1))/${#configs[@]} ===" - echo "Make command: make -C $dir $target" - echo "Environment:" - echo "- PGOPTIONS: ${PG_OPTS}" - - # Create unique log file for this configuration - config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" - - # Clean up any existing core files - echo "Cleaning up existing core files..." - rm -f /tmp/cloudberry-cores/core-* - - # Execute test script with proper environment setup - if ! time su - gpadmin -c "cd ${SRC_DIR} && \ - MAKE_NAME='${{ matrix.test }}-config$i' \ - MAKE_TARGET='$target' \ - MAKE_DIRECTORY='-C $dir' \ - PGOPTIONS='${PG_OPTS}' \ - SRC_DIR='${SRC_DIR}' \ - ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ - 2>&1 | tee "$config_log"; then - echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" - overall_status=1 - fi - - # Check for results directory - results_dir="${dir}/results" - - if [[ -d "$results_dir" ]]; then - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - - find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - else - echo "-----------------------------------------" - echo "Results directory $results_dir does not exit" - echo "-----------------------------------------" - fi - - # Analyze any core files generated by this test configuration - echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." - test_id="${{ matrix.test }}-config$i" - - # List the cores directory - echo "-----------------------------------------" - echo "Cores directory: /tmp/cloudberry-cores" - echo "Contents of cores directory:" - ls -Rl "/tmp/cloudberry-cores" - echo "-----------------------------------------" - - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" - core_analysis_rc=$? - case "$core_analysis_rc" in - 0) echo "No core dumps found for this configuration" ;; - 1) echo "Core dumps were found and analyzed successfully" ;; - 2) echo "::warning::Issues encountered during core dump analysis" ;; - *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; - esac - - echo "Log file: $config_log" - echo "=== End configuration $((i+1)) execution ===" - echo "" - done - - echo "=== Test execution completed ===" - echo "Log files:" - ls -l build-logs/details/ - - # Store number of configurations for parsing step - echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" - - # Report overall status - if [ $overall_status -eq 0 ]; then - echo "All test executions completed successfully" - else - echo "::warning::Some test executions failed, check individual logs for details" - fi - - exit $overall_status - - - name: "Parse Test Results: ${{ matrix.test }}" - id: test-results - if: always() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - shell: bash {0} - run: | - set -o pipefail - - overall_status=0 - - # Get configs array to create context for results - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting results parsing for ${{ matrix.test }} ===" - echo "Number of configurations to parse: ${#configs[@]}" - echo "" - - # Parse each configuration's results independently - for ((i=0; i "test_results.$i.txt" - overall_status=1 - continue - fi - - # Parse this configuration's results - - MAKE_NAME="${{ matrix.test }}-config$i" \ - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" - status_code=$? - - { - echo "SUITE_NAME=${{ matrix.test }}" - echo "DIR=${dir}" - echo "TARGET=${target}" - } >> test_results.txt - - # Process return code - case $status_code in - 0) # All tests passed - echo "All tests passed successfully" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - ;; - 1) # Tests failed but parsed successfully - echo "Test failures detected but properly parsed" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - overall_status=1 - ;; - 2) # Parse error or missing file - echo "::warning::Could not parse test results properly for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=parse_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - *) # Unexpected error - echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=unknown_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - esac - - echo "Results stored in test_results.$i.txt" - echo "=== End parsing for configuration $((i+1)) ===" - echo "" - done - - # Report status of results files - echo "=== Results file status ===" - echo "Generated results files:" - for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-logs-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Test Metadata - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-metadata-${{ matrix.test }}-rocky8 - path: | - test_results*.txt - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test results files - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test regression logs - if: failure() || cancelled() - uses: actions/upload-artifact@v4 - with: - name: regression-logs-${{ matrix.test }}-rocky8-${{ needs.build.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - gpAux/gpdemo/datadirs/standby/log/ - gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ - gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ - gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - ## ====================================================================== - ## Job: report - ## ====================================================================== - - report: - name: Generate Apache Cloudberry Build Report (Rocky 8) - needs: [check-skip, build, prepare-test-matrix, rpm-install-test, test] - if: always() - runs-on: ubuntu-22.04 - steps: - - name: Generate Final Report - run: | - { - echo "# Apache Cloudberry Build Pipeline Report (Rocky 8)" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## CI Skip Status" - echo "✅ CI checks skipped via skip flag" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - else - echo "## Job Status" - echo "- Build Job: ${{ needs.build.result }}" - echo "- Test Job: ${{ needs.test.result }}" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.build.result }}" == "success" && "${{ needs.test.result }}" == "success" ]]; then - echo "✅ Pipeline completed successfully" - else - echo "⚠️ Pipeline completed with failures" - - if [[ "${{ needs.build.result }}" != "success" ]]; then - echo "### Build Job Failure" - echo "Check build logs for details" - fi - - if [[ "${{ needs.test.result }}" != "success" ]]; then - echo "### Test Job Failure" - echo "Check test logs and regression files for details" - fi - fi - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Notify on failure - if: | - needs.check-skip.outputs.should_skip != 'true' && - (needs.build.result != 'success' || needs.test.result != 'success') - run: | - echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" - echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "Build Result: ${{ needs.build.result }}" - echo "Test Result: ${{ needs.test.result }}" diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 8cb0cbb2665..7abb658632d 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -17,13 +17,14 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -# GitHub Actions Workflow: Apache Cloudberry Build Pipeline +# GitHub Actions Workflow: Apache Cloudberry Build Pipeline (Rocky Linux) # -------------------------------------------------------------------- # Description: # # This workflow builds, tests, and packages Apache Cloudberry on -# Rocky Linux 9. It ensures artifact integrity, performs installation -# tests, validates key operations, and provides detailed test reports, +# Rocky Linux 8, 9, and 10 using a consolidated matrix strategy. +# It ensures artifact integrity, performs installation tests, +# validates key operations, and provides detailed test reports, # including handling for ignored test cases. # # Workflow Overview: @@ -39,19 +40,19 @@ # - **Example Usage**: # - Add `[skip ci]` to a commit message, PR title, or body to skip the workflow. # -# 2. **Build Job**: -# - Configures and builds Apache Cloudberry. +# 2. **Build Job (Matrix: Rocky 8/9/10)**: +# - Configures and builds Apache Cloudberry for each Rocky version. # - Supports debug build configuration via ENABLE_DEBUG flag. # - Runs unit tests and verifies build artifacts. # - Creates RPM packages (regular or debug), source tarballs, and logs. # - **Key Artifacts**: RPM package, source tarball, build logs. # -# 3. **RPM Install Test Job**: +# 3. **RPM Install Test Job (Matrix: Rocky 8/9/10)**: # - Verifies RPM integrity and installs Cloudberry. # - Validates successful installation. # - **Key Artifacts**: Installation logs, verification results. # -# 4. **Test Job (Matrix)**: +# 4. **Test Job (Matrix: Rocky 8/9/10 × Test Configurations)**: # - Executes a test matrix to validate different scenarios. # - Creates a demo cluster and runs installcheck tests. # - Parses and reports test results, including failed and ignored tests. @@ -67,20 +68,20 @@ # - Sends failure notifications if any step fails. # # Execution Environment: -# - **Runs On**: ubuntu-22.04 with Rocky Linux 9 containers. +# - **Runs On**: ubuntu-22.04 with Rocky Linux 8/9/10 containers. # - **Resource Requirements**: # - Disk: Minimum 20GB free space. # - Memory: Minimum 8GB RAM. # - CPU: Recommended 4+ cores. # # Triggers: -# - Push to `main` branch. -# - Pull requests to `main` branch. +# - Push to `main` or `REL_2_STABLE` branch. +# - Pull requests to `main` or `REL_2_STABLE` branch. # - Manual workflow dispatch. # -# Container Images: -# - **Build**: `apache/incubator-cloudberry:cbdb-build-rocky9-latest` -# - **Test**: `apache/incubator-cloudberry:cbdb-test-rocky9-latest` +# Container Images (per Rocky version): +# - **Build**: `apache/incubator-cloudberry:cbdb-build-rocky{version}-latest` +# - **Test**: `apache/incubator-cloudberry:cbdb-test-rocky{version}-latest` # # Artifacts: # - RPM Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). @@ -90,15 +91,16 @@ # - Core Dump Analyses (retention: ${{ env.LOG_RETENTION_DAYS }} days). # # Notes: -# - Supports concurrent job execution. +# - Supports concurrent job execution across Rocky versions. # - Includes robust skip logic for pull requests and pushes. # - Handles ignored test cases, ensuring results are comprehensive. # - Provides detailed logs and error handling for failed and ignored tests. # - Analyzes core dumps generated during test execution. # - Supports debug builds with preserved symbols. +# - All Rocky versions run the same set of tests for maximum coverage. # -------------------------------------------------------------------- -name: Apache Cloudberry Build +name: Apache Cloudberry Build (Rocky Linux) on: push: @@ -108,6 +110,11 @@ on: types: [opened, synchronize, reopened, edited] workflow_dispatch: inputs: + rocky_version: + description: 'Rocky Linux version (default: all)' + required: false + default: 'all' + type: string test_selection: description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' required: false @@ -216,6 +223,7 @@ jobs: run: | echo "=== Matrix Preparation Diagnostics ===" echo "Event type: ${{ github.event_name }}" + echo "Rocky version input: '${{ github.event.inputs.rocky_version }}'" echo "Test selection input: '${{ github.event.inputs.test_selection }}'" # Define defaults @@ -224,7 +232,7 @@ jobs: DEFAULT_ENABLE_CORE_CHECK=true DEFAULT_PG_SETTINGS_OPTIMIZER="" - # Define base test configurations + # Define base test configurations (union of all Rocky 8/9/10 tests) ALL_TESTS='{ "include": [ {"test":"ic-good-opt-off", @@ -275,6 +283,11 @@ jobs: "make_configs":["gpcontrib/gp_stats_collector:installcheck"], "extension":"gp_stats_collector" }, + {"test":"gpcontrib-gp-relsizes-stats", + "make_configs":["gpcontrib/gp_relsizes_stats:installcheck"], + "extension":"gp_relsizes_stats", + "shared_preload_libraries":"gp_relsizes_stats" + }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] }, @@ -347,6 +360,10 @@ jobs: }, {"test":"ic-orca-parallel", "make_configs":["src/test/regress:installcheck-orca-parallel"] + }, + {"test":"ic-recovery", + "make_configs":["src/test/recovery:installcheck"], + "enable_core_check":false } ] }' @@ -369,6 +386,22 @@ jobs: get_defaults * .' } + # Determine rocky versions + ROCKY_INPUT="${{ github.event.inputs.rocky_version }}" + if [[ "$ROCKY_INPUT" == "all" || -z "$ROCKY_INPUT" ]]; then + ROCKY_VERSIONS_JSON='["8","9","10"]' + else + ROCKY_VERSIONS_JSON=$(echo "$ROCKY_INPUT" | jq -R 'split(",") | map(gsub("\\s+";"")) | map(select(length > 0)) | unique') + # Validate rocky versions + VALID_VERSIONS='["8","9","10"]' + INVALID_VERSIONS=$(echo "$ROCKY_VERSIONS_JSON" | jq -r --argjson valid "$VALID_VERSIONS" '[.[] | select(. as $v | $valid | index($v) | not)] | join(",")') + if [[ -n "$INVALID_VERSIONS" ]]; then + echo "::error::Invalid Rocky version(s): $INVALID_VERSIONS (valid: 8,9,10)" + exit 1 + fi + fi + echo "Rocky versions: $ROCKY_VERSIONS_JSON" + # Extract all valid test names from ALL_TESTS VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') @@ -414,14 +447,21 @@ jobs: done RESULT="${RESULT}]}" + # Expand by rocky versions: cross-join each test with each rocky version + FINAL_RESULT=$(jq -n \ + --argjson tests "$RESULT" \ + --argjson versions "$ROCKY_VERSIONS_JSON" \ + '[ $tests.include[] as $t | $versions[] as $v | $t + {rocky_version: $v} ] | {include: .}') + # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . + echo "Final matrix configuration (first 3 entries):" + echo "$FINAL_RESULT" | jq '.include[:3]' + echo "Total matrix entries: $(echo "$FINAL_RESULT" | jq '.include | length')" # Fix: Use block redirection { echo "matrix<> "$GITHUB_OUTPUT" @@ -432,18 +472,21 @@ jobs: ## ====================================================================== build: - name: Build Apache Cloudberry RPM + name: ${{ matrix.rocky_version == '9' && 'Build Apache Cloudberry RPM' || format('Build RPM (Rocky {0})', matrix.rocky_version) }} env: JOB_TYPE: build needs: [check-skip] runs-on: ubuntu-22.04 timeout-minutes: 120 if: github.event.inputs.reuse_artifacts_from_run_id == '' - outputs: - build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + + strategy: + fail-fast: false + matrix: + rocky_version: ['8', '9', '10'] container: - image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + image: apache/incubator-cloudberry:cbdb-build-rocky${{ matrix.rocky_version }}-latest options: >- --user root -h cdw @@ -490,7 +533,7 @@ jobs: - name: Checkout Apache Cloudberry if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 submodules: true @@ -532,7 +575,7 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' run: | { - echo "# Build Job Summary" + echo "# Build Job Summary (Rocky ${{ matrix.rocky_version }})" echo "## Environment" echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" @@ -655,7 +698,10 @@ jobs: "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${CBDB_VERSION}" --release "${BUILD_NUMBER}" "${DEBUG_RPMBUILD_OPT}" # Get OS version and move RPM - os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) + # build-rpm.sh normalizes the published file name to the historical + # format (without the "-" segment); the package Name metadata + # still embeds the major version. + os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release) RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm cp "${RPM_FILE}" "${SRC_DIR}" RPM_DEBUG="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-debuginfo-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm @@ -668,7 +714,7 @@ jobs: # Verify critical files in RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -705,18 +751,18 @@ jobs: - name: Upload build logs if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: build-logs-${{ env.BUILD_TIMESTAMP }} + name: build-logs-rocky${{ matrix.rocky_version }}-${{ env.BUILD_TIMESTAMP }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky${{ matrix.rocky_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -724,9 +770,9 @@ jobs: - name: Upload Cloudberry source build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: apache-cloudberry-db-incubating-source-build-artifacts + name: apache-cloudberry-db-incubating-source-build-artifacts-rocky${{ matrix.rocky_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -737,7 +783,7 @@ jobs: ## ====================================================================== rpm-install-test: - name: RPM Install Test Apache Cloudberry + name: ${{ matrix.rocky_version == '9' && 'RPM Install Test Apache Cloudberry' || format('RPM Install Test (Rocky {0})', matrix.rocky_version) }} needs: [check-skip, build] if: | !cancelled() && @@ -746,8 +792,13 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + rocky_version: ['8', '9', '10'] + container: - image: apache/incubator-cloudberry:cbdb-test-rocky9-latest + image: apache/incubator-cloudberry:cbdb-test-rocky${{ matrix.rocky_version }}-latest options: >- --user root -h cdw @@ -786,9 +837,9 @@ jobs: - name: Download Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky${{ matrix.rocky_version }} path: ${{ github.workspace }}/rpm_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -860,7 +911,7 @@ jobs: # Verify expected binaries are in the RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -906,24 +957,24 @@ jobs: rm -rf /usr/local/cloudberry-db # Install RPM with retry logic for mirror issues - # Use --releasever=9 to pin to stable Rocky Linux 9 repos (not bleeding-edge 9.6) + # Use --releasever to pin to the correct Rocky Linux repo version echo "Starting installation..." - if ! time dnf install -y --setopt=retries=10 --releasever=9 "${RPM_FILE}"; then + if ! time dnf install -y --setopt=retries=10 --releasever=${{ matrix.rocky_version }} "${RPM_FILE}"; then echo "::error::RPM installation failed" exit 1 fi echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi echo "Installed files:" - rpm -ql apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -ql } 2>&1 | tee -a install-logs/details/rpm-installation.log - name: Upload install logs if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: install-logs-${{ needs.build.outputs.build_timestamp }} + name: install-logs-rocky${{ matrix.rocky_version }} path: | install-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -936,7 +987,7 @@ jobs: echo "# Installed Package Summary" echo "\`\`\`" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi echo "\`\`\`" } >> "$GITHUB_STEP_SUMMARY" || true @@ -945,7 +996,7 @@ jobs: ## ====================================================================== test: - name: ${{ matrix.test }} + name: ${{ matrix.rocky_version == '9' && matrix.test || format('{0} (Rocky {1})', matrix.test, matrix.rocky_version) }} needs: [check-skip, build, prepare-test-matrix] if: | !cancelled() && @@ -958,7 +1009,7 @@ jobs: matrix: ${{ fromJson(needs.prepare-test-matrix.outputs.test-matrix) }} container: - image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + image: apache/incubator-cloudberry:cbdb-build-rocky${{ matrix.rocky_version }}-latest options: >- --privileged --user root @@ -1000,11 +1051,6 @@ jobs: echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" exit 0 - - name: Use timestamp from previous job - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "Timestamp from output: ${{ needs.build.outputs.build_timestamp }}" - - name: Cloudberry Environment Initialization env: LOGS_DIR: build-logs @@ -1185,7 +1231,7 @@ jobs: if: always() run: | { - echo "# Test Job Summary: ${{ matrix.test }}" + echo "# Test Job Summary: ${{ matrix.test }} (Rocky ${{ matrix.rocky_version }})" echo "## Environment" echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" @@ -1199,9 +1245,9 @@ jobs: - name: Download Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-rpm-build-artifacts + name: apache-cloudberry-db-incubating-rpm-build-artifacts-rocky${{ matrix.rocky_version }} path: ${{ github.workspace }}/rpm_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1209,9 +1255,9 @@ jobs: - name: Download Cloudberry Source build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-source-build-artifacts + name: apache-cloudberry-db-incubating-source-build-artifacts-rocky${{ matrix.rocky_version }} path: ${{ github.workspace }}/source_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1270,7 +1316,7 @@ jobs: # Verify expected binaries are in the RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -1316,15 +1362,15 @@ jobs: rm -rf /usr/local/cloudberry-db # Install RPM with retry logic for mirror issues - # Use --releasever=9 to pin to stable Rocky Linux 9 repos (not bleeding-edge 9.6) + # Use --releasever to pin to the correct Rocky Linux repo version echo "Starting installation..." - if ! time dnf install -y --setopt=retries=10 --releasever=9 "${RPM_FILE}"; then + if ! time dnf install -y --setopt=retries=10 --releasever=${{ matrix.rocky_version }} "${RPM_FILE}"; then echo "::error::RPM installation failed" exit 1 fi echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi } 2>&1 | tee -a build-logs/details/rpm-installation.log # Clean up downloaded RPM artifacts to free disk space @@ -1417,6 +1463,10 @@ jobs: run: | set -o pipefail + # Grant gpadmin write access to the install directory + # -H follows the command-line symlink to the real directory. + chown -RH gpadmin:gpadmin "${BUILD_DESTINATION}/" + # Initialize test status overall_status=0 @@ -1459,6 +1509,20 @@ jobs: exit 1 fi ;; + gp_relsizes_stats) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_relsizes_stats' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_relsizes_stats; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_relsizes_stats extension" + exit 1 + fi + ;; *) echo "Unknown extension: ${{ matrix.extension }}" exit 1 @@ -1852,26 +1916,26 @@ jobs: - name: Upload test logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: test-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} + name: test-logs-${{ matrix.test }}-rocky${{ matrix.rocky_version }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload Test Metadata if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: test-metadata-${{ matrix.test }} + name: test-metadata-${{ matrix.test }}-rocky${{ matrix.rocky_version }} path: | test_results*.txt retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload test results files - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: results-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} + name: results-${{ matrix.test }}-rocky${{ matrix.rocky_version }} path: | **/regression.out **/regression.diffs @@ -1880,9 +1944,9 @@ jobs: - name: Upload test regression logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: regression-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} + name: regression-logs-${{ matrix.test }}-rocky${{ matrix.rocky_version }} path: | **/regression.out **/regression.diffs @@ -1910,7 +1974,7 @@ jobs: - name: Generate Final Report run: | { - echo "# Apache Cloudberry Build Pipeline Report" + echo "# Apache Cloudberry Build Pipeline Report (Rocky Linux)" if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then echo "## CI Skip Status" diff --git a/.github/workflows/build-dbg-cloudberry.yml b/.github/workflows/build-dbg-cloudberry.yml index 967fc259f0b..4c3c4fd0e27 100644 --- a/.github/workflows/build-dbg-cloudberry.yml +++ b/.github/workflows/build-dbg-cloudberry.yml @@ -338,7 +338,7 @@ jobs: - name: Checkout Apache Cloudberry if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 submodules: true @@ -503,7 +503,10 @@ jobs: "${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${CBDB_VERSION}" --release "${BUILD_NUMBER}" "${DEBUG_RPMBUILD_OPT}" # Get OS version and move RPM - os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release) + # build-rpm.sh normalizes the published file name to the historical + # format (without the "-" segment); the package Name metadata + # still embeds the major version. + os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release) RPM_FILE="${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-"${CBDB_VERSION}"-"${BUILD_NUMBER}""${DEBUG_IDENTIFIER}".el"${os_version}".x86_64.rpm cp "${RPM_FILE}" "${SRC_DIR}" @@ -514,7 +517,7 @@ jobs: # Verify critical files in RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -551,7 +554,7 @@ jobs: - name: Upload build logs if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: build-logs-${{ env.BUILD_TIMESTAMP }} path: | @@ -560,7 +563,7 @@ jobs: - name: Upload Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: apache-cloudberry-db-incubating-rpm-build-artifacts retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -570,7 +573,7 @@ jobs: - name: Upload Cloudberry source build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: apache-cloudberry-db-incubating-source-build-artifacts retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -603,7 +606,7 @@ jobs: - name: Download Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: apache-cloudberry-db-incubating-rpm-build-artifacts path: ${{ github.workspace }}/rpm_build_artifacts @@ -675,7 +678,7 @@ jobs: # Verify expected binaries are in the RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -723,14 +726,14 @@ jobs: fi echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi echo "Installed files:" - rpm -ql apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -ql } 2>&1 | tee -a install-logs/details/rpm-installation.log - name: Upload install logs if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: install-logs-${{ needs.build.outputs.build_timestamp }} path: | @@ -745,7 +748,7 @@ jobs: echo "# Installed Package Summary" echo "\`\`\`" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi echo "\`\`\`" } >> "$GITHUB_STEP_SUMMARY" || true @@ -980,7 +983,7 @@ jobs: - name: Download Cloudberry RPM build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: apache-cloudberry-db-incubating-rpm-build-artifacts path: ${{ github.workspace }}/rpm_build_artifacts @@ -988,7 +991,7 @@ jobs: - name: Download Cloudberry Source build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: apache-cloudberry-db-incubating-source-build-artifacts path: ${{ github.workspace }}/source_build_artifacts @@ -1047,7 +1050,7 @@ jobs: # Verify expected binaries are in the RPM echo "Verifying critical files in RPM..." for binary in "bin/postgres" "bin/psql"; do - if ! rpm -qlp "${RPM_FILE}" | grep -q "${binary}$"; then + if ! rpm -qlp "${RPM_FILE}" | grep "${binary}$" >/dev/null 2>&1; then echo "::error::Critical binary '${binary}' not found in RPM" exit 1 fi @@ -1095,7 +1098,7 @@ jobs: fi echo "Installation completed successfully" - rpm -qi apache-cloudberry-db-incubating + rpm -qa 'apache-cloudberry-db-incubating*' | xargs -r rpm -qi } 2>&1 | tee -a build-logs/details/rpm-installation.log - name: Extract source tarball @@ -1553,7 +1556,7 @@ jobs: - name: Upload test logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} path: | @@ -1562,7 +1565,7 @@ jobs: - name: Upload Test Metadata if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-metadata-${{ matrix.test }} path: | @@ -1570,7 +1573,7 @@ jobs: retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload test results files - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: results-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} path: | @@ -1581,7 +1584,7 @@ jobs: - name: Upload test regression logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: regression-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }} path: | diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml deleted file mode 100644 index 072a0e77258..00000000000 --- a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml +++ /dev/null @@ -1,1893 +0,0 @@ -# -------------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed -# with this work for additional information regarding copyright -# ownership. The ASF licenses this file to You under the Apache -# License, Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. -# -# -------------------------------------------------------------------- -# GitHub Actions Workflow: Apache Cloudberry Build Pipeline -# -------------------------------------------------------------------- -# Description: -# -# This workflow builds, tests, and packages Apache Cloudberry on -# Ubuntu 24.04. It ensures artifact integrity and performs installation -# tests. -# -# Workflow Overview: -# 1. **Build Job**: -# - Configures and builds Apache Cloudberry. -# - Supports debug build configuration via ENABLE_DEBUG flag. -# - Runs unit tests and verifies build artifacts. -# - Creates DEB packages (regular and debug), source tarball -# and additional files for dupload utility. -# - **Key Artifacts**: DEB package, source tarball, changes and dsc files, build logs. -# -# 2. **DEB Install Test Job**: -# - Verifies DEB integrity and installs Cloudberry. -# - Validates successful installation. -# - **Key Artifacts**: Installation logs, verification results. -# -# 3. **Report Job**: -# - Aggregates job results into a final report. -# - Sends failure notifications if any step fails. -# -# Execution Environment: -# - **Runs On**: ubuntu-22.04 with ubuntu-24.04 containers. -# - **Resource Requirements**: -# - Disk: Minimum 20GB free space. -# - Memory: Minimum 8GB RAM. -# - CPU: Recommended 4+ cores. -# -# Triggers: -# - Push to `main` branch. -# - Pull request that modifies this workflow file. -# - Scheduled: Every Monday at 02:00 UTC. -# - Manual workflow dispatch. -# -# Container Images: -# - **Build**: `apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest` -# - **Test**: `apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest` -# -# Artifacts: -# - DEB Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Changes and DSC files (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# -# Notes: -# - Supports concurrent job execution. -# - Supports debug builds with preserved symbols. -# -------------------------------------------------------------------- - -name: Apache Cloudberry Debian Build - -on: - push: - branches: [main, REL_2_STABLE] - pull_request: - paths: - - '.github/workflows/build-deb-cloudberry-ubuntu24.04.yml' - # We can enable the PR test when needed - # branches: [main, REL_2_STABLE] - # types: [opened, synchronize, reopened, edited] - schedule: - # Run every Monday at 02:00 UTC - - cron: '0 2 * * 1' - workflow_dispatch: # Manual trigger - inputs: - test_selection: - description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' - required: false - default: 'all' - type: string - reuse_artifacts_from_run_id: - description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' - required: false - default: '' - type: string - -# Note: Step details, logs, and artifacts require users to be logged into GitHub -# even for public repositories. This is a GitHub security feature and cannot -# be overridden by permissions. - -permissions: - # READ permissions allow viewing repository contents - contents: read # Required for checking out code and reading repository files - - # READ permissions for packages (Container registry, etc) - packages: read # Allows reading from GitHub package registry - - # WRITE permissions for actions includes read access to: - # - Workflow runs - # - Artifacts (requires GitHub login) - # - Logs (requires GitHub login) - actions: write - - # READ permissions for checks API: - # - Step details visibility (requires GitHub login) - # - Check run status and details - checks: read - - # READ permissions for pull request metadata: - # - PR status - # - Associated checks - # - Review states - pull-requests: read - -env: - LOG_RETENTION_DAYS: 7 - ENABLE_DEBUG: false - -jobs: - - ## ====================================================================== - ## Job: check-skip - ## ====================================================================== - - check-skip: - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip-check.outputs.should_skip }} - steps: - - id: skip-check - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - PR_TITLE: ${{ github.event.pull_request.title || '' }} - PR_BODY: ${{ github.event.pull_request.body || '' }} - run: | - # Default to not skipping - echo "should_skip=false" >> "$GITHUB_OUTPUT" - - # Apply skip logic only for pull_request events - if [[ "$EVENT_NAME" == "pull_request" ]]; then - # Combine PR title and body for skip check - MESSAGE="${PR_TITLE}\n${PR_BODY}" - - # Escape special characters using printf %s - ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") - - echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" - - # Check for skip patterns - if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then - echo "should_skip=true" >> "$GITHUB_OUTPUT" - fi - else - echo "Skip logic is not applied for $EVENT_NAME events." - fi - - - name: Report Skip Status - if: steps.skip-check.outputs.should_skip == 'true' - run: | - echo "CI Skip flag detected in PR - skipping all checks." - exit 0 - - ## ====================================================================== - ## Job: prepare-test-matrix-deb - ## ====================================================================== - - prepare-test-matrix-deb: - runs-on: ubuntu-22.04 - needs: [check-skip] - if: needs.check-skip.outputs.should_skip != 'true' - outputs: - test-matrix: ${{ steps.set-matrix.outputs.matrix }} - - steps: - - id: set-matrix - run: | - echo "=== Matrix Preparation Diagnostics ===" - echo "Event type: ${{ github.event_name }}" - echo "Test selection input: '${{ github.event.inputs.test_selection }}'" - - # Define defaults - DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 - DEFAULT_ENABLE_CGROUPS=false - DEFAULT_ENABLE_CORE_CHECK=true - DEFAULT_PG_SETTINGS_OPTIMIZER="" - - # Define base test configurations - ALL_TESTS='{ - "include": [ - {"test":"ic-deb-good-opt-off", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"off"} - }, - {"test":"ic-deb-good-opt-on", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"on"} - }, - {"test":"pax-ic-deb-good-opt-off", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"off", - "default_table_access_method":"pax" - } - }, - {"test":"pax-ic-deb-good-opt-on", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"on", - "default_table_access_method":"pax" - } - }, - {"test":"ic-deb-contrib", - "make_configs":["contrib/auto_explain:installcheck", - "contrib/amcheck:installcheck", - "contrib/citext:installcheck", - "contrib/btree_gin:installcheck", - "contrib/btree_gist:installcheck", - "contrib/dblink:installcheck", - "contrib/dict_int:installcheck", - "contrib/dict_xsyn:installcheck", - "contrib/extprotocol:installcheck", - "contrib/file_fdw:installcheck", - "contrib/formatter_fixedwidth:installcheck", - "contrib/hstore:installcheck", - "contrib/indexscan:installcheck", - "contrib/pg_trgm:installcheck", - "contrib/indexscan:installcheck", - "contrib/pgcrypto:installcheck", - "contrib/pgstattuple:installcheck", - "contrib/tablefunc:installcheck", - "contrib/try_convert:installcheck", - "contrib/passwordcheck:installcheck", - "contrib/pg_buffercache:installcheck", - "contrib/sslinfo:installcheck"] - }, - {"test":"ic-deb-gpcontrib", - "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/zstd:installcheck", - "gpcontrib/gp_sparse_vector:installcheck", - "gpcontrib/gp_toolkit:installcheck"] - }, - {"test":"gpcontrib-gp-stats-collector", - "make_configs":["gpcontrib/gp_stats_collector:installcheck"], - "extension":"gp_stats_collector" - }, - {"test":"ic-cbdb-parallel", - "make_configs":["src/test/regress:installcheck-cbdb-parallel"] - } - ] - }' - - # Function to apply defaults - apply_defaults() { - echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ - --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ - --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ - --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ - 'def get_defaults: - { - num_primary_mirror_pairs: ($npm|tonumber), - enable_cgroups: $ec, - enable_core_check: $ecc, - pg_settings: { - optimizer: $opt - } - }; - get_defaults * .' - } - - # Extract all valid test names from ALL_TESTS - VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') - - # Parse input test selection - IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" - - # Default to all tests if selection is empty or 'all' - if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then - mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" - fi - - # Validate and filter selected tests - INVALID_TESTS=() - FILTERED_TESTS=() - for TEST in "${SELECTED_TESTS[@]}"; do - TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace - if echo "$VALID_TESTS" | grep -qw "$TEST"; then - FILTERED_TESTS+=("$TEST") - else - INVALID_TESTS+=("$TEST") - fi - done - - # Handle invalid tests - if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then - echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" - echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" - exit 1 - fi - - # Build result JSON with defaults applied - RESULT='{"include":[' - FIRST=true - for TEST in "${FILTERED_TESTS[@]}"; do - CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") - FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") - if [[ "$FIRST" == true ]]; then - FIRST=false - else - RESULT="${RESULT}," - fi - RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" - done - RESULT="${RESULT}]}" - - # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . - - # Fix: Use block redirection - { - echo "matrix<> "$GITHUB_OUTPUT" - - echo "=== Matrix Preparation Complete ===" - - ## ====================================================================== - ## Job: build-deb - ## ====================================================================== - - build-deb: - name: Build Apache Cloudberry DEB (Ubuntu 24.04) - env: - JOB_TYPE: build - needs: [check-skip] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - if: github.event.inputs.reuse_artifacts_from_run_id == '' - outputs: - build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} - - container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Set build timestamp - id: set_timestamp # Add an ID to reference this step - run: | - timestamp=$(date +'%Y%m%d_%H%M%S') - echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs - echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable - - - name: Checkout Apache Cloudberry - uses: actions/checkout@v4 - with: - fetch-depth: 1 - submodules: true - - - name: Cloudberry Environment Initialization - shell: bash - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Generate Build Job Summary Start - run: | - { - echo "# Build Job Summary (Ubuntu 24.04)" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" - echo "- OS Version: $(lsb_release -sd)" - echo "- GCC Version: $(gcc --version | head -n1)" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Run Apache Cloudberry configure script - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then - echo "::error::Configure script failed" - exit 1 - fi - - - name: Run Apache Cloudberry build script - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then - echo "::error::Build script failed" - exit 1 - fi - - - name: Verify build artifacts - shell: bash - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - echo "Verifying build artifacts..." - { - echo "=== Build Artifacts Verification ===" - echo "Timestamp: $(date -u)" - - if [ ! -d "${BUILD_DESTINATION}" ]; then - echo "::error::Build artifacts directory not found" - exit 1 - fi - - # Verify critical binaries - critical_binaries=( - "${BUILD_DESTINATION}/bin/postgres" - "${BUILD_DESTINATION}/bin/psql" - ) - - echo "Checking critical binaries..." - for binary in "${critical_binaries[@]}"; do - if [ ! -f "$binary" ]; then - echo "::error::Critical binary missing: $binary" - exit 1 - fi - if [ ! -x "$binary" ]; then - echo "::error::Binary not executable: $binary" - exit 1 - fi - echo "Binary verified: $binary" - ls -l "$binary" - done - - # Test binary execution - echo "Testing binary execution..." - if ! ${BUILD_DESTINATION}/bin/postgres --version; then - echo "::error::postgres binary verification failed" - exit 1 - fi - if ! ${BUILD_DESTINATION}/bin/psql --version; then - echo "::error::psql binary verification failed" - exit 1 - fi - - echo "All build artifacts verified successfully" - } 2>&1 | tee -a build-logs/details/build-verification.log - - - name: Create Source tarball, create DEB and verify artifacts - shell: bash - env: - CBDB_VERSION: 99.0.0 - BUILD_NUMBER: 1 - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Artifact Creation Log ===" - echo "Timestamp: $(date -u)" - - cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu24.04/* debian/ - chown -R "$(whoami)" debian - chmod -x debian/*install - - # replace not supported symbols in version - CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/\//./g") - CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/_/-/g") - - echo "We will built ${CBDB_VERSION}" - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - if ! ${SRC_DIR}/devops/build/packaging/deb/build-deb.sh -v $CBDB_VERSION; then - echo "::error::Build script failed" - exit 1 - fi - - ARCH=$(dpkg --print-architecture) - # Detect OS distribution (e.g., ubuntu24.04, debian12) - if [ -f /etc/os-release ]; then - . /etc/os-release - OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') - else - OS_DISTRO="unknown" - fi - CBDB_PKG_VERSION=${CBDB_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} - - echo "Produced artifacts" - ls -l ../ - - echo "Copy artifacts to subdirectory for sign/upload" - mkdir ${SRC_DIR}/deb - DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".deb - DBG_DEB_FILE="apache-cloudberry-db-incubating-dbgsym_${CBDB_PKG_VERSION}"_"${ARCH}".ddeb - CHANGES_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".changes - BUILDINFO_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".buildinfo - DSC_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".dsc - SOURCE_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".tar.xz - cp ../"${DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${DBG_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${CHANGES_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${BUILDINFO_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${DSC_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${SOURCE_FILE}" "${SRC_DIR}/deb" - mkdir "${SRC_DIR}/deb/debian" - cp debian/changelog "${SRC_DIR}/deb/debian" - - # Get package information - echo "Package Information:" - dpkg --info "${SRC_DIR}/deb/${DEB_FILE}" - dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" - - # Verify critical files in DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" | grep -c "${binary}$"; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - # Record checksums - echo "Calculating checksums..." - sha256sum "${SRC_DIR}/deb/${DEB_FILE}" | tee -a build-logs/details/checksums.log - - echo "Artifacts created and verified successfully" - - - } 2>&1 | tee -a build-logs/details/artifact-creation.log - - - name: Run Apache Cloudberry unittest script - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then - echo "::error::Unittest script failed" - exit 1 - fi - - - name: Generate Build Job Summary End - run: | - { - echo "## Build Results" - echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload build logs - uses: actions/upload-artifact@v4 - with: - name: build-logs-ubuntu24.04-${{ env.BUILD_TIMESTAMP }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Cloudberry DEB build artifacts - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - deb/*.deb - deb/*.ddeb - - - name: Upload Cloudberry deb source build artifacts - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - deb/*.tar.xz - deb/*.changes - deb/*.dsc - deb/*.buildinfo - deb/debian/changelog - - ## ====================================================================== - ## Job: deb-install-test - ## ====================================================================== - - deb-install-test: - name: DEB Install Test Apache Cloudberry (Ubuntu 24.04) - needs: [check-skip, build-deb] - if: | - !cancelled() && - (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') && - github.event.inputs.reuse_artifacts_from_run_id == '' - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - container: - image: apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "DEB install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Download Cloudberry DEB build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - path: ${{ github.workspace }}/deb_build_artifacts - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - merge-multiple: false - - - name: Cloudberry Environment Initialization - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - LOGS_DIR: install-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Verify DEB artifacts - id: verify-artifacts - shell: bash - run: | - set -eo pipefail - - DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) - if [ ! -f "${DEB_FILE}" ]; then - echo "::error::DEB file not found" - exit 1 - fi - - echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying DEB artifacts..." - { - echo "=== DEB Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - - # Get DEB metadata and verify contents - echo "Package Information:" - dpkg-deb -f "${DEB_FILE}" - - # Get key DEB attributes for verification - DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) - DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) - echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - echo "DEB Details:" - echo "- Version: ${DEB_VERSION}" - echo "- Release: ${DEB_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${DEB_FILE}" - - } 2>&1 | tee -a install-logs/details/deb-verification.log - - - name: Install Cloudberry DEB - shell: bash - env: - DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} - DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} - DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${DEB_FILE}" ]; then - echo "::error::DEB_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== DEB Installation Log ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - echo "Version: ${DEB_VERSION}" - echo "Release: ${DEB_RELEASE}" - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install DEB - echo "Starting installation..." - apt-get update - if ! apt-get -y install "${DEB_FILE}"; then - echo "::error::DEB installation failed" - exit 1 - fi - - # Change ownership back to gpadmin - it is needed for future tests - chown -R gpadmin:gpadmin /usr/local/cloudberry-db - - echo "Installation completed successfully" - dpkg-query -s apache-cloudberry-db-incubating - echo "Installed files:" - dpkg-query -L apache-cloudberry-db-incubating - } 2>&1 | tee -a install-logs/details/deb-installation.log - - - name: Upload install logs - uses: actions/upload-artifact@v4 - with: - name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - install-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Generate Install Test Job Summary End - if: always() - shell: bash {0} - run: | - { - echo "# Installed Package Summary (Ubuntu 24.04)" - echo "\`\`\`" - - dpkg-query -s apache-cloudberry-db-incubating - echo "\`\`\`" - } >> "$GITHUB_STEP_SUMMARY" || true - - ## ====================================================================== - ## Job: test-deb - ## ====================================================================== - - test-deb: - name: ${{ matrix.test }} (Ubuntu 24.04) - needs: [check-skip, build-deb, prepare-test-matrix-deb] - if: | - !cancelled() && - (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') - runs-on: ubuntu-22.04 - timeout-minutes: 120 - # actionlint-allow matrix[*].pg_settings - strategy: - fail-fast: false # Continue with other tests if one fails - matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} - - container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest - options: >- - --privileged - --user root - --hostname cdw - --shm-size=2gb - --ulimit core=-1 - --cgroupns=host - -v /sys/fs/cgroup:/sys/fs/cgroup:rw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Use timestamp from previous job - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "Timestamp from output: ${{ needs.build-deb.outputs.build_timestamp }}" - - - name: Cloudberry Environment Initialization - shell: bash - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Setup cgroups - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - run: | - set -uxo pipefail - - if [ "${{ matrix.enable_cgroups }}" = "true" ]; then - - echo "Current mounts:" - mount | grep cgroup - - CGROUP_BASEDIR=/sys/fs/cgroup - - # 1. Basic setup with permissions - sudo chmod -R 777 ${CGROUP_BASEDIR}/ - sudo mkdir -p ${CGROUP_BASEDIR}/gpdb - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 2. Enable controllers - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true - - # 3. CPU settings - sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true - sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true - sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true - - # 4. Memory settings - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true - - # 5. IO settings - echo "Available block devices:" - lsblk - - sudo bash -c " - if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then - echo 'Detected IO devices:' - cat \${CGROUP_BASEDIR}/gpdb/io.stat - fi - echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true - " - - # 6. Fix permissions again after all writes - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 7. Check required files - echo "Checking required files:" - required_files=( - "cgroup.procs" - "cpu.max" - "cpu.pressure" - "cpu.weight" - "cpu.weight.nice" - "cpu.stat" - "cpuset.cpus" - "cpuset.mems" - "cpuset.cpus.effective" - "cpuset.mems.effective" - "memory.current" - "io.max" - ) - - for file in "${required_files[@]}"; do - if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then - echo "✓ $file exists" - ls -l "${CGROUP_BASEDIR}/gpdb/$file" - else - echo "✗ $file missing" - fi - done - - # 8. Test subdirectory creation - echo "Testing subdirectory creation..." - sudo -u gpadmin bash -c " - TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 - if mkdir -p \$TEST_DIR; then - echo 'Created test directory' - sudo chmod -R 777 \$TEST_DIR - if echo \$\$ > \$TEST_DIR/cgroup.procs; then - echo 'Successfully wrote to cgroup.procs' - cat \$TEST_DIR/cgroup.procs - # Move processes back to parent before cleanup - echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs - else - echo 'Failed to write to cgroup.procs' - ls -la \$TEST_DIR/cgroup.procs - fi - ls -la \$TEST_DIR/ - rmdir \$TEST_DIR || { - echo 'Moving all processes to parent before cleanup' - cat \$TEST_DIR/cgroup.procs | while read pid; do - echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true - done - rmdir \$TEST_DIR - } - else - echo 'Failed to create test directory' - fi - " - - # 9. Verify setup as gpadmin user - echo "Testing cgroup access as gpadmin..." - sudo -u gpadmin bash -c " - echo 'Checking mounts...' - mount | grep cgroup - - echo 'Checking /proc/self/mounts...' - cat /proc/self/mounts | grep cgroup - - if ! grep -q cgroup2 /proc/self/mounts; then - echo 'ERROR: cgroup2 mount NOT visible to gpadmin' - exit 1 - fi - echo 'SUCCESS: cgroup2 mount visible to gpadmin' - - if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then - echo 'ERROR: gpadmin cannot write to gpdb cgroup' - exit 1 - fi - echo 'SUCCESS: gpadmin can write to gpdb cgroup' - - echo 'Verifying key files content:' - echo 'cpu.max:' - cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' - echo 'cpuset.cpus:' - cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' - echo 'cgroup.subtree_control:' - cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' - " - - # 10. Show final state - echo "Final cgroup state:" - ls -la ${CGROUP_BASEDIR}/gpdb/ - echo "Cgroup setup completed successfully" - else - echo "Cgroup setup skipped" - fi - - - name: "Generate Test Job Summary Start: ${{ matrix.test }}" - if: always() - run: | - { - echo "# Test Job Summary: ${{ matrix.test }} (Ubuntu 24.04)" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## Skip Status" - echo "✓ Test execution skipped via CI skip flag" - else - echo "- OS Version: $(cat /etc/redhat-release)" - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Download Cloudberry DEB build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - path: ${{ github.workspace }}/deb_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Download Cloudberry Source build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts - path: ${{ github.workspace }}/source_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Verify DEB artifacts - if: needs.check-skip.outputs.should_skip != 'true' - id: verify-artifacts - shell: bash - run: | - set -eo pipefail - - SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-db-incubating_*.tar.xz) - if [ ! -f "${SRC_TARBALL_FILE}" ]; then - echo "::error::SRC TARBALL file not found" - exit 1 - fi - - echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying SRC TARBALL artifacts..." - { - echo "=== SRC TARBALL Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${SRC_TARBALL_FILE}" - - } 2>&1 | tee -a build-logs/details/src-tarball-verification.log - - DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) - if [ ! -f "${DEB_FILE}" ]; then - echo "::error::DEB file not found" - exit 1 - fi - - echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying DEB artifacts..." - { - echo "=== DEB Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - - # Get DEB metadata and verify contents - echo "Package Information:" - dpkg-deb -f "${DEB_FILE}" - - # Get key DEB attributes for verification - DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) - DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) - echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - echo "DEB Details:" - echo "- Version: ${DEB_VERSION}" - echo "- Release: ${DEB_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${DEB_FILE}" - - } 2>&1 | tee -a build-logs/details/deb-verification.log - - - name: Install Cloudberry DEB - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} - DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} - DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${DEB_FILE}" ]; then - echo "::error::DEB_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== DEB Installation Log ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - echo "Version: ${DEB_VERSION}" - echo "Release: ${DEB_RELEASE}" - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install DEB - echo "Starting installation..." - apt-get update - if ! apt-get -y install "${DEB_FILE}"; then - echo "::error::DEB installation failed" - exit 1 - fi - - # Change ownership back to gpadmin - it is needed for future tests - chown -R gpadmin:gpadmin /usr/local/cloudberry-db - - echo "Installation completed successfully" - dpkg-query -s apache-cloudberry-db-incubating - echo "Installed files:" - dpkg-query -L apache-cloudberry-db-incubating - } 2>&1 | tee -a build-logs/details/deb-installation.log - - - name: Extract source tarball - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Source Extraction Log ===" - echo "Timestamp: $(date -u)" - - echo "Starting extraction..." - file "${SRC_TARBALL_FILE}" - if ! time tar xf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then - echo "::error::Source extraction failed" - exit 1 - fi - - echo "Extraction completed successfully" - echo "Extracted contents:" - ls -la "${SRC_DIR}/../cloudberry" - echo "Directory size:" - du -sh "${SRC_DIR}/../cloudberry" - } 2>&1 | tee -a build-logs/details/source-extraction.log - - - name: Prepare DEB Environment - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - - # change ownership to gpadmin - chown -R gpadmin "${SRC_DIR}/../cloudberry" - touch build-logs/sections.log - chown gpadmin build-logs/sections.log - chmod 777 build-logs - - # configure link lib directory to temporary location, fix it - rm -rf "${SRC_DIR}"/debian/build/lib - ln -sf /usr/cloudberry-db/lib "${SRC_DIR}"/debian/build/lib - - # check if regress.so exists in src directory - it is needed for contrib/dblink tests - if [ ! -f ${SRC_DIR}/src/test/regress/regress.so ]; then - ln -sf /usr/cloudberry-db/lib/postgresql/regress.so ${SRC_DIR}/src/test/regress/regress.so - fi - - # FIXME - # temporary install gdb - delete after creating new docker build/test contaners - apt-get update - apt-get -y install gdb - - } 2>&1 | tee -a build-logs/details/prepare-deb-env.log - - - name: Create Apache Cloudberry demo cluster - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh - - # Build BLDWRAP_POSTGRES_CONF_ADDONS for shared_preload_libraries if specified - EXTRA_CONF="" - if [[ -n "${{ matrix.shared_preload_libraries }}" ]]; then - EXTRA_CONF="shared_preload_libraries='${{ matrix.shared_preload_libraries }}'" - echo "Adding shared_preload_libraries: ${{ matrix.shared_preload_libraries }}" - fi - - if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' BLDWRAP_POSTGRES_CONF_ADDONS=\"${EXTRA_CONF}\" SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then - echo "::error::Demo cluster creation failed" - exit 1 - fi - - } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log - - - name: "Run Tests: ${{ matrix.test }}" - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - BUILD_DESTINATION: ${{ github.workspace }}/debian/build - shell: bash {0} - run: | - set -o pipefail - - # Initialize test status - overall_status=0 - - # Create logs directory structure - mkdir -p build-logs/details - - # Core file config - mkdir -p "/tmp/cloudberry-cores" - chmod 1777 "/tmp/cloudberry-cores" - sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" - sysctl kernel.core_pattern - su - gpadmin -c "ulimit -c" - - # WARNING: PostgreSQL Settings - # When adding new pg_settings key/value pairs: - # 1. Add a new check below for the setting - # 2. Follow the same pattern as optimizer - # 3. Update matrix entries to include the new setting - - - # Create extension if required - if [[ "${{ matrix.extension != '' }}" == "true" ]]; then - case "${{ matrix.extension }}" in - gp_stats_collector) - if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ - source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ - gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ - gpstop -ra && \ - echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ - SHOW shared_preload_libraries; \ - TABLE pg_extension;' | \ - psql postgres" - then - echo "Error creating gp_stats_collector extension" - exit 1 - fi - ;; - *) - echo "Unknown extension: ${{ matrix.extension }}" - exit 1 - ;; - esac - fi - - # Set PostgreSQL options if defined - PG_OPTS="" - if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" - fi - - if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" - fi - - # Read configs into array - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting test execution for ${{ matrix.test }} ===" - echo "Number of configurations to execute: ${#configs[@]}" - echo "" - - # Execute each config separately - for ((i=0; i<${#configs[@]}; i++)); do - config="${configs[$i]}" - IFS=':' read -r dir target <<< "$config" - - echo "=== Executing configuration $((i+1))/${#configs[@]} ===" - echo "Make command: make -C $dir $target" - echo "Environment:" - echo "- PGOPTIONS: ${PG_OPTS}" - - # Create unique log file for this configuration - config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" - - # Clean up any existing core files - echo "Cleaning up existing core files..." - rm -f /tmp/cloudberry-cores/core-* - - # Execute test script with proper environment setup - if ! time su - gpadmin -c "cd ${SRC_DIR} && \ - MAKE_NAME='${{ matrix.test }}-config$i' \ - MAKE_TARGET='$target' \ - MAKE_DIRECTORY='-C $dir' \ - PGOPTIONS='${PG_OPTS}' \ - SRC_DIR='${SRC_DIR}' \ - ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ - 2>&1 | tee "$config_log"; then - echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" - overall_status=1 - fi - - # Check for results directory - results_dir="${dir}/results" - - if [[ -d "$results_dir" ]]; then - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - - find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - else - echo "-----------------------------------------" - echo "Results directory $results_dir does not exit" - echo "-----------------------------------------" - fi - - # Analyze any core files generated by this test configuration - echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." - test_id="${{ matrix.test }}-config$i" - - # List the cores directory - echo "-----------------------------------------" - echo "Cores directory: /tmp/cloudberry-cores" - echo "Contents of cores directory:" - ls -Rl "/tmp/cloudberry-cores" - echo "-----------------------------------------" - - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" - core_analysis_rc=$? - case "$core_analysis_rc" in - 0) echo "No core dumps found for this configuration" ;; - 1) echo "Core dumps were found and analyzed successfully" ;; - 2) echo "::warning::Issues encountered during core dump analysis" ;; - *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; - esac - - echo "Log file: $config_log" - echo "=== End configuration $((i+1)) execution ===" - echo "" - done - - echo "=== Test execution completed ===" - echo "Log files:" - ls -l build-logs/details/ - - # Store number of configurations for parsing step - echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" - - # Report overall status - if [ $overall_status -eq 0 ]; then - echo "All test executions completed successfully" - else - echo "::warning::Some test executions failed, check individual logs for details" - fi - - exit $overall_status - - - name: "Parse Test Results: ${{ matrix.test }}" - id: test-results - if: always() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - shell: bash {0} - run: | - set -o pipefail - - overall_status=0 - - # Get configs array to create context for results - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting results parsing for ${{ matrix.test }} ===" - echo "Number of configurations to parse: ${#configs[@]}" - echo "" - - # Parse each configuration's results independently - for ((i=0; i "test_results.$i.txt" - overall_status=1 - continue - fi - - # Parse this configuration's results - - MAKE_NAME="${{ matrix.test }}-config$i" \ - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" - status_code=$? - - { - echo "SUITE_NAME=${{ matrix.test }}" - echo "DIR=${dir}" - echo "TARGET=${target}" - } >> test_results.txt - - # Process return code - case $status_code in - 0) # All tests passed - echo "All tests passed successfully" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - ;; - 1) # Tests failed but parsed successfully - echo "Test failures detected but properly parsed" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - overall_status=1 - ;; - 2) # Parse error or missing file - echo "::warning::Could not parse test results properly for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=parse_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - *) # Unexpected error - echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=unknown_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - esac - - echo "Results stored in test_results.$i.txt" - echo "=== End parsing for configuration $((i+1)) ===" - echo "" - done - - # Report status of results files - echo "=== Results file status ===" - echo "Generated results files:" - for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Test Metadata - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-metadata-${{ matrix.test }} - path: | - test_results*.txt - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test results files - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test regression logs - if: failure() || cancelled() - uses: actions/upload-artifact@v4 - with: - name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - gpAux/gpdemo/datadirs/standby/log/ - gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ - gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ - gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - ## ====================================================================== - ## Job: report-deb - ## ====================================================================== - - report-deb: - name: Generate Apache Cloudberry Build Report (Ubuntu 24.04) - needs: [check-skip, build-deb, prepare-test-matrix-deb, deb-install-test, test-deb] - if: always() - runs-on: ubuntu-22.04 - steps: - - name: Generate Final Report - run: | - { - echo "# Apache Cloudberry Build Pipeline Report" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## CI Skip Status" - echo "✅ CI checks skipped via skip flag" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - else - echo "## Job Status" - echo "- Build Job: ${{ needs.build-deb.result }}" - echo "- Test Job: ${{ needs.test-deb.result }}" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.build-deb.result }}" == "success" && "${{ needs.test-deb.result }}" == "success" ]]; then - echo "✅ Pipeline completed successfully" - else - echo "⚠️ Pipeline completed with failures" - - if [[ "${{ needs.build-deb.result }}" != "success" ]]; then - echo "### Build Job Failure" - echo "Check build logs for details" - fi - - if [[ "${{ needs.test-deb.result }}" != "success" ]]; then - echo "### Test Job Failure" - echo "Check test logs and regression files for details" - fi - fi - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Notify on failure - if: | - needs.check-skip.outputs.should_skip != 'true' && - (needs.build-deb.result != 'success' || needs.test-deb.result != 'success') - run: | - echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" - echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "Build Result: ${{ needs.build-deb.result }}" - echo "Test Result: ${{ needs.test-deb.result }}" diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index b01eb2a1385..e4232cd4a04 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -80,6 +80,11 @@ on: types: [opened, synchronize, reopened, edited] workflow_dispatch: # Manual trigger inputs: + ubuntu_version: + description: 'Ubuntu version (default: all)' + required: false + default: 'all' + type: string test_selection: description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' required: false @@ -281,6 +286,15 @@ jobs: get_defaults * .' } + # Determine ubuntu versions + UBUNTU_INPUT="${{ github.event.inputs.ubuntu_version }}" + if [[ "$UBUNTU_INPUT" == "all" || -z "$UBUNTU_INPUT" ]]; then + UBUNTU_VERSIONS_JSON='["22.04","24.04"]' + else + UBUNTU_VERSIONS_JSON=$(echo "$UBUNTU_INPUT" | jq -R 'split(",") | map(gsub("\\s+";"")) | map(select(length > 0)) | unique') + fi + echo "Ubuntu versions: $UBUNTU_VERSIONS_JSON" + # Extract all valid test names from ALL_TESTS VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') @@ -326,14 +340,21 @@ jobs: done RESULT="${RESULT}]}" + # Expand by ubuntu versions: cross-join each test with each ubuntu version + FINAL_RESULT=$(jq -n \ + --argjson tests "$RESULT" \ + --argjson versions "$UBUNTU_VERSIONS_JSON" \ + '[ $tests.include[] as $t | $versions[] as $v | $t + {ubuntu_version: $v} ] | {include: .}') + # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . + echo "Final matrix configuration (first 3 entries):" + echo "$FINAL_RESULT" | jq '.include[:3]' + echo "Total matrix entries: $(echo "$FINAL_RESULT" | jq '.include | length')" # Fix: Use block redirection { echo "matrix<> "$GITHUB_OUTPUT" @@ -344,7 +365,7 @@ jobs: ## ====================================================================== build-deb: - name: Build Apache Cloudberry DEB + name: ${{ matrix.ubuntu_version == '22.04' && 'Build Apache Cloudberry DEB' || format('Build Apache Cloudberry DEB (Ubuntu {0})', matrix.ubuntu_version) }} env: JOB_TYPE: build needs: [check-skip] @@ -354,8 +375,13 @@ jobs: outputs: build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + strategy: + fail-fast: false + matrix: + ubuntu_version: ['22.04', '24.04'] + container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-build-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --user root -h cdw @@ -400,7 +426,7 @@ jobs: echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable - name: Checkout Apache Cloudberry - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 submodules: true @@ -543,7 +569,7 @@ jobs: echo "=== Artifact Creation Log ===" echo "Timestamp: $(date -u)" - cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu22.04/* debian/ + cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu${{ matrix.ubuntu_version }}/* debian/ chown -R "$(whoami)" debian chmod -x debian/*install @@ -633,17 +659,17 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload build logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: build-logs-${{ env.BUILD_TIMESTAMP }} + name: build-logs-ubuntu${{ matrix.ubuntu_version }}-${{ env.BUILD_TIMESTAMP }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload Cloudberry DEB build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -651,9 +677,9 @@ jobs: deb/*.ddeb - name: Upload Cloudberry deb source build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts + name: apache-cloudberry-db-incubating-deb-source-build-artifacts-ubuntu${{ matrix.ubuntu_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -668,7 +694,7 @@ jobs: ## ====================================================================== deb-install-test: - name: DEB Install Test Apache Cloudberry + name: ${{ matrix.ubuntu_version == '22.04' && 'DEB Install Test Apache Cloudberry' || format('DEB Install Test Apache Cloudberry (Ubuntu {0})', matrix.ubuntu_version) }} needs: [check-skip, build-deb] if: | !cancelled() && @@ -677,8 +703,13 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + ubuntu_version: ['22.04', '24.04'] + container: - image: apache/incubator-cloudberry:cbdb-test-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-test-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --user root -h cdw @@ -717,12 +748,13 @@ jobs: - name: Download Cloudberry DEB build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/deb_build_artifacts - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Cloudberry Environment Initialization if: needs.check-skip.outputs.should_skip != 'true' @@ -849,9 +881,9 @@ jobs: } 2>&1 | tee -a install-logs/details/deb-installation.log - name: Upload install logs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} + name: install-logs-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | install-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -873,7 +905,7 @@ jobs: ## ====================================================================== test-deb: - name: ${{ matrix.test }} + name: ${{ matrix.ubuntu_version == '22.04' && matrix.test || format('{0} (Ubuntu {1})', matrix.test, matrix.ubuntu_version) }} needs: [check-skip, build-deb, prepare-test-matrix-deb] if: | !cancelled() && @@ -886,7 +918,7 @@ jobs: matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-build-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --privileged --user root @@ -1128,9 +1160,9 @@ jobs: - name: Download Cloudberry DEB build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/deb_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1138,9 +1170,9 @@ jobs: - name: Download Cloudberry Source build artifacts if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts + name: apache-cloudberry-db-incubating-deb-source-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/source_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1787,26 +1819,26 @@ jobs: - name: Upload test logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: test-logs-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload Test Metadata if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: test-metadata-${{ matrix.test }} + name: test-metadata-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }} path: | test_results*.txt retention-days: ${{ env.LOG_RETENTION_DAYS }} - name: Upload test results files - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: results-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | **/regression.out **/regression.diffs @@ -1815,9 +1847,9 @@ jobs: - name: Upload test regression logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: regression-logs-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | **/regression.out **/regression.diffs diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 2b6a81c91f4..96d29d1faf4 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -59,7 +59,7 @@ jobs: steps: - name: Checkout Apache Cloudberry - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 submodules: true diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 538b4e9b179..71e2a403002 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -87,7 +87,7 @@ jobs: steps: # Checkout repository code with full history - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 # Generate version information for image tags # - BUILD_DATE: Current date in YYYYMMDD format diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 4d0fb8def33..eee8fafea2f 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -74,7 +74,7 @@ jobs: steps: # Checkout repository code - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 # Generate version information for image tags - name: Set version diff --git a/.github/workflows/package-convenience-binaries.yml b/.github/workflows/package-convenience-binaries.yml index 3b3ed736e17..d8ca3ac4922 100644 --- a/.github/workflows/package-convenience-binaries.yml +++ b/.github/workflows/package-convenience-binaries.yml @@ -242,7 +242,7 @@ jobs: - name: Checkout git ref for test packaging if: github.event.inputs.source_mode == 'git_ref_test' - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: ${{ github.event.inputs.git_ref }} fetch-depth: 1 @@ -313,7 +313,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload verified source release - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ steps.validate.outputs.artifact_name }} retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -343,6 +343,8 @@ jobs: - {target_os: rocky8, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest} - {target_os: rocky9, target_arch: x86_64, package_type: rpm, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest} - {target_os: rocky9, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest} + - {target_os: rocky10, target_arch: x86_64, package_type: rpm, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest} + - {target_os: rocky10, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest} # DEB Package - {target_os: ubuntu22.04, target_arch: x86_64, package_type: deb, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest} - {target_os: ubuntu22.04, target_arch: arm64, package_type: deb, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest} @@ -374,7 +376,7 @@ jobs: su - gpadmin -c "/tmp/init_system.sh" - name: Download verified source release - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: ${{ needs.verify-source-release.outputs.artifact_name }} path: ${{ github.workspace }}/verified-source @@ -520,7 +522,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Upload convenience package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: packages-${{ matrix.platform.target_os }}-${{ matrix.platform.target_arch }}-${{ matrix.platform.package_type }} retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -550,6 +552,8 @@ jobs: - {target_os: rocky8, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, test_container_image: apache/incubator-cloudberry:cbdb-test-rocky8-latest, label: rocky8-arm64-rpm} - {target_os: rocky9, target_arch: x86_64, package_type: rpm, runner: ubuntu-24.04, test_container_image: apache/incubator-cloudberry:cbdb-test-rocky9-latest, label: rocky9-x86_64-rpm} - {target_os: rocky9, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, test_container_image: apache/incubator-cloudberry:cbdb-test-rocky9-latest, label: rocky9-arm64-rpm} + - {target_os: rocky10, target_arch: x86_64, package_type: rpm, runner: ubuntu-24.04, test_container_image: apache/incubator-cloudberry:cbdb-test-rocky10-latest, label: rocky10-x86_64-rpm} + - {target_os: rocky10, target_arch: arm64, package_type: rpm, runner: ubuntu-24.04-arm, test_container_image: apache/incubator-cloudberry:cbdb-test-rocky10-latest, label: rocky10-arm64-rpm} # DEB Package - {target_os: ubuntu22.04, target_arch: x86_64, package_type: deb, runner: ubuntu-24.04, test_container_image: apache/incubator-cloudberry:cbdb-test-ubuntu22.04-latest, label: ubuntu22.04-x86_64-deb} - {target_os: ubuntu22.04, target_arch: arm64, package_type: deb, runner: ubuntu-24.04-arm, test_container_image: apache/incubator-cloudberry:cbdb-test-ubuntu22.04-latest, label: ubuntu22.04-arm64-deb} @@ -564,7 +568,7 @@ jobs: su - gpadmin -c "/tmp/init_system.sh" - name: Download package artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: packages-${{ matrix.platform.target_os }}-${{ matrix.platform.target_arch }}-${{ matrix.platform.package_type }} path: ${{ github.workspace }}/package-artifacts diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 93379d184ea..98645bd74f0 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -70,7 +70,7 @@ jobs: steps: - name: Checkout Apache Cloudberry - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis submodules: true @@ -83,7 +83,7 @@ jobs: fi - name: Install Build Wrapper - uses: SonarSource/sonarqube-scan-action/install-build-wrapper@v6 + uses: SonarSource/sonarqube-scan-action/install-build-wrapper@713881670b6b3676cda39549040e2d88c70d582e # v8.2.0 - name: Run Build Wrapper run: | @@ -122,7 +122,7 @@ jobs: - name: SonarQube Scan if: ${{ github.event_name != 'pull_request' }} - uses: SonarSource/sonarqube-scan-action@v6 + uses: SonarSource/sonarqube-scan-action@713881670b6b3676cda39549040e2d88c70d582e # v8.2.0 env: SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} with: diff --git a/contrib/interconnect/README.md b/contrib/interconnect/README.md index 26a220908b3..95ea75087f5 100644 --- a/contrib/interconnect/README.md +++ b/contrib/interconnect/README.md @@ -1,6 +1,6 @@ # Intercontect -This subtree contains interconnect module && test && benchmark that different with other subtree inside {cbdb_src}/contrib. Other moudles are not part of the core CloudBerry system, but interconnect module split from `cdb module`, it **must be preload with CloudBerry**, otherwise CloudBerry system will not work properly. +This subtree contains interconnect module && test && benchmark that different with other subtree inside {cbdb_src}/contrib. Other moudles are not part of the core Cloudberry system, but interconnect module split from `cdb module`, it **must be preload with Cloudberry**, otherwise Cloudberry system will not work properly. **The interconnect module will be preloaded by default as a library.** When the compile option `--disable-preload-ic-module` is turned on, then the interconnect module will not be preloaded, then users need to add `interconnect` into guc `shared_preload_libraries`. diff --git a/contrib/pax_storage/src/test/regress/expected/AOCO_Compression.out b/contrib/pax_storage/src/test/regress/expected/AOCO_Compression.out index 0b8917a4295..2e06c56b842 100644 --- a/contrib/pax_storage/src/test/regress/expected/AOCO_Compression.out +++ b/contrib/pax_storage/src/test/regress/expected/AOCO_Compression.out @@ -3572,6 +3572,7 @@ Access method: ao_column -- When I insert data insert into mpp17012_compress_test2 values('a',generate_series(1,250),'ksjdhfksdhfksdhfksjhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh','bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('mpp17012_compress_test2')), get_ao_compression_ratio('mpp17012_compress_test2'); pg_size_pretty | get_ao_compression_ratio @@ -3579,6 +3580,31 @@ get_ao_compression_ratio('mpp17012_compress_test2'); 712 bytes | 36.75 (1 row) +-- end_ignore +select + case + when + abs((pg_relation_size('mpp17012_compress_test2') - 712.0) / 712.0) < 0.1 + then 'test passed' + else 'test failed' + end as test_relation_size; + test_relation_size +-------------------- + test passed +(1 row) + +select + case + when + abs((get_ao_compression_ratio('mpp17012_compress_test2') - 36.75) / 36.75) < 0.1 + then 'test passed' + else 'test failed' + end as test_ao_compression_ratio; + test_ao_compression_ratio +--------------------------- + test passed +(1 row) + -- Test that an AO/CO table with compresstype zlib and invalid compress level will error at create create table a_aoco_table_with_zlib_and_invalid_compression_level(col text) WITH (APPENDONLY=true, COMPRESSTYPE=zlib, compresslevel=-1, ORIENTATION=column); ERROR: value -1 out of bounds for option "compresslevel" @@ -3604,6 +3630,7 @@ select pg_size_pretty(pg_relation_size('a_aoco_table_with_rle_type_compression') -- When I insert data insert into a_aoco_table_with_rle_type_compression select i from generate_series(1,100)i; -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('a_aoco_table_with_rle_type_compression')), get_ao_compression_ratio('a_aoco_table_with_rle_type_compression'); pg_size_pretty | get_ao_compression_ratio @@ -3611,6 +3638,31 @@ select pg_size_pretty(pg_relation_size('a_aoco_table_with_rle_type_compression') 296 bytes | 1.81 (1 row) +-- end_ignore +select + case + when + abs((pg_relation_size('a_aoco_table_with_rle_type_compression') - 296.0) / 296.0) < 0.1 + then 'test passed' + else 'test failed' + end as test_relation_size; + test_relation_size +-------------------- + test passed +(1 row) + +select + case + when + abs((get_ao_compression_ratio('a_aoco_table_with_rle_type_compression') - 1.81) / 1.81) < 0.1 + then 'test passed' + else 'test failed' + end as test_ao_compression_ratio; + test_ao_compression_ratio +--------------------------- + test passed +(1 row) + -- Test that an AO/CO table with compresstype rle and invalid compress level will error at create create table a_aoco_table_with_rle_type_and_invalid_compression_level(col int) WITH (APPENDONLY=true, COMPRESSTYPE=rle_type, compresslevel=-1, ORIENTATION=column); ERROR: value -1 out of bounds for option "compresslevel" diff --git a/contrib/pax_storage/src/test/regress/sql/AOCO_Compression.sql b/contrib/pax_storage/src/test/regress/sql/AOCO_Compression.sql index 251da040730..6ce42e9eabe 100644 --- a/contrib/pax_storage/src/test/regress/sql/AOCO_Compression.sql +++ b/contrib/pax_storage/src/test/regress/sql/AOCO_Compression.sql @@ -1748,8 +1748,25 @@ get_ao_compression_ratio('mpp17012_compress_test2'); -- When I insert data insert into mpp17012_compress_test2 values('a',generate_series(1,250),'ksjdhfksdhfksdhfksjhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh','bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('mpp17012_compress_test2')), get_ao_compression_ratio('mpp17012_compress_test2'); +-- end_ignore +select + case + when + abs((pg_relation_size('mpp17012_compress_test2') - 712.0) / 712.0) < 0.1 + then 'test passed' + else 'test failed' + end as test_relation_size; + +select + case + when + abs((get_ao_compression_ratio('mpp17012_compress_test2') - 36.75) / 36.75) < 0.1 + then 'test passed' + else 'test failed' + end as test_ao_compression_ratio; -- Test that an AO/CO table with compresstype zlib and invalid compress level will error at create create table a_aoco_table_with_zlib_and_invalid_compression_level(col text) WITH (APPENDONLY=true, COMPRESSTYPE=zlib, compresslevel=-1, ORIENTATION=column); @@ -1764,8 +1781,25 @@ select pg_size_pretty(pg_relation_size('a_aoco_table_with_rle_type_compression') -- When I insert data insert into a_aoco_table_with_rle_type_compression select i from generate_series(1,100)i; -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('a_aoco_table_with_rle_type_compression')), get_ao_compression_ratio('a_aoco_table_with_rle_type_compression'); +-- end_ignore +select + case + when + abs((pg_relation_size('a_aoco_table_with_rle_type_compression') - 296.0) / 296.0) < 0.1 + then 'test passed' + else 'test failed' + end as test_relation_size; + +select + case + when + abs((get_ao_compression_ratio('a_aoco_table_with_rle_type_compression') - 1.81) / 1.81) < 0.1 + then 'test passed' + else 'test failed' + end as test_ao_compression_ratio; -- Test that an AO/CO table with compresstype rle and invalid compress level will error at create create table a_aoco_table_with_rle_type_and_invalid_compression_level(col int) WITH (APPENDONLY=true, COMPRESSTYPE=rle_type, compresslevel=-1, ORIENTATION=column); diff --git a/contrib/udp2/README.md b/contrib/udp2/README.md index 9cfe9f9c989..c49400a3a79 100644 --- a/contrib/udp2/README.md +++ b/contrib/udp2/README.md @@ -21,7 +21,7 @@ ## Project Background -UDP2 is a next-generation interconnect protocol implementation based on the original UDP protocol, located in the `contrib/udp2` directory. In CloudBerry Database, the interconnect is responsible for data transmission and synchronization between nodes, serving as a core component for distributed query execution. +UDP2 is a next-generation interconnect protocol implementation based on the original UDP protocol, located in the `contrib/udp2` directory. In Apache Cloudberry, the interconnect is responsible for data transmission and synchronization between nodes, serving as a core component for distributed query execution. Currently, the database supports three interconnect protocol implementations: - **TCP** (`contrib/interconnect/tcp`) - Reliable transmission based on TCP protocol diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 5bb24e63734..0f3289b4ee0 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -198,7 +198,7 @@ log_section_end "Configure" # Capture version information log_section "Version Information" -execute_cmd ag "GP_VERSION | GP_VERSION_NUM | PG_VERSION | PG_VERSION_NUM | PG_VERSION_STR" src/include/pg_config.h +execute_cmd grep -E "GP_VERSION|GP_VERSION_NUM|PG_VERSION|PG_VERSION_NUM|PG_VERSION_STR" src/include/pg_config.h || true log_section_end "Version Information" # Log completion diff --git a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec index e228f8fe76a..22be01e0e54 100644 --- a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec +++ b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec @@ -15,9 +15,38 @@ # specific language governing permissions and limitations # under the License. -%define cloudberry_install_dir /usr/local/cloudberry-db +# Validate required macros early so rpmbuild fails with a clear message +# before reaching the header. +%{!?version:%{error:The macro 'version' must be supplied as --define 'version ...'}} +%{!?release:%{error:The macro 'release' must be supplied as --define 'release ...'}} + +%define cloudberry_base_dir /usr/local +%define cloudberry_name cloudberry-db +%define cloudberry_install_dir %{cloudberry_base_dir}/%{cloudberry_name} + +# Major version, used to build a versioned package Name so that different +# major versions can be installed side by side (like greenplum-db-6 vs +# greenplum-db-7). Derived from the version macro by default (single source +# of truth), but can be overridden via --define 'cloudberry_major_version N'. +%{!?cloudberry_major_version:%define cloudberry_major_version %(echo %{version} | cut -d. -f1)} + +# Suppress build-id links so they are not created outside the relocatable prefix. +%define _build_id_links none + +# Do not expose bundled/private shared libraries as RPM Provides. +# (e.g., libpq.so.5 would conflict with system postgresql-libs.) +%global __provides_exclude_from ^%{cloudberry_install_dir}-%{version}/.*\.so + +# Do not require these bundled libraries from the system; +# they are shipped inside the package and located via RPATH. +# +# This list must cover every soname the package installs, because +# __provides_exclude_from above drops the matching Provides: the .so +# symlinks we ship (libfoo.so -> libfoo.so.N.M) make rpm generate a +# Requires on libfoo.so.N, which would otherwise be resolved outside the +# package -- or not at all. build-rpm.sh verifies the two stay in sync. +%global __requires_exclude ^(libecpg\.so|libecpg_compat\.so|libgppc\.so|libpax\.so|libpaxformat\.so|libpgtypes\.so|libpostgres\.so|libpq\.so\.5|libxerces-c-3\.3\.so) -# Add at the top of the spec file # Default to non-debug build %bcond_with debug @@ -27,7 +56,11 @@ %define __strip /bin/true %endif -Name: apache-cloudberry-db-incubating +Name: apache-cloudberry-db-incubating-%{cloudberry_major_version} +# Replace the previous unversioned package on upgrade. This targets only the +# old fixed name; it does NOT match apache-cloudberry-db-incubating-, +# so different major versions still coexist. +Obsoletes: apache-cloudberry-db-incubating < %{version}-%{release} Version: %{version} # In the release definition section %if %{with debug} @@ -37,11 +70,11 @@ Release: %{release}%{?dist} %endif Summary: High-performance, open-source data warehouse based on PostgreSQL/Greenplum -License: ASL 2.0 +License: Apache-2.0 URL: https://cloudberry.apache.org Vendor: Apache Cloudberry (Incubating) Group: Applications/Databases -Prefix: %{cloudberry_install_dir} +Prefix: %{cloudberry_base_dir} # Disabled as we are shipping GO programs (e.g. gpbackup) %define _missing_build_ids_terminate_build 0 @@ -62,6 +95,10 @@ Requires: openssh-server Requires: rsync Requires: which +# Scriptlet dependencies (ln, readlink, rm are from coreutils). +Requires(post): coreutils +Requires(postun): coreutils + %if 0%{?rhel} == 8 Requires: apr Requires: audit @@ -105,6 +142,7 @@ Requires: pam Requires: pcre2 Requires: perl Requires: readline +Requires: python3 Requires: xz %endif @@ -145,7 +183,7 @@ project has yet to be fully endorsed by the ASF. # No prep needed for binary RPM %build -# No prep needed for binary RPM +# No build needed for binary RPM %install rm -rf %{buildroot} @@ -153,33 +191,80 @@ rm -rf %{buildroot} # Create the versioned directory mkdir -p %{buildroot}%{cloudberry_install_dir}-%{version} -cp -R %{cloudberry_install_dir}/* %{buildroot}%{cloudberry_install_dir}-%{version} +# Use cp -a with /. to include all the files +cp -a %{cloudberry_install_dir}/. %{buildroot}%{cloudberry_install_dir}-%{version}/ # Copy Apache mandatory compliance files from the SOURCES directory into the installation directory cp %{_sourcedir}/LICENSE %{buildroot}%{cloudberry_install_dir}-%{version}/ cp %{_sourcedir}/NOTICE %{buildroot}%{cloudberry_install_dir}-%{version}/ cp %{_sourcedir}/DISCLAIMER %{buildroot}%{cloudberry_install_dir}-%{version}/ -cp -R %{_sourcedir}/licenses %{buildroot}%{cloudberry_install_dir}-%{version}/ - -# Create the symbolic link -ln -sfn %{cloudberry_install_dir}-%{version} %{buildroot}%{cloudberry_install_dir} +cp -a %{_sourcedir}/licenses %{buildroot}%{cloudberry_install_dir}-%{version}/ %files -%{prefix}-%{version} -%{prefix} +%{cloudberry_install_dir}-%{version} +%config(noreplace) %{cloudberry_install_dir}-%{version}/cloudberry-env.sh %debug_package %post -# Change ownership to gpadmin.gpadmin if the gpadmin user exists -if id "gpadmin" &>/dev/null; then - chown -R gpadmin:gpadmin %{cloudberry_install_dir}-%{version} - chown gpadmin:gpadmin %{cloudberry_install_dir} +# RPM_INSTALL_PREFIX is set dynamically by RPM to the actual --prefix value. +# Fall back to cloudberry_base_dir when --prefix was not used. +INSTALL_PREFIX="${RPM_INSTALL_PREFIX:-%{cloudberry_base_dir}}" +INSTALL_BASE="${INSTALL_PREFIX%/}" + +LINK_PATH="${INSTALL_BASE}/%{cloudberry_name}" +VERSIONED_DIR="${INSTALL_BASE}/%{cloudberry_name}-%{version}" +LINK_TARGET_REL="%{cloudberry_name}-%{version}" + +if [ ! -e "${LINK_PATH}" ] && [ ! -L "${LINK_PATH}" ]; then + # Nothing at the symlink location yet — create it. + ln -s "${LINK_TARGET_REL}" "${LINK_PATH}" || : +elif [ -L "${LINK_PATH}" ]; then + # A symlink already exists. Update it when it points to a + # recognized Cloudberry versioned directory. + EXISTING_TARGET=$(readlink -f -- "${LINK_PATH}" 2>/dev/null || :) + EXISTING_NAME=${EXISTING_TARGET##*/} + + case "${EXISTING_NAME}" in + %{cloudberry_name}-*) + EXISTING_VERSION=${EXISTING_NAME#%{cloudberry_name}-} + EXISTING_MAJOR=${EXISTING_VERSION%%.*} + if [ "${EXISTING_MAJOR}" = "%{cloudberry_major_version}" ]; then + # Same major version: move the generic symlink to this build. + ln -sfnT "${LINK_TARGET_REL}" "${LINK_PATH}" || : + else + # Different major version: leave the existing symlink untouched + # so that multiple major versions can coexist under one prefix. + echo "Warning: ${LINK_PATH} points to Cloudberry major version ${EXISTING_MAJOR}; leaving it unchanged so major versions can coexist" >&2 + fi + ;; + *) + echo "Warning: ${LINK_PATH} does not point to a recognized Cloudberry installation; leaving it unchanged" >&2 + ;; + esac +else + echo "Warning: ${LINK_PATH} exists and is not a symbolic link; leaving it unchanged" >&2 fi +exit 0 + %postun -if [ $1 -eq 0 ] ; then - if [ "$(readlink -f "%{cloudberry_install_dir}")" == "%{cloudberry_install_dir}-%{version}" ]; then - unlink "%{cloudberry_install_dir}" || true - fi +INSTALL_PREFIX="${RPM_INSTALL_PREFIX:-%{cloudberry_base_dir}}" +INSTALL_BASE="${INSTALL_PREFIX%/}" + +LINK_PATH="${INSTALL_BASE}/%{cloudberry_name}" +VERSIONED_DIR="${INSTALL_BASE}/%{cloudberry_name}-%{version}" + +if [ -L "${LINK_PATH}" ]; then + LINK_TARGET=$(readlink "${LINK_PATH}" 2>/dev/null || :) + + # Remove the symlink only when it still points to the version + # being removed (handles both absolute and relative targets). + case "${LINK_TARGET}" in + "${VERSIONED_DIR}"|"%{cloudberry_name}-%{version}") + rm -f -- "${LINK_PATH}" || : + ;; + esac fi + +exit 0 diff --git a/devops/build/packaging/rpm/build-rpm.sh b/devops/build/packaging/rpm/build-rpm.sh index 2c490166f45..6b893206990 100755 --- a/devops/build/packaging/rpm/build-rpm.sh +++ b/devops/build/packaging/rpm/build-rpm.sh @@ -178,10 +178,91 @@ fi # Run rpmbuild with the provided options echo "Building RPM with Version: $VERSION, Release: $RELEASE$([ "$DEBUG_BUILD" = true ] && echo ", Debug: enabled")..." + +# Relax rpm's check-rpaths QA check. +# +# Cloudberry ships shared objects (e.g. PL/Python's _pg*.so) whose RUNPATH +# points at the product's install prefix (/usr/local/cloudberry-db/lib). +# This is intentional, but check-rpaths classifies such absolute, +# non-standard rpaths as "invalid" (0x0002) and, on the el10 toolchain, +# turns it into a fatal "%install" error. Setting QA_RPATHS downgrades the +# standard (0x0001), invalid (0x0002) and empty (0x0010) rpath findings to +# warnings so packaging succeeds. This is a no-op relaxation on el8/el9. +export QA_RPATHS=$(( 0x0001 | 0x0002 | 0x0010 )) + if ! eval "$RPMBUILD_CMD"; then echo "Error: rpmbuild failed." exit 1 fi +# Normalize published artifact file names. +# +# The RPM's Name metadata embeds the major version +# (apache-cloudberry-db-incubating-) so that different major versions +# can be installed side by side. rpmbuild therefore emits files named +# apache-cloudberry-db-incubating---....rpm. +# +# For published artifacts we keep the historical file name that omits the +# "-" segment (e.g. apache-cloudberry-db-incubating-2.1.0-1.el8.x86_64.rpm), +# matching the distribution naming used by Greenplum. Renaming the file does +# NOT change the package identity: RPM reads Name/Version/Release from the +# package header, not from the file name, so install/upgrade/coexistence and +# `dnf`/`rpm` queries are unaffected. +MAJOR_VERSION="${VERSION%%.*}" +RPMS_DIR="$(rpm --eval '%{_rpmdir}')" + +shopt -s nullglob +for rpm_path in "${RPMS_DIR}"/*/apache-cloudberry-db-incubating-"${MAJOR_VERSION}"-*"${VERSION}"-*.rpm; do + rpm_dir="$(dirname "$rpm_path")" + rpm_base="$(basename "$rpm_path")" + # Drop the "-" that immediately follows the fixed name prefix. + new_base="${rpm_base/apache-cloudberry-db-incubating-${MAJOR_VERSION}-/apache-cloudberry-db-incubating-}" + if [ "$new_base" != "$rpm_base" ]; then + mv -f "$rpm_path" "${rpm_dir}/${new_base}" + echo "Renamed published artifact: ${rpm_base} -> ${new_base}" + fi +done +shopt -u nullglob + +# Verify that the package does not depend on sonames it ships itself. +# +# The spec drops the auto-generated Provides for every shared object under +# the install prefix (%__provides_exclude_from, a path pattern) but removes +# only a hand-maintained list of sonames from the auto-generated Requires +# (%__requires_exclude, a name pattern). The .so symlinks in the package +# (libfoo.so -> libfoo.so.N.M) make rpm emit a Requires on libfoo.so.N, so +# whenever a bundled library is missing from that list the dependency turns +# into an external one. Such a package either fails to install ("nothing +# provides libfoo.so.N") or silently resolves against a system library. +# +# rpmbuild cannot detect this, and the failure only surfaces in downstream +# install jobs, so check it here while the failing artifact is at hand. +shopt -s nullglob +for rpm_path in "${RPMS_DIR}"/*/apache-cloudberry-db-incubating-*"${VERSION}"-*.rpm; do + case "$rpm_path" in + *debuginfo*|*debugsource*) continue ;; + esac + + required_sonames="$(mktemp)" + shipped_sonames="$(mktemp)" + + # "libfoo.so.1(GLIBC_2.34)(64bit)" -> "libfoo.so.1" + rpm -qp --requires "$rpm_path" | awk '{print $1}' | sed -E 's/\(.*//' \ + | grep -E '\.so' | sort -u > "$required_sonames" || true + rpm -qpl "$rpm_path" | sed 's#.*/##' \ + | grep -E '\.so(\.[0-9]+)*$' | sort -u > "$shipped_sonames" || true + + self_requires="$(comm -12 "$required_sonames" "$shipped_sonames")" + rm -f "$required_sonames" "$shipped_sonames" + + if [ -n "$self_requires" ]; then + echo "Error: $(basename "$rpm_path") requires sonames that it ships itself:" + echo "$self_requires" | sed 's/^/ /' + echo "Add them to %__requires_exclude in apache-cloudberry-db-incubating.spec." + exit 1 + fi +done +shopt -u nullglob + # Print completion message echo "RPM build completed successfully with Version: $VERSION, Release: $RELEASE" diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 3ae00c63b76..61f7076db34 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -20,6 +20,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_inject_fault \ gp_exttable_fdw \ gp_legacy_string_agg \ + gp_relsizes_stats \ gp_replica_check \ gp_toolkit \ pg_hint_plan \ @@ -29,6 +30,7 @@ else gp_distribution_policy \ gp_internal_tools \ gp_legacy_string_agg \ + gp_relsizes_stats \ gp_exttable_fdw \ gp_toolkit \ pg_hint_plan diff --git a/gpcontrib/diskquota/tests/regress/expected/test_max_monitored_databases.out b/gpcontrib/diskquota/tests/regress/expected/test_max_monitored_databases.out index 84568bc114a..ea15eb460ff 100644 --- a/gpcontrib/diskquota/tests/regress/expected/test_max_monitored_databases.out +++ b/gpcontrib/diskquota/tests/regress/expected/test_max_monitored_databases.out @@ -1,7 +1,6 @@ --start_ignore -\! gpconfig -c diskquota.max_monitored_databases -v 3 -20230905:12:39:55:332748 gpconfig:zhrt:zhrt-[INFO]:-completed successfully with parameters '-c diskquota.max_monitored_databases -v 3' -\! gpstop -ari +\! gpconfig -c diskquota.max_monitored_databases -v 3 > /dev/null +\! gpstop -arf > /dev/null --end_ignore \c DROP DATABASE IF EXISTS test_db1; @@ -68,37 +67,6 @@ DROP DATABASE test_db1; DROP DATABASE test_db2; DROP DATABASE test_db3; -- start_ignore -\! gpconfig -r diskquota.max_monitored_databases -20230905:12:40:29:350921 gpconfig:zhrt:zhrt-[INFO]:-completed successfully with parameters '-r diskquota.max_monitored_databases' -\! gpstop -ari -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Starting gpstop with args: -ari -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Gathering information and validating the environment... -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Obtaining Greenplum Master catalog information -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Obtaining Segment details from master... -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 6.24.4+dev.45.gad3671f087 build dev' -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Commencing Master instance shutdown with mode='immediate' -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Master segment instance directory=/home/zhrt/workspace/gpdb6/gpAux/gpdemo/datadirs/qddir/demoDataDir-1 -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Attempting forceful termination of any leftover master process -20230905:12:40:30:352551 gpstop:zhrt:zhrt-[INFO]:-Terminating processes for segment /home/zhrt/workspace/gpdb6/gpAux/gpdemo/datadirs/qddir/demoDataDir-1 -20230905:12:40:37:352551 gpstop:zhrt:zhrt-[INFO]:-Stopping master standby host zhrt mode=immediate -20230905:12:40:38:352551 gpstop:zhrt:zhrt-[INFO]:-Successfully shutdown standby process on zhrt -20230905:12:40:38:352551 gpstop:zhrt:zhrt-[INFO]:-Targeting dbid [2, 5, 3, 6, 4, 7] for shutdown -20230905:12:40:38:352551 gpstop:zhrt:zhrt-[INFO]:-Commencing parallel primary segment instance shutdown, please wait... -20230905:12:40:38:352551 gpstop:zhrt:zhrt-[INFO]:-0.00% of jobs completed -20230905:12:40:43:352551 gpstop:zhrt:zhrt-[INFO]:-100.00% of jobs completed -20230905:12:40:43:352551 gpstop:zhrt:zhrt-[INFO]:-Commencing parallel mirror segment instance shutdown, please wait... -20230905:12:40:43:352551 gpstop:zhrt:zhrt-[INFO]:-0.00% of jobs completed -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-100.00% of jobs completed -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:----------------------------------------------------- -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:- Segments stopped successfully = 6 -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:- Segments with errors during stop = 0 -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:----------------------------------------------------- -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-Successfully shutdown 6 of 6 segment instances -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-Database successfully shutdown with no errors reported -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-Cleaning up leftover gpmmon process -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-No leftover gpmmon process found -20230905:12:40:46:352551 gpstop:zhrt:zhrt-[INFO]:-Cleaning up leftover gpsmon processes -20230905:12:40:47:352551 gpstop:zhrt:zhrt-[INFO]:-No leftover gpsmon processes on some hosts. not attempting forceful termination on these hosts -20230905:12:40:47:352551 gpstop:zhrt:zhrt-[INFO]:-Cleaning up leftover shared memory -20230905:12:40:48:352551 gpstop:zhrt:zhrt-[INFO]:-Restarting System... +\! gpconfig -r diskquota.max_monitored_databases > /dev/null +\! gpstop -arf > /dev/null -- end_ignore diff --git a/gpcontrib/diskquota/tests/regress/sql/test_max_monitored_databases.sql b/gpcontrib/diskquota/tests/regress/sql/test_max_monitored_databases.sql index f0e2e8c1aa9..516461c2746 100644 --- a/gpcontrib/diskquota/tests/regress/sql/test_max_monitored_databases.sql +++ b/gpcontrib/diskquota/tests/regress/sql/test_max_monitored_databases.sql @@ -1,6 +1,6 @@ --start_ignore -\! gpconfig -c diskquota.max_monitored_databases -v 3 -\! gpstop -ari +\! gpconfig -c diskquota.max_monitored_databases -v 3 > /dev/null +\! gpstop -arf > /dev/null --end_ignore \c @@ -43,6 +43,6 @@ DROP DATABASE test_db2; DROP DATABASE test_db3; -- start_ignore -\! gpconfig -r diskquota.max_monitored_databases -\! gpstop -ari +\! gpconfig -r diskquota.max_monitored_databases > /dev/null +\! gpstop -arf > /dev/null -- end_ignore \ No newline at end of file diff --git a/gpcontrib/gp_relsizes_stats/.clang-format b/gpcontrib/gp_relsizes_stats/.clang-format new file mode 100644 index 00000000000..efcf9ff4160 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/.clang-format @@ -0,0 +1,192 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveMacros: None +AlignConsecutiveAssignments: None +AlignConsecutiveBitFields: None +AlignConsecutiveDeclarations: None +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: true +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +QualifierAlignment: Leave +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: Signature +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PointerAlignment: Right +PPIndentWidth: -1 +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + BeforeNonEmptyParentheses: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/gpcontrib/gp_relsizes_stats/.gitignore b/gpcontrib/gp_relsizes_stats/.gitignore new file mode 100644 index 00000000000..ebe888c0253 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/.gitignore @@ -0,0 +1,6 @@ +*.o +*.so +src/protos/ +results +.vscode +compile_commands.json diff --git a/gpcontrib/gp_relsizes_stats/Makefile b/gpcontrib/gp_relsizes_stats/Makefile new file mode 100644 index 00000000000..fdb8d22cae5 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/Makefile @@ -0,0 +1,62 @@ +#------------------------------------------------------------------------- +# +# Makefile for gp_relsizes_stats extension. +# +# By default this Makefile is intended to be included from the surrounding +# Cloudberry source tree (in-tree build); pass USE_PGXS=1 to build against an +# already-installed Cloudberry/PostgreSQL using pg_config. +# +# IDENTIFICATION +# gpcontrib/gp_relsizes_stats/Makefile +# +#------------------------------------------------------------------------- + +MODULE_big = gp_relsizes_stats +OBJS = ./src/gp_relsizes_stats.o +EXTENSION = gp_relsizes_stats +EXTVERSION = 1.0 +DATA = $(wildcard sql/*--*.sql) +REGRESS = grants gp_relsizes_stats +REGRESS_OPTS = --inputdir=test/ --temp-config=$(srcdir)/test/postgresql.conf.add +PGFILEDESC = "gp_relsizes_stats - an extension to track table on-disc sizes in Cloudberry" +PG_CXXFLAGS += $(COMMON_CPP_FLAGS) + +# Auto-detect build mode: if we are sitting inside a Cloudberry source tree +# (i.e. ../../src/Makefile.global is reachable), build in-tree; otherwise fall +# back to a standalone PGXS build driven by pg_config. USE_PGXS=1 forces PGXS +# even when an in-tree Makefile.global is available. +ifeq ($(USE_PGXS),) + ifeq ($(wildcard ../../src/Makefile.global),) + USE_PGXS = 1 + endif +endif + +ifdef USE_PGXS + PG_CONFIG ?= pg_config + PGXS := $(shell $(PG_CONFIG) --pgxs) + include $(PGXS) +else + subdir = gpcontrib/gp_relsizes_stats + top_builddir = ../.. + include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk +endif + +# The extension registers its background worker only when loaded +# via shared_preload_libraries (see _PG_init in src/gp_relsizes_stats.c). +# Without that, relsizes_collect_stats_once() silently does nothing and the +# regression tests get (0 rows) everywhere. Ensure the preload is set up +# before installcheck runs. +installcheck: preload-bgworker + +.PHONY: preload-bgworker +preload-bgworker: + @if [ -z "$$COORDINATOR_DATA_DIRECTORY$$MASTER_DATA_DIRECTORY" ]; then \ + echo "ERROR: COORDINATOR_DATA_DIRECTORY (or MASTER_DATA_DIRECTORY) is not set;" >&2; \ + echo " source cloudberry-env.sh and gpdemo-env.sh before running installcheck." >&2; \ + exit 1; \ + fi + @echo "==> Ensuring shared_preload_libraries contains gp_relsizes_stats and restarting cluster" + gpconfig -c shared_preload_libraries -v "'gp_relsizes_stats'" --skipvalidation + gpstop -ra + psql -d postgres -c "SHOW shared_preload_libraries;" diff --git a/gpcontrib/gp_relsizes_stats/README.md b/gpcontrib/gp_relsizes_stats/README.md new file mode 100644 index 00000000000..04209704b0f --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/README.md @@ -0,0 +1,71 @@ + + +# gp_relsizes_stats: Table sizes monitoring tool for Cloudberry + +### Features +gp_relsizes_stats is an extension for the Cloudberry database that calculates and stores statistics on the size of files and tables, occupied space on the disks of the master and segment hosts. + +#### Features include +- BackgroundWorker support for collecting statistics automatically +- the ability to fine-tune the timeout values between actions, for example, between launches for different databases, or during file processing to distribute the load over time + +### Supported versions and platforms +At the moment, the program is being tested only for Cloudberry and Linux. + +### Installation +This extension is part of the Cloudberry monorepo under `gpcontrib/gp_relsizes_stats`. + +Build and install from the Cloudberry monorepo root: +```bash +make -C gpcontrib/gp_relsizes_stats +sudo make -C gpcontrib/gp_relsizes_stats install +``` + +### Configuration +gp_relsizes_stats configuration parameters: +| **Parameter** | **Type** | **Default** | **Description** | +| ---------------- | --------------- | ------------ | ------------ | +| `gp_relsizes_stats.enabled` | bool | false | You can enable/disable background stats collection for database where extension installed (actually enable/disable background worker which collecting stats).| +| `gp_relsizes_stats.save_history` | bool | true | You can disable the collection of statistics records in the history table (table_sizes_history).| +| `gp_relsizes_stats.restart_naptime` | int | 21600000 | You can set naptime between each startup of collecting process. Value set time in milliseconds. Default is equal to 6 hours.| +| `gp_relsizes_stats.database_naptime` | int | 0 | You can set naptime between collecting stats for each databases. Value set time in milliseconds. Default is equal to 0 milliseconds.| +| `gp_relsizes_stats.file_naptime` | int | 1 | You can set naptime between each file stats calculating. Value set time in milliseconds. Default is equal to 1 millisecond.| + +### Usage +You can use a background worker to collect statistics, but if you sometimes need to change the format of the settings or if you don't want to collect statistics on a regular basis, you can do so. In these situations, you could set +``` +gp_relsizes_stats.enabled = off +``` + +And use the function +``` +relsizes_stats_schema.relsizes_collect_stats_once() +``` +which can be called manually using 'select'. +It will launch a single statistics collection procedure. + + +### About collected data and tables +| Name of table | Row description | Description | +| ------------- | --------------- | ----------- | +| relsizes_stats_schema.segment_file_sizes | (segment, relfilenode, filepath, size, mtime) | Current size and last modify time of each file of specific relation on specific segment | +| relsizes_stats_schema.namespace_sizes | (nspname, nspsize) | Current size of namespace | +| relsizes_stats_schema.table_sizes | (nspname, relname, relsize) | Current size of relation in specific namespace | +| relsizes_stats_schema.table_sizes_history | (insert_date, nspname, relname, size, mtime) | Size and last modify time of relation in specific namespace with date when information was collected | diff --git a/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control b/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control new file mode 100644 index 00000000000..e28a3526b30 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control @@ -0,0 +1,5 @@ +# gp_relsizes_stats extension +comment = 'gp_relsizes_stats - an extension to track table on-disc sizes in cloudberry' +default_version = '1.0' +module_pathname = '$libdir/gp_relsizes_stats' +trusted = true diff --git a/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0.sql b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0.sql new file mode 100644 index 00000000000..4886bf77f89 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0.sql @@ -0,0 +1,158 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_relsizes_stats" to load this file. \quit + + +-- CREATE TABLE IF NOT EXISTS ... (....) DISTRIBUTED BY ... +CREATE SCHEMA IF NOT EXISTS relsizes_stats_schema; + +-- create table +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.segment_file_map + (segment INTEGER, reloid OID, relfilenode OID) + WITH (appendonly=true) DISTRIBUTED RANDOMLY; +-- create table +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.segment_file_sizes + (segment INTEGER, relfilenode OID, filepath TEXT, size BIGINT, mtime BIGINT) + WITH (appendonly=true) DISTRIBUTED RANDOMLY; +-- create table for backup info +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.table_sizes_history + (insert_date date NOT NULL, nspname text NOT NULL, relname text NOT NULL, size bigint NOT NULL, mtime timestamp NOT NULL) + DISTRIBUTED RANDOMLY; + + +CREATE OR REPLACE VIEW relsizes_stats_schema.table_files AS + /* + * Recursive enumeration of all relations together with their partition + * trees, preserving the original GP6 view semantics: + * + * - Every relation appears as a "self" row with own_oid = true + * (relname carries its own name). + * - Additionally, for every relation that is a "root" of a partition + * tree (has no parent in pg_inherits), we walk the entire tree of + * descendants and add a row per descendant with own_oid = false; + * the descendant inherits the relname of the root (so SUMs in + * table_sizes / namespace_sizes attribute leaf data to the root + * partitioned table). + * + * - Intermediate partitioned tables (which are themselves children + * of another partitioned table) only contribute a self-row; they + * do NOT roll up their own leaves — those are already counted + * under the root. + * + * On Cloudberry / PG14 partition root and intermediate partitioned + * tables have relfilenode = 0 — no physical storage. To keep them + * visible in the view (with size = 0) we LEFT JOIN segment_file_sizes + * instead of using INNER JOIN. + */ + WITH RECURSIVE all_rels AS ( + SELECT n.nspname, c.relname, c.oid + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relkind IN ('r', 'p') + AND c.reltablespace != (SELECT oid FROM pg_tablespace WHERE spcname = 'pg_global') + ), + /* "Roots" — relations that do not inherit from any table/partitioned table. */ + roots AS ( + SELECT ar.nspname, ar.relname, ar.oid + FROM all_rels ar + WHERE NOT EXISTS ( + SELECT 1 + FROM pg_inherits pi + JOIN pg_class pc ON pi.inhparent = pc.oid + WHERE pi.inhrelid = ar.oid + AND pc.relkind IN ('r', 'p') + ) + ), + /* Walk the entire partition tree starting from each root. */ + descendants AS ( + SELECT r.nspname, r.relname AS root_relname, r.oid AS cur_oid, 0 AS depth + FROM roots r + UNION ALL + SELECT d.nspname, d.root_relname, c2.oid, d.depth + 1 + FROM descendants d + JOIN pg_inherits pi ON d.cur_oid = pi.inhparent + JOIN pg_class c2 ON pi.inhrelid = c2.oid + WHERE c2.relkind IN ('r', 'p') + ), + part_oids AS ( + /* Self-row for every relation. */ + SELECT nspname, relname, oid, true AS own_oid + FROM all_rels + UNION ALL + /* Descendant rows for every root (excluding the root itself). */ + SELECT nspname, root_relname AS relname, cur_oid AS oid, false AS own_oid + FROM descendants + WHERE depth > 0 + ), + table_oids AS ( + SELECT po.nspname, po.relname, po.oid, po.own_oid, 'main' AS kind + FROM part_oids po + UNION ALL + SELECT po.nspname, po.relname, t.reltoastrelid, po.own_oid, 'toast' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ti.indexrelid, po.own_oid, 'toast_idx' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + JOIN pg_index ti ON t.reltoastrelid = ti.indrelid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ao.segrelid, po.own_oid, 'ao' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimaprelid, po.own_oid, 'ao_vm' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimapidxid, po.own_oid, 'ao_vm_idx' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + ) + SELECT table_oids.nspname, + table_oids.relname, + m.segment, + m.relfilenode, + fs.filepath, + kind, + COALESCE(fs.size, 0) AS size, + COALESCE(fs.mtime, 0) AS mtime, + table_oids.own_oid AS own_file + FROM table_oids + JOIN relsizes_stats_schema.segment_file_map m + ON table_oids.oid = m.reloid + /* + * LEFT JOIN, not INNER JOIN: partitioned (root + intermediate) tables + * on Cloudberry have no physical file, so segment_file_sizes does not contain + * a matching row. We still want to surface them with size = 0. + */ + LEFT JOIN relsizes_stats_schema.segment_file_sizes fs + ON m.segment = fs.segment AND m.relfilenode = fs.relfilenode; +CREATE OR REPLACE VIEW relsizes_stats_schema.table_sizes AS + SELECT nspname, relname, sum(size) AS size, to_timestamp(MAX(mtime)) AS mtime FROM relsizes_stats_schema.table_files + GROUP BY nspname, relname; +CREATE OR REPLACE VIEW relsizes_stats_schema.namespace_sizes AS + SELECT nspname, sum(size) AS size FROM relsizes_stats_schema.table_files + WHERE own_file + GROUP BY nspname; +-- Here go any C or PL/SQL functions, table or view definitions etc +-- for example: + +CREATE FUNCTION relsizes_stats_schema.get_stats_for_database(dboid OID, fast BOOL) +RETURNS TABLE (segment INTEGER, relfilenode OID, filepath TEXT, size BIGINT, mtime BIGINT) +AS 'MODULE_PATHNAME', 'get_stats_for_database' +LANGUAGE C STRICT EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION relsizes_stats_schema.relsizes_collect_stats_once() +RETURNS void +AS 'MODULE_PATHNAME', 'relsizes_collect_stats_once' +LANGUAGE C STRICT; + + +DO $$ +BEGIN + EXECUTE 'GRANT USAGE ON SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; + EXECUTE 'GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; +END +$$; diff --git a/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c b/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c new file mode 100755 index 00000000000..bc176d0b0ad --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c @@ -0,0 +1,1046 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gp_relsizes_stats.c + * + * IDENTIFICATION + * gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +/* Required headers for background workers */ +#include "miscadmin.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lwlock.h" +#include "storage/proc.h" +#include "storage/shmem.h" + +/* Additional headers for extension functionality */ +#include "access/xact.h" +#include "executor/spi.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "pgstat.h" +#include "tcop/utility.h" + +#include "catalog/namespace.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "funcapi.h" + +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/guc.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#include +#include +#include +#include +#include + +#define FILEINFO_ARGS_CNT 5 +#define HOUR_TIME 3600000 /* milliseconds in hour */ +#define MINUTE_TIME 60000 /* milliseconds in minute */ +#define FILE_NAPTIME 1 /* default naptime between file processing in milliseconds */ + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(get_stats_for_database); +PG_FUNCTION_INFO_V1(relsizes_collect_stats_once); +Datum get_stats_for_database(PG_FUNCTION_ARGS); +Datum relsizes_collect_stats_once(PG_FUNCTION_ARGS); + +static void worker_sigterm(SIGNAL_ARGS); +static void worker_sighup(SIGNAL_ARGS); +static Oid *get_databases_oids(int *databases_cnt, MemoryContext ctx, bool create_transaction); +static int update_segment_file_map_table(void); +static int update_table_sizes_history(void); +static void get_stats_for_databases(Oid *databases_oids, int databases_cnt, bool fast); +static void run_database_stats_worker(bool fast, Oid db); +static int plugin_created(void); +static BgwHandleStatus WaitForBackgroundWorkerShutdownSafely(BackgroundWorkerHandle *handle); +static int delete_data_in_history(void); +static int put_data_into_history(void); +void _PG_init(void); + +PGDLLEXPORT void relsizes_collect_stats(Datum main_arg); +PGDLLEXPORT void relsizes_database_stats_job(Datum args); + +/* Global variables */ +static int worker_restart_naptime = 0; +static int worker_database_naptime = 0; +static int worker_file_naptime = 0; +static bool enabled = false; +static bool save_history = true; + +static volatile sig_atomic_t got_sigterm = false; +static volatile sig_atomic_t got_sighup = false; + +typedef union DbWorkerArg { + Datum d; + struct { + Oid db; + bool fast; + } s; +} DbWorkerArg; + +StaticAssertDecl(sizeof(Datum) == sizeof(DbWorkerArg), + "Invalid size of structure in DbWorkerArg"); + +/* + * Signal handler for SIGTERM + * Set a flag to let the main loop to terminate, and set our latch to wake + * it up. + */ +static void worker_sigterm(SIGNAL_ARGS) { + int save_errno = errno; + got_sigterm = true; + if (MyProc) { + SetLatch(&MyProc->procLatch); + } + errno = save_errno; +} + +/* + * Signal handler for SIGHUP + * Set a flag to tell the main loop to reread the config file, and set + * our latch to wake it up. + */ +static void worker_sighup(SIGNAL_ARGS) { + int save_errno = errno; + got_sighup = true; + if (MyProc) { + SetLatch(&MyProc->procLatch); + } + errno = save_errno; +} + +/* + * Wait for a background worker to stop with timeout and error handling. + * + * This is a modified version that adds timeout functionality and improved + * error handling to prevent infinite loops in case of hung workers. + * Returns BGWH_STOPPED on success, BGWH_POSTMASTER_DIED on error/timeout. + */ +static BgwHandleStatus WaitForBackgroundWorkerShutdownSafely(BackgroundWorkerHandle *handle) { + BgwHandleStatus status = BGWH_NOT_YET_STARTED; + int rc; + int attempts = 0; + const int max_attempts = 5 * HOUR_TIME / 100; /* maximum 5 hours wait time */ + + PG_TRY(); + { + while (attempts < max_attempts) { + pid_t pid; + + status = GetBackgroundWorkerPid(handle, &pid); + if (status == BGWH_STOPPED) { + return status; + } + + /* Add 100ms timeout instead of infinite wait */ + rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 100L, WAIT_EVENT_BGWORKER_SHUTDOWN); + + ResetLatch(&MyProc->procLatch); + + if (rc & WL_POSTMASTER_DEATH) { + status = BGWH_POSTMASTER_DIED; + break; + } + + /* Check for interrupts but don't let them break the entire process */ + if (QueryCancelPending || ProcDiePending) { + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: received interrupt signal, stopping wait"))); + status = BGWH_POSTMASTER_DIED; /* Return status as if postmaster died */ + break; + } + + attempts++; + } + + /* If maximum attempts reached */ + if (attempts >= max_attempts) { + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: timeout after %d attempts", max_attempts))); + status = BGWH_POSTMASTER_DIED; /* Return error status */ + } + } + PG_CATCH(); + { + /* Log error but do NOT re-throw exception */ + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: caught exception, returning error status"))); + /* Return error status instead of PG_RE_THROW() */ + return BGWH_POSTMASTER_DIED; + } + PG_END_TRY(); + return status; +} + +/* + * Retrieve list of database OIDs from the catalog. + * + * This function queries pg_database to get all user databases (excluding + * system databases like template0, template1, diskquota, and gpperfmon). + * + * Parameters: + * databases_cnt - Output parameter, set to number of databases found + * ctx - Memory context to allocate result in (for cross-call persistence) + * create_transaction - Whether to create a new transaction for the query + * + * Returns: + * Array of OIDs allocated in ctx, or NULL on error. + * The array length is databases_cnt. + * + * Note: Caller is responsible for freeing the returned memory when done. + */ +static Oid *get_databases_oids(int *databases_cnt, MemoryContext ctx, bool create_transaction) { + const char *sql = + "SELECT oid" + " FROM pg_database" + " WHERE datname NOT IN ('template0', 'template1', 'diskquota', 'gpperfmon')"; + const char *error = NULL; + + Oid *databases_oids = NULL; + *databases_cnt = 0; + + if (create_transaction) { + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + PushActiveSnapshot(GetTransactionSnapshot()); + pgstat_report_activity(STATE_RUNNING, sql); + } + + if (SPI_connect() < 0) { + error = "get_databases_oids: SPI_connect failed"; + goto finish_transaction; + } + + if (SPI_execute(sql, true, 0) != SPI_OK_SELECT) { + error = "get_databases_oids: SPI_execute failed (select datname, oid)"; + goto finish_spi; + } + + /* Prepare tuple processing variables */ + + *databases_cnt = SPI_processed; + MemoryContext old_context = MemoryContextSwitchTo(ctx); + databases_oids = palloc((*databases_cnt) * sizeof(*databases_oids)); + MemoryContextSwitchTo(old_context); + + for (int i = 0; i < SPI_processed; ++i) { + Datum oid_datum; + bool oid_isnull; + + oid_datum = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &oid_isnull); + + databases_oids[i] = DatumGetObjectId(oid_datum); + } + +finish_spi: + SPI_finish(); +finish_transaction: + if (create_transaction) { + PopActiveSnapshot(); + CommitTransactionCommand(); + pgstat_report_stat(false); + pgstat_report_activity(STATE_IDLE, NULL); + } + + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + return NULL; /* Return NULL on error */ + } + + return databases_oids; +} + +/* + * Update the segment_file_map table with current relation file mappings. + * + * This function refreshes the mapping between relation OIDs and their + * physical file nodes across all segments. It first deletes the existing + * data and then repopulates it by querying pg_class on all segments. + * + * The mapping is essential for correlating file statistics collected + * from the filesystem with actual database relations. + * + * Returns: + * 0 on success, negative value on error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int update_segment_file_map_table() { + int retcode = 0; + char *sql_delete = "DELETE FROM relsizes_stats_schema.segment_file_map"; + char *sql_insert = "INSERT INTO relsizes_stats_schema.segment_file_map SELECT gp_segment_id, oid, relfilenode FROM " + "gp_dist_random('pg_class')"; + char *error = NULL; + pgstat_report_activity(STATE_RUNNING, sql_delete); + retcode = SPI_execute(sql_delete, false, 0); + if (retcode != SPI_OK_DELETE) { + error = "update_segment_file_map_table: failed to delete from table"; + goto cleanup; + } + + pgstat_report_activity(STATE_RUNNING, sql_insert); + retcode = SPI_execute(sql_insert, false, 0); + if (retcode != SPI_OK_INSERT) { + error = "update_segment_file_map_table: failed to insert new rows into table"; + goto cleanup; + } + +cleanup: + pgstat_report_activity(STATE_IDLE, NULL); + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + } + return retcode; +} + +/* + * Check if a character is a digit (0-9). + * + * Simple utility function used by fill_relfilenode() to parse + * numeric portions of filenames. + * + * Returns: + * true if character is a digit, false otherwise + */ +static bool is_number(char symbol) { return '0' <= symbol && symbol <= '9'; } + +/* + * Extract relfilenode from filename by finding the first sequence of digits + * in the filename and converting it to numeric value + */ +static unsigned int fill_relfilenode(char *name) { + unsigned int result = 0, pos = 0; + size_t name_len = strlen(name); + + while (pos < name_len && !is_number(name[pos])) { + ++pos; + } + while (pos < name_len && is_number(name[pos])) { + /* Check for overflow to prevent integer overflow */ + if (result > (UINT_MAX - (name[pos] - '0')) / 10) { + break; /* Stop on potential overflow */ + } + result = (result * 10 + (name[pos] - '0')); + ++pos; + } + return result; +} + +/* + * Background worker entry point for database-specific statistics collection. + * + * This function is executed by dynamically spawned background workers to + * collect file size statistics for a specific database. Each worker: + * 1. Connects to the target database + * 2. Verifies the extension is installed + * 3. Updates the segment file mapping + * 4. Collects file size statistics from all segments + * 5. Updates the historical statistics table + * + * The function runs within its own transaction and handles errors gracefully + * by logging warnings rather than aborting the entire collection process. + * + * Parameters: + * args - Background worker argument which contains database OID and the flag + * which indicates make pauses or not + * + * Note: This function is called via the background worker framework and + * should not be called directly. + */ +PGDLLEXPORT void relsizes_database_stats_job(Datum args) { + int retcode = 0; + char *error = NULL; + DbWorkerArg wa = { .d = args }; + + optimizer = false; + pqsignal(SIGTERM, worker_sigterm); + pqsignal(SIGHUP, worker_sighup); + BackgroundWorkerUnblockSignals(); + + BackgroundWorkerInitializeConnectionByOid(wa.s.db, InvalidOid, 0); + + if (IS_QUERY_DISPATCHER() && !IS_SINGLENODE()) + Gp_role = GP_ROLE_DISPATCH; + + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + + retcode = SPI_connect(); + if (retcode < 0) { + error = "relsizes_database_stats_job: SPI_connect failed"; + goto finish_transaction; + } + PushActiveSnapshot(GetTransactionSnapshot()); + + /* Verify extension is installed */ + int created = plugin_created(); + if (created < 0) { + error = "relsizes_database_stats_job: SPI execute failed while looking for plugin"; + goto finish_spi; + } else if (created == 0) { + goto finish_spi; + } + + retcode = update_segment_file_map_table(); + if (retcode < 0) { + error = "relsizes_database_stats_job: updating segment_file_map failed"; + goto finish_spi; + } + + char *sql_delete = "DELETE FROM relsizes_stats_schema.segment_file_sizes"; + pgstat_report_activity(STATE_RUNNING, sql_delete); + retcode = SPI_execute(sql_delete, false, 0); + if (retcode != SPI_OK_DELETE) { + error = "relsizes_database_stats_job: SPI_execute failed (delete from segment_file_sizes)"; + goto finish_spi; + } + + /* Remove this condition after decision how to upgrade extensions is made. */ + if (SearchSysCacheExists3(PROCNAMEARGSNSP, + CStringGetDatum("get_stats_for_database"), + PointerGetDatum(buildoidvector((Oid[]){INT4OID}, 1)), + ObjectIdGetDatum(get_namespace_oid("relsizes_stats_schema", true)))) + { + const char* sql_get_stats = + "INSERT INTO relsizes_stats_schema.segment_file_sizes (segment, relfilenode, filepath, size, mtime) " + "SELECT * FROM relsizes_stats_schema.get_stats_for_database($1)"; + pgstat_report_activity(STATE_RUNNING, sql_get_stats); + retcode = SPI_execute_with_args(sql_get_stats, 1, + (Oid[]){INT4OID}, + (Datum[]){Int32GetDatum((int32) MyDatabaseId)}, + NULL, false, 0); + } else { + const char* sql_get_stats = + "INSERT INTO relsizes_stats_schema.segment_file_sizes (segment, relfilenode, filepath, size, mtime) " + "SELECT * FROM relsizes_stats_schema.get_stats_for_database($1, $2)"; + pgstat_report_activity(STATE_RUNNING, sql_get_stats); + retcode = SPI_execute_with_args(sql_get_stats, 2, + (Oid[]){OIDOID, BOOLOID}, + (Datum[]){Int32GetDatum((int32) MyDatabaseId), BoolGetDatum(wa.s.fast)}, + NULL, false, 0); + } + if (retcode != SPI_OK_INSERT) { + error = "relsizes_database_stats_job: SPI_execute failed (insert into segment_file_sizes)"; + goto finish_spi; + } + + if (save_history) { + retcode = update_table_sizes_history(); + if (retcode < 0) { + error = "relsizes_database_stats_job: updating tables sizes history table failed"; + goto finish_spi; + } + } + +finish_spi: + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + /* Don't abort execution, continue with cleanup */ + } + SPI_finish(); +finish_transaction: + if (ActiveSnapshotSet()) + PopActiveSnapshot(); + CommitTransactionCommand(); + pgstat_report_stat(false); + pgstat_report_activity(STATE_IDLE, NULL); +} + +/* + * Spawn and manage a background worker for database statistics collection. + * + * This function creates a new background worker to collect statistics for + * a specific database. + * + * The function: + * 1. Configures a new background worker with appropriate settings + * 2. Registers and starts the worker + * 3. Waits for the worker to complete + * 4. Handles any errors during worker execution + * + * If the worker fails to start or encounters errors during execution, + * warnings are logged but the function returns normally to allow + * processing of remaining databases. + * + * Parameters: + * fast - Don't make pauses + * db - OID of the database which worker will collect statistics from + * + * Note: This function may take significant time to complete as it waits + * for the background worker to finish processing the entire database. + */ +static void run_database_stats_worker(bool fast, Oid db) { + bool ret; + MemoryContext old_ctx; + BackgroundWorkerHandle *handle; + BgwHandleStatus status; + + /* Configure background worker */ + BackgroundWorker database_worker = { + .bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION, + .bgw_start_time = BgWorkerStart_RecoveryFinished, + .bgw_restart_time = BGW_NEVER_RESTART, + .bgw_library_name = "gp_relsizes_stats", + .bgw_function_name = "relsizes_database_stats_job", + .bgw_notify_pid = MyProcPid, + .bgw_main_arg = ((DbWorkerArg){ .s.db = db, .s.fast = fast }).d, + .bgw_start_rule = NULL, + }; + snprintf(database_worker.bgw_name, BGW_MAXLEN, "database_relsizes_collector_worker for %u", db); + old_ctx = MemoryContextSwitchTo(TopMemoryContext); + ret = RegisterDynamicBackgroundWorker(&database_worker, &handle); + MemoryContextSwitchTo(old_ctx); + if (!ret) { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("could not register background process"), + errhint("You may need to increase max_worker_processes."))); + } + pid_t pid; + status = WaitForBackgroundWorkerStartup(handle, &pid); + if (status == BGWH_STOPPED) + return; + if (status != BGWH_STARTED) { + ereport(WARNING, (errmsg("Failed to start background worker [%s], skipping", database_worker.bgw_name))); + return; + } + status = WaitForBackgroundWorkerShutdownSafely(handle); + if (status != BGWH_STOPPED) { + ereport(WARNING, (errmsg("Failure during background worker execution [%s], continuing", database_worker.bgw_name))); + /* Don't abort execution, just log and continue */ + } +} + +/* + * SQL-callable function to collect file statistics for a database. + * + * This function scans the filesystem directory corresponding to a database + * and returns statistics for all regular files found. It's designed to run + * on individual segments to collect local file information. + * + * The function: + * 1. Validates the function call context (must support returning a set) + * 2. Sets up a tuplestore for result collection + * 3. Scans the database directory (base//) + * 4. For each regular file, extracts relfilenode from filename + * 5. Collects file size and modification time via lstat() + * 6. Returns results as a set of tuples + * + * Parameters: + * Database OID (oid) - identifies which database directory to scan + * Fast (bool) - When true, don't sleep between each collect-phase for files + * + * Returns: + * Set of tuples containing: + * - segment: current segment ID + * - relfilenode: extracted from filename + * - filepath: full path to the file + * - size: file size in bytes + * - mtime: modification time as Unix timestamp + * + * Note: Includes configurable delays between file processing to reduce I/O load + */ +/* + * Scan a single directory and add file stats to the tuple store. + * Returns false if the directory could not be opened (non-fatal). + */ +static void +scan_db_dir(const char *dir_path, int segment_id, bool fast, + TupleDesc tupdesc, Tuplestorestate *tupstore) +{ + DIR *current_dir = AllocateDir(dir_path); + if (!current_dir) + { + ereport(WARNING, + (errmsg("get_stats_for_database: could not open directory \"%s\": %m", + dir_path))); + return; + } + + struct dirent *file; + while ((file = ReadDir(current_dir, dir_path)) != NULL) + { + char *filename = file->d_name; + if (strcmp(filename, ".") == 0 || strcmp(filename, "..") == 0) + continue; + + char *file_path = psprintf("%s/%s", dir_path, filename); + struct stat stb; + if (lstat(file_path, &stb) < 0) + { + ereport(WARNING, + (errmsg("get_stats_for_database: lstat failed for \"%s\" (unexpected behavior)", + file_path))); + pfree(file_path); + continue; + } + + if (S_ISREG(stb.st_mode)) + { + unsigned int relfilenode = fill_relfilenode(filename); + if (relfilenode == 0) + { + /* Skip non-relation files (PG_VERSION, pg_filenode.map, etc.) */ + pfree(file_path); + continue; + } + + Datum outputValues[FILEINFO_ARGS_CNT]; + bool outputNulls[FILEINFO_ARGS_CNT] = { false }; + + outputValues[0] = Int32GetDatum(segment_id); + outputValues[1] = ObjectIdGetDatum(relfilenode); + outputValues[2] = CStringGetTextDatum(file_path); + outputValues[3] = Int64GetDatum(stb.st_size); + outputValues[4] = Int64GetDatum(stb.st_mtime); + + tuplestore_putvalues(tupstore, tupdesc, outputValues, outputNulls); + + if (fast) + CHECK_FOR_INTERRUPTS(); + else + { + int retcode = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + worker_file_naptime, WAIT_EVENT_BUFFER_IO); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + if (retcode & WL_POSTMASTER_DEATH) + proc_exit(1); + } + } + pfree(file_path); + } + + FreeDir(current_dir); +} + +Datum get_stats_for_database(PG_FUNCTION_ARGS) { + int segment_id = GpIdentity.segindex; + Oid dboid = PG_GETARG_OID(0); + bool fast = (PG_NARGS() < 2) ? false : PG_GETARG_BOOL(1); + + const char *error = NULL; + + ReturnSetInfo *rsinfo = (ReturnSetInfo *)fcinfo->resultinfo; + /* Validate function call context */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) { + error = "get_stats_for_database: set-valued function called in context that cannot accept a set"; + goto finish_data; + } + if (!(rsinfo->allowedModes & SFRM_Materialize)) { + error = "get_stats_for_database: materialize mode required, but it is not allowed in this context"; + goto finish_data; + } + + /* Setup output tuple store */ + MemoryContext oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); + TupleDesc tupdesc; + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) { + MemoryContextSwitchTo(oldcontext); + error = "get_stats_for_database: incorrect return type in fcinfo (must be a row type)"; + goto finish_data; + } + tupdesc = BlessTupleDesc(tupdesc); + + bool randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + Tuplestorestate *tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + /* Scan default tablespace: $DataDir/base/ */ + { + char *default_dir = psprintf("%s/base/%u", DataDir, dboid); + scan_db_dir(default_dir, segment_id, fast, tupdesc, tupstore); + pfree(default_dir); + } + + /* Scan non-default tablespaces: $DataDir/pg_tblspc/// */ + { + char *tblspc_base = psprintf("%s/pg_tblspc", DataDir); + DIR *tblspc_dir = AllocateDir(tblspc_base); + if (tblspc_dir) + { + struct dirent *spc_entry; + while ((spc_entry = ReadDir(tblspc_dir, tblspc_base)) != NULL) + { + if (strcmp(spc_entry->d_name, ".") == 0 || + strcmp(spc_entry->d_name, "..") == 0) + continue; + + /* + * Each entry is a symlink to the tablespace directory. + * Inside it there is a version subdirectory (e.g. PG_14_202107181) + * and then per-database subdirectories named by dboid. + */ + char *spc_path = psprintf("%s/%s", tblspc_base, spc_entry->d_name); + DIR *ver_dir = AllocateDir(spc_path); + if (ver_dir) + { + struct dirent *ver_entry; + while ((ver_entry = ReadDir(ver_dir, spc_path)) != NULL) + { + if (strcmp(ver_entry->d_name, ".") == 0 || + strcmp(ver_entry->d_name, "..") == 0) + continue; + + char *db_dir = psprintf("%s/%s/%u", + spc_path, ver_entry->d_name, dboid); + scan_db_dir(db_dir, segment_id, fast, tupdesc, tupstore); + pfree(db_dir); + } + FreeDir(ver_dir); + } + pfree(spc_path); + } + FreeDir(tblspc_dir); + } + pfree(tblspc_base); + } + +finish_data: + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + /* Don't abort execution, return result */ + } + + return (Datum)0; +} + +/* + * Orchestrate statistics collection across multiple databases. + * + * This function iterates through a list of databases and spawns a background + * worker for each one to collect file statistics. It implements load balancing + * by distributing the database processing naptime across all databases. + * + * The function: + * 1. Iterates through the provided database list + * 2. Spawns a background worker for each database + * 3. Waits between databases based on configured naptime + * 4. Handles interrupts and postmaster death gracefully + * + * Parameters: + * databases_oids - Array of [name, oid] pairs for databases to process + * databases_cnt - Number of databases in the array + * fast - Don't make pauses + * + * Note: The inter-database naptime is divided by the number of databases + * to maintain consistent overall collection timing. + */ +static void get_stats_for_databases(Oid *databases_oids, int databases_cnt, bool fast) { + for (int i = 0; i < databases_cnt; ++i) { + run_database_stats_worker(fast, databases_oids[i]); + + if (fast) + CHECK_FOR_INTERRUPTS(); + else { + int naptime = (databases_cnt > 0) ? (worker_database_naptime / databases_cnt) : worker_database_naptime; + int retcode = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, naptime, WAIT_EVENT_BGWORKER_STARTUP); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + /* emergency bailout if postmaster has died */ + if (retcode & WL_POSTMASTER_DEATH) { + proc_exit(1); + } + } + } +} + +/* + * Check if the gp_relsizes_stats extension is installed in the current database. + * + * This function queries pg_extension to verify that the extension has been + * properly installed before attempting to collect statistics. This prevents + * errors when the background worker tries to access extension-specific tables + * and functions. + * + * Returns: + * Number of matching extension records (should be 1 if installed), + * or -1 on query execution error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int plugin_created() { + char *sql = "SELECT * FROM pg_extension WHERE extname = 'gp_relsizes_stats'"; + pgstat_report_activity(STATE_RUNNING, sql); + int retcode = SPI_execute(sql, true, 0); + pgstat_report_activity(STATE_IDLE, NULL); + return (retcode == SPI_OK_SELECT ? SPI_processed : -1); +} + +/* + * Clear all data from the table_sizes_history table. + * + * This function deletes the historical statistics table as part of the + * statistics refresh process. The table is cleared before inserting new + * current statistics to maintain a snapshot of table sizes at collection time. + * + * Returns: + * 0 on successful truncation, -1 on error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int delete_data_in_history() { + char *sql = "DELETE FROM relsizes_stats_schema.table_sizes_history"; + + pgstat_report_activity(STATE_RUNNING, sql); + return (SPI_execute(sql, false, 0) == SPI_OK_DELETE ? 0 : -1); +} + +/* + * Insert current table size statistics into the history table. + * + * This function populates the table_sizes_history table with current + * statistics from the table_sizes view, adding the current date as + * the collection timestamp. This creates a historical record of + * table sizes for trend analysis. + * + * Returns: + * 0 on successful insertion, -1 on error or if no rows were inserted + * + * Note: This function assumes it's running within an active SPI context. + */ +static int put_data_into_history() { + char *sql = "INSERT INTO relsizes_stats_schema.table_sizes_history SELECT CURRENT_DATE, * FROM " + "relsizes_stats_schema.table_sizes"; + + pgstat_report_activity(STATE_RUNNING, sql); + return (SPI_execute(sql, false, 0) == SPI_OK_INSERT && SPI_processed >= 0 ? 0 : -1); +} + +/* + * Refresh the table_sizes_history table with current statistics. + * + * This function implements a complete refresh of the historical statistics + * table by first clearing all existing data and then inserting fresh + * statistics from the current collection. This ensures the history table + * contains a consistent snapshot of table sizes at the time of collection. + * + * The function performs these operations: + * 1. Deletes from the existing history table + * 2. Inserts current statistics with today's date + * + * Returns: + * 0 on successful update, negative value on error + * + * Note: This function assumes it's running within an active SPI context. + * Errors are logged as warnings but don't abort the operation. + */ +static int update_table_sizes_history() { + int retcode = 0; + char *error = NULL; + + retcode = delete_data_in_history(); + if (retcode < 0) { + error = "update_table_sizes_history: delete old data failed"; + goto cleanup; + } + + retcode = put_data_into_history(); + if (retcode < 0) { + error = "update_table_sizes_history: put actual data into history failed"; + } + +cleanup: + pgstat_report_activity(STATE_IDLE, NULL); + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + } + return retcode; +} + +/* + * One cycle of the main background worker. + * + * The function performs these operations: + * 1. Retrieves list of all user databases + * 2. Spawns background workers to collect statistics for each database + * 3. Waits for all workers to complete before returning + * + * Parameters: + * from_worker - true when the worker calls the function, false when + * the function is called from user query. + */ +static void relsizes_collect_stats_once_internal(bool from_worker) { + int databases_cnt; + Oid *databases_oids; + + databases_oids = get_databases_oids(&databases_cnt, CurrentMemoryContext, from_worker); + if (databases_oids != NULL) { + get_stats_for_databases(databases_oids, databases_cnt, !from_worker); + pfree(databases_oids); + } else { + ereport(WARNING, (errmsg("Failed to get database OIDs"))); + } +} + +/* + * Main background worker entry point for continuous statistics collection. + * + * This function implements the main loop for the primary background worker + * that periodically collects table size statistics across all databases. + * It runs continuously until terminated by a SIGTERM signal. + * + * The worker performs these operations in each cycle: + * 1. Checks if the extension is enabled via GUC parameter + * 2. Collects statistics for each user database + * 3. Sleeps for the configured restart_naptime before next cycle + * + * The function handles: + * - Graceful shutdown on SIGTERM + * - Postmaster death detection + * - Configuration changes (enable/disable) + * - Database list changes between cycles + * - Error recovery (continues operation if individual database fails) + * + * Parameters: + * main_arg - Background worker main argument (currently unused) + * + * Note: This function should only be called via the background worker + * framework and runs in the "postgres" database context. + */ +PGDLLEXPORT void relsizes_collect_stats(Datum main_arg) { + optimizer = false; + pqsignal(SIGTERM, worker_sigterm); + pqsignal(SIGHUP, worker_sighup); + BackgroundWorkerUnblockSignals(); + BackgroundWorkerInitializeConnection("postgres", NULL, 0); + + while (!got_sigterm) { + if (got_sighup) { + got_sighup = false; + ProcessConfigFile(PGC_SIGHUP); + } + if (enabled) + relsizes_collect_stats_once_internal(true); + + int retcode = + WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, worker_restart_naptime, WAIT_EVENT_BGWORKER_STARTUP); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + if (retcode & WL_POSTMASTER_DEATH) { + proc_exit(1); + } + } +} + +/* + * SQL-callable function to perform one-time statistics collection. + * + * This function provides a way to manually trigger statistics collection + * for all databases without relying on the background worker. It's useful + * for on-demand collection, testing, or when the background worker is disabled. + * The function performs the same operations as one cycle of the main + * background worker. + * + * Unlike the continuous background worker, this function: + * - Runs in the context of the calling session + * - Does not check the enabled GUC parameter + * - Returns after a single collection cycle + * - Can be called from any database where the extension is installed + * + * Returns: + * void (success/failure indicated by exception or completion) + * + * Usage: + * SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + */ +Datum relsizes_collect_stats_once(PG_FUNCTION_ARGS) { + relsizes_collect_stats_once_internal(false); + + PG_RETURN_VOID(); +} + +/* + * Extension initialization function. + * + * This function is called when the extension library is loaded. It performs + * all necessary setup for the extension including: + * 1. Defining GUC (configuration) parameters + * 2. Registering the main background worker + * + * GUC Parameters defined: + * - gp_relsizes_stats.enabled: Enable/disable the background worker + * - gp_relsizes_stats.save_history: Enable saving table sizes to history table + * - gp_relsizes_stats.restart_naptime: Delay between collection cycles (ms) + * - gp_relsizes_stats.database_naptime: Delay between database processing (ms) + * - gp_relsizes_stats.file_naptime: Delay between file processing (ms) + * + * The function only registers the background worker if called during + * shared_preload_libraries processing, ensuring proper initialization order. + * + * Background Worker Configuration: + * - Name: "gp_relsizes_stats_worker" + * - Entry point: relsizes_collect_stats() + * - Database: "postgres" (for catalog access) + * - Restart: Never (manual restart required) + * - Start time: After recovery completion + * + * Note: This function is called automatically by PostgreSQL when the + * extension is loaded via shared_preload_libraries. + */ +void _PG_init(void) { + /* Define GUC variables */ + DefineCustomBoolVariable("gp_relsizes_stats.enabled", "Enable main background worker flag", NULL, &enabled, false, + PGC_SIGHUP, GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + DefineCustomBoolVariable("gp_relsizes_stats.save_history", + "Enable saving table sizes to history table.", + NULL, &save_history, true, + PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.restart_naptime", "Duration between every collect-phases (in ms).", NULL, + &worker_restart_naptime, + 6 * HOUR_TIME, /* 6 hours delay between collect-phases */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.database_naptime", "Duration between collect-phase for db (in ms).", + NULL, &worker_database_naptime, + 0, /* No delay between databases by default */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.file_naptime", "Duration between each collect-phase for files (in ms).", + NULL, &worker_file_naptime, + FILE_NAPTIME, /* 1ms delay between files */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + + if (process_shared_preload_libraries_in_progress) { + /* Configure and register main background worker */ + RegisterBackgroundWorker(&(BackgroundWorker){ + .bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION, + .bgw_start_time = BgWorkerStart_RecoveryFinished, + .bgw_restart_time = BGW_NEVER_RESTART, + .bgw_library_name = "gp_relsizes_stats", + .bgw_function_name = "relsizes_collect_stats", + .bgw_notify_pid = 0, + .bgw_start_rule = NULL, + .bgw_name = "gp_relsizes_stats_worker" + }); + } +} diff --git a/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out b/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out new file mode 100644 index 00000000000..0e00389d6f2 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out @@ -0,0 +1,251 @@ +\set QUIET off +SET client_min_messages TO error; +SET +CREATE EXTENSION gp_relsizes_stats; +CREATE EXTENSION +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); +CREATE TABLE +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); +INSERT 0 3 +-- Default is save_history = on — history write +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +------- + 65536 +(1 row) + +-- Fill table with a lot of different rows +insert into employees (first_name, last_name, department_id, date_of_birth) +select 'First' || i, 'Last' || i, (i % 10) + 1, DATE '1980-01-01' + (i % 365 * 365 / 30) +from generate_series(1, 10001)i; +INSERT 0 10001 +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +-- Check that collected stats are correct +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +-------- + 950272 +(1 row) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +-- Validate that after rerun stats collection size of table has not change +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +-------- + 950272 +(1 row) + +-- Cleanup +DROP TABLE employees; +DROP TABLE +-- +-- relsizes_collect_stats_once should collect files sizes without pauses +-- The naptime value is 1ms, so the pauses take at least 10s to process 10k files. +-- Check that relsizes_collect_stats_once completes in significantly less time. +SELECT EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) t1 \gset +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT (EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) - :t1) < 5; + ?column? +---------- + t +(1 row) + +-- Cleanup +DROP TABLE t; +DROP TABLE +-- +-- Check that schema size is calculated correctly when the schema +-- contains partitioned tables and ordinary ones. +CREATE SCHEMA test; +CREATE SCHEMA +CREATE TABLE test.t1 (i INT, j INT) +DISTRIBUTED BY (i) +PARTITION BY RANGE (i) + SUBPARTITION BY RANGE (j) + SUBPARTITION TEMPLATE (SUBPARTITION sp START (0) END (2) EVERY(1)) +(PARTITION p START (0) END (3) EVERY(1)); +CREATE TABLE +INSERT INTO test.t1 (i, j) +SELECT a % 3, a % 2 FROM generate_series(0, 2 * 3 - 1) a; +INSERT 0 6 +CREATE TABLE test.t2 AS SELECT 1 i DISTRIBUTED BY(i); +SELECT 1 +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT size = 32768 * 2 * 3 /* t1 */ + 32768 /* t2 */ + FROM relsizes_stats_schema.namespace_sizes + WHERE nspname = 'test'; + ?column? +---------- + t +(1 row) + +SELECT relname, segment, own_file, size + FROM relsizes_stats_schema.table_files + WHERE nspname = 'test' +ORDER BY relname, segment, own_file; + relname | segment | own_file | size +-------------------------+---------+----------+------- + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 32768 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 32768 + t1 | 0 | f | 0 + t1 | 0 | t | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 32768 + t1 | 1 | f | 0 + t1 | 1 | f | 32768 + t1 | 1 | f | 32768 + t1 | 1 | f | 32768 + t1 | 1 | f | 0 + t1 | 1 | t | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | t | 0 + t1_1_prt_p_1 | 0 | t | 0 + t1_1_prt_p_1 | 1 | t | 0 + t1_1_prt_p_1 | 2 | t | 0 + t1_1_prt_p_1_2_prt_sp_1 | 0 | t | 0 + t1_1_prt_p_1_2_prt_sp_1 | 1 | t | 32768 + t1_1_prt_p_1_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_1_2_prt_sp_2 | 0 | t | 0 + t1_1_prt_p_1_2_prt_sp_2 | 1 | t | 32768 + t1_1_prt_p_1_2_prt_sp_2 | 2 | t | 0 + t1_1_prt_p_2 | 0 | t | 0 + t1_1_prt_p_2 | 1 | t | 0 + t1_1_prt_p_2 | 2 | t | 0 + t1_1_prt_p_2_2_prt_sp_1 | 0 | t | 0 + t1_1_prt_p_2_2_prt_sp_1 | 1 | t | 32768 + t1_1_prt_p_2_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_2_2_prt_sp_2 | 0 | t | 0 + t1_1_prt_p_2_2_prt_sp_2 | 1 | t | 32768 + t1_1_prt_p_2_2_prt_sp_2 | 2 | t | 0 + t1_1_prt_p_3 | 0 | t | 0 + t1_1_prt_p_3 | 1 | t | 0 + t1_1_prt_p_3 | 2 | t | 0 + t1_1_prt_p_3_2_prt_sp_1 | 0 | t | 32768 + t1_1_prt_p_3_2_prt_sp_1 | 1 | t | 0 + t1_1_prt_p_3_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_3_2_prt_sp_2 | 0 | t | 32768 + t1_1_prt_p_3_2_prt_sp_2 | 1 | t | 0 + t1_1_prt_p_3_2_prt_sp_2 | 2 | t | 0 + t2 | 0 | t | 0 + t2 | 1 | t | 32768 + t2 | 2 | t | 0 +(60 rows) + +DROP SCHEMA test CASCADE; +DROP SCHEMA +-- +-- Check that save_history option controls writing to table_sizes_history +-- start_ignore +DROP TABLE IF EXISTS t_history_test; +DROP TABLE +-- end_ignore +CREATE TABLE t_history_test (i INT) DISTRIBUTED BY (i); +CREATE TABLE +INSERT INTO t_history_test VALUES (1); +INSERT 0 1 +-- Disable option - history should not write +ALTER SYSTEM SET gp_relsizes_stats.save_history = off; +ALTER SYSTEM +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT count(*) FROM relsizes_stats_schema.table_sizes_history + WHERE relname = 't_history_test'; + count +------- + 0 +(1 row) + +-- Enable option - history should write +ALTER SYSTEM SET gp_relsizes_stats.save_history = on; +ALTER SYSTEM +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT count(*) FROM relsizes_stats_schema.table_sizes_history + WHERE relname = 't_history_test'; + count +------- + 1 +(1 row) + +-- Cleanup +ALTER SYSTEM RESET gp_relsizes_stats.save_history; +ALTER SYSTEM +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +DROP TABLE t_history_test; +DROP TABLE diff --git a/gpcontrib/gp_relsizes_stats/test/expected/grants.out b/gpcontrib/gp_relsizes_stats/test/expected/grants.out new file mode 100644 index 00000000000..c04cc1e8306 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/expected/grants.out @@ -0,0 +1,107 @@ +-- Check that user who has created gp_relsizes_stats have privileges to use all +-- tables, views and functions from the extension. +-- Check that this user can grant this privileges to others. +SELECT '\! cp "' || setting || '/pg_hba.conf" "' || setting || '/pg_hba.conf.backup"' as cp_backup +FROM pg_settings +WHERE name = 'data_directory' \gset +:cp_backup +SELECT '\! { echo "local all user1,user2 trust"; ' + || 'echo "host all user1,user2 127.0.0.1/32 trust"; ' + || 'echo "host all user1,user2 ::1/128 trust"; ' + || '} >> ' || setting || '/pg_hba.conf' as add_users +FROM pg_settings +WHERE name = 'data_directory' \gset +:add_users +CREATE ROLE user1 LOGIN RESOURCE QUEUE pg_default; +CREATE ROLE user2 LOGIN RESOURCE QUEUE pg_default; +CREATE DATABASE db1 OWNER user1; +\set initial_user :USER +\set initial_db :DBNAME +\c db1 user1 +CREATE EXTENSION gp_relsizes_stats; +-- Check that user who has created gp_relsizes_stats can use the extension +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +-- +(0 rows) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; +-- +(0 rows) + +-- Check that user who has created gp_relsizes_stats can grant privileges to others +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; +-- Check that user2 has got required privileges +\c - user2 +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +-- +(0 rows) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; +-- +(0 rows) + +-- Cleanup +\c :"initial_db" :"initial_user" +DROP DATABASE db1; +DROP ROLE user1, user2; +SELECT '\! mv "' || setting || '/pg_hba.conf.backup" "' || setting || '/pg_hba.conf"' as cp_restore +FROM pg_settings +WHERE name = 'data_directory' \gset +:cp_restore diff --git a/gpcontrib/gp_relsizes_stats/test/postgresql.conf.add b/gpcontrib/gp_relsizes_stats/test/postgresql.conf.add new file mode 100644 index 00000000000..3c2492efd93 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/postgresql.conf.add @@ -0,0 +1 @@ +shared_preload_libraries = 'gp_relsizes_stats' diff --git a/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql b/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql new file mode 100644 index 00000000000..5ff0c1c8bf8 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql @@ -0,0 +1,128 @@ +\set QUIET off +SET client_min_messages TO error; +CREATE EXTENSION gp_relsizes_stats; + +-- start_ignore +DROP TABLE IF EXISTS employees; +-- end_ignore +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); + +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); + +-- Default is save_history = on — history write +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +-- Fill table with a lot of different rows +insert into employees (first_name, last_name, department_id, date_of_birth) +select 'First' || i, 'Last' || i, (i % 10) + 1, DATE '1980-01-01' + (i % 365 * 365 / 30) +from generate_series(1, 10001)i; + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +-- Check that collected stats are correct +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +-- Validate that after rerun stats collection size of table has not change +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +-- Cleanup +DROP TABLE employees; + + +-- +-- relsizes_collect_stats_once should collect files sizes without pauses +-- The naptime value is 1ms, so the pauses take at least 10s to process 10k files. +-- Check that relsizes_collect_stats_once completes in significantly less time. + +-- start_ignore +DROP TABLE IF EXISTS t; +CREATE TABLE t (i int) +DISTRIBUTED RANDOMLY +PARTITION BY RANGE (i) (PARTITION a START (0) END (10000) EVERY (1)); +-- end_ignore + +SELECT EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) t1 \gset + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT (EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) - :t1) < 5; + +-- Cleanup +DROP TABLE t; + + +-- +-- Check that schema size is calculated correctly when the schema +-- contains partitioned tables and ordinary ones. + +-- start_ignore +DROP SCHEMA IF EXISTS test CASCADE; +-- end_ignore +CREATE SCHEMA test; + +CREATE TABLE test.t1 (i INT, j INT) +DISTRIBUTED BY (i) +PARTITION BY RANGE (i) + SUBPARTITION BY RANGE (j) + SUBPARTITION TEMPLATE (SUBPARTITION sp START (0) END (2) EVERY(1)) +(PARTITION p START (0) END (3) EVERY(1)); + +INSERT INTO test.t1 (i, j) +SELECT a % 3, a % 2 FROM generate_series(0, 2 * 3 - 1) a; + +CREATE TABLE test.t2 AS SELECT 1 i DISTRIBUTED BY(i); + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size = 32768 * 2 * 3 /* t1 */ + 32768 /* t2 */ + FROM relsizes_stats_schema.namespace_sizes + WHERE nspname = 'test'; + +SELECT relname, segment, own_file, size + FROM relsizes_stats_schema.table_files + WHERE nspname = 'test' +ORDER BY relname, segment, own_file; + +DROP SCHEMA test CASCADE; + + +-- +-- Check that save_history option controls writing to table_sizes_history + +-- start_ignore +DROP TABLE IF EXISTS t_history_test; +-- end_ignore +CREATE TABLE t_history_test (i INT) DISTRIBUTED BY (i); +INSERT INTO t_history_test VALUES (1); + +-- Disable option - history should not write +ALTER SYSTEM SET gp_relsizes_stats.save_history = off; +SELECT pg_reload_conf(); +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); +SELECT count(*) FROM relsizes_stats_schema.table_sizes_history + WHERE relname = 't_history_test'; + +-- Enable option - history should write +ALTER SYSTEM SET gp_relsizes_stats.save_history = on; +SELECT pg_reload_conf(); +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); +SELECT count(*) FROM relsizes_stats_schema.table_sizes_history + WHERE relname = 't_history_test'; + +-- Cleanup +ALTER SYSTEM RESET gp_relsizes_stats.save_history; +SELECT pg_reload_conf(); +DROP TABLE t_history_test; diff --git a/gpcontrib/gp_relsizes_stats/test/sql/grants.sql b/gpcontrib/gp_relsizes_stats/test/sql/grants.sql new file mode 100644 index 00000000000..46fe68ccbfd --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/sql/grants.sql @@ -0,0 +1,85 @@ +-- Check that user who has created gp_relsizes_stats have privileges to use all +-- tables, views and functions from the extension. +-- Check that this user can grant this privileges to others. + +-- start_ignore +DROP DATABASE IF EXISTS db1; +DROP ROLE IF EXISTS user1, user2; +-- end_ignore + +SELECT '\! cp "' || setting || '/pg_hba.conf" "' || setting || '/pg_hba.conf.backup"' as cp_backup +FROM pg_settings +WHERE name = 'data_directory' \gset + +:cp_backup + +SELECT '\! { echo "local all user1,user2 trust"; ' + || 'echo "host all user1,user2 127.0.0.1/32 trust"; ' + || 'echo "host all user1,user2 ::1/128 trust"; ' + || '} >> ' || setting || '/pg_hba.conf' as add_users +FROM pg_settings +WHERE name = 'data_directory' \gset + +:add_users + +-- start_ignore +\! gpstop -u +-- end_ignore + +CREATE ROLE user1 LOGIN RESOURCE QUEUE pg_default; +CREATE ROLE user2 LOGIN RESOURCE QUEUE pg_default; +CREATE DATABASE db1 OWNER user1; + +\set initial_user :USER +\set initial_db :DBNAME + +\c db1 user1 + +CREATE EXTENSION gp_relsizes_stats; + +-- Check that user who has created gp_relsizes_stats can use the extension +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; + +-- Check that user who has created gp_relsizes_stats can grant privileges to others +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; + +-- Check that user2 has got required privileges +\c - user2 +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; + +-- Cleanup +\c :"initial_db" :"initial_user" + +DROP DATABASE db1; +DROP ROLE user1, user2; + +SELECT '\! mv "' || setting || '/pg_hba.conf.backup" "' || setting || '/pg_hba.conf"' as cp_restore +FROM pg_settings +WHERE name = 'data_directory' \gset + +:cp_restore + +-- start_ignore +\! gpstop -u +-- end_ignore diff --git a/gpcontrib/pg_hint_plan/sql/maskout.sh b/gpcontrib/pg_hint_plan/sql/maskout.sh index 141d679479f..43ce959fabf 100755 --- a/gpcontrib/pg_hint_plan/sql/maskout.sh +++ b/gpcontrib/pg_hint_plan/sql/maskout.sh @@ -1,4 +1,4 @@ #! /bin/sh cat $1 | \ sed 's/cost=10\{7\}[\.0-9]\+ /cost={inf}..{inf} /;s/cost=[\.0-9]\+ /cost=xxx..xxx /;s/width=[0-9]\+\([^0-9]\)/width=xxx\1/;s/^ *QUERY PLAN *$/ QUERY PLAN/;s/^--*$/----------------/' |\ -egrep -v "^ *((Planning time|JIT|Functions|Options):|\([0-9]* rows\))" +grep -E -v "^ *((Planning time|JIT|Functions|Options):|\([0-9]* rows\))" diff --git a/pom.xml b/pom.xml index 710105b720e..4045e1d7ed4 100644 --- a/pom.xml +++ b/pom.xml @@ -1488,6 +1488,14 @@ code or new licensing patterns. gpcontrib/yezzey/yezzey/devops/packaging/ubuntu/script/build_cloudberry_deb.sh gpcontrib/yezzey/ystat.h + gpcontrib/gp_relsizes_stats/Makefile + gpcontrib/gp_relsizes_stats/.clang-format + gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control + gpcontrib/gp_relsizes_stats/test/postgresql.conf.add + + gpcontrib/reject_partition_fullscan/Makefile + gpcontrib/reject_partition_fullscan/reject_partition_fullscan.control + diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index 6084abe1c62..d350326fd4c 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -4014,6 +4014,49 @@ CTranslatorDXLToPlStmt::CreateProjectSetNodeTree(const CDXLNode *result_dxlnode, return project_set_parent_plan; } +//--------------------------------------------------------------------------- +// @function: +// restore_unknown_locale_resname +// +// @doc: +// ORCA represents strings using wide characters. Converting a multibyte +// name to wide format uses vswprintf(), which depends on the database +// LC_CTYPE. When that locale cannot interpret the name (e.g. LC_CTYPE=C +// with a UTF-8 alias), ORCA substitutes the generic "UNKNOWN" string +// (see gpos::clib::Vswprintf). This function restores the original name +// from the query tree. +// +// Only the topmost plan node is translated with a context that carries +// the original query (see GetPlannedStmtFromDXL); everywhere else query +// is NULL and this is a no-op. The topmost projection list produces the +// query's output columns in order, so the original name is the non-junk +// query targetList entry with the same resno. Matching only top-level +// entries (never descending into subqueries or expressions) also keeps +// a legitimate alias named "UNKNOWN" intact: its positional match is +// the entry itself, making the restore a no-op. +//--------------------------------------------------------------------------- +static void +restore_unknown_locale_resname(const Query *query, TargetEntry *target_entry) +{ + if (nullptr == query || 0 != strcmp(target_entry->resname, "UNKNOWN")) + { + return; + } + + ListCell *lc; + ForEach(lc, query->targetList) + { + TargetEntry *te = (TargetEntry *) lfirst(lc); + + if (!te->resjunk && nullptr != te->resname && + te->resno == target_entry->resno) + { + target_entry->resname = te->resname; + return; + } + } +} + //------------------------------------------------------------------------------ // If a result plan node is not required on top of a project set node then the // alias parameter needs to be set for all the project set nodes else not @@ -4023,7 +4066,7 @@ CTranslatorDXLToPlStmt::CreateProjectSetNodeTree(const CDXLNode *result_dxlnode, void SetupAliasParameter(const BOOL will_require_result_node, const CDXLNode *project_list_dxlnode, - Plan *project_set_parent_plan) + Plan *project_set_parent_plan, const Query *query) { if (!will_require_result_node) { @@ -4052,6 +4095,9 @@ SetupAliasParameter(const BOOL will_require_result_node, sc_proj_elem_dxlop->GetMdNameAlias() ->GetMDName() ->GetBuffer()); + + // restore aliases that failed the wide character conversion + restore_unknown_locale_resname(query, te); ul++; } } @@ -4226,7 +4272,7 @@ CTranslatorDXLToPlStmt::TranslateDXLResult( } SetupAliasParameter(will_require_result_node, project_list_dxlnode, - project_set_parent_plan); + project_set_parent_plan, output_context->GetQuery()); Plan *final_plan = nullptr; @@ -4483,6 +4529,10 @@ CTranslatorDXLToPlStmt::TranslateDXLAppend( sc_proj_elem_dxlop->GetMdNameAlias()->GetMDName()->GetBuffer()); target_entry->resno = attno; + // restore aliases that failed the wide character conversion + restore_unknown_locale_resname(output_context->GetQuery(), + target_entry); + // add column mapping to output translation context output_context->InsertMapping(sc_proj_elem_dxlop->Id(), target_entry); @@ -5882,51 +5932,6 @@ CTranslatorDXLToPlStmt::ProcessDXLTblDescr( return index; } -//--------------------------------------------------------------------------- -// @function: -// update_unknown_locale_walker -// -// @doc: -// Given an expression tree and a TargetEntry pointer context, look for a -// matching target entry in the expression tree and overwrite the given -// TargetEntry context's resname with the original found in the expression -// tree. -// -//--------------------------------------------------------------------------- -static bool -update_unknown_locale_walker(Node *node, void *context) -{ - if (node == nullptr) - { - return false; - } - - TargetEntry *unknown_target_entry = (TargetEntry *) context; - - if (IsA(node, TargetEntry)) - { - TargetEntry *te = (TargetEntry *) node; - - if (te->resorigtbl == unknown_target_entry->resorigtbl && - te->resno == unknown_target_entry->resno) - { - unknown_target_entry->resname = te->resname; - return false; - } - } - else if (IsA(node, Query)) - { - Query *query = (Query *) node; - - return gpdb::WalkExpressionTree( - (Node *) query->targetList, - (bool (*)(Node *, void *)) update_unknown_locale_walker, (void *) context); - } - - return gpdb::WalkExpressionTree( - node, (bool (*)(Node *, void *)) update_unknown_locale_walker, (void *) context); -} - //--------------------------------------------------------------------------- // @function: // CTranslatorDXLToPlStmt::TranslateDXLProjList @@ -6031,24 +6036,15 @@ CTranslatorDXLToPlStmt::TranslateDXLProjList( } target_entry->resorigtbl = pteOriginal->resorigtbl; target_entry->resorigcol = pteOriginal->resorigcol; - - // ORCA represents strings using wide characters. That can - // require converting from multibyte characters using - // vswprintf(). However, vswprintf() is dependent on the system - // locale which is set at the database level. When that locale - // cannot interpret the string correctly, it fails. ORCA - // bypasses the failure by using a generic "UNKNOWN" string. - // When that happens, the following code translates it back to - // the original multibyte string. - if (strcmp(target_entry->resname, "UNKNOWN") == 0) - { - update_unknown_locale_walker( - (Node *) output_context->GetQuery(), - (void *) target_entry); - } } } + // restore aliases that failed the wide character conversion; this + // must cover not only Vars but also other expressions (e.g. Consts + // and Aggrefs) whose aliases can equally fail the conversion + restore_unknown_locale_resname(output_context->GetQuery(), + target_entry); + // add column mapping to output translation context output_context->InsertMapping(sc_proj_elem_dxlop->Id(), target_entry); diff --git a/src/backend/gpopt/translate/CTranslatorUtils.cpp b/src/backend/gpopt/translate/CTranslatorUtils.cpp index 0887c72725c..90e0a70ba23 100644 --- a/src/backend/gpopt/translate/CTranslatorUtils.cpp +++ b/src/backend/gpopt/translate/CTranslatorUtils.cpp @@ -918,7 +918,7 @@ CTranslatorUtils::GetColumnAttnosForGroupBy( case GROUPING_SET_EMPTY: { col_attnos_arr_current = GPOS_NEW(mp) CBitSetArray(mp); - CBitSet *bset = GPOS_NEW(mp) CBitSet(mp); + CBitSet *bset = GPOS_NEW(mp) CBitSet(mp, num_cols); col_attnos_arr_current->Append(bset); break; } @@ -1113,11 +1113,12 @@ CTranslatorUtils::CreateGroupingSetsForRollup(CMemoryPool *mp, GPOS_ASSERT(grouping_set->kind == GROUPING_SET_ROLLUP); CBitSetArray *col_attnos_arr = GPOS_NEW(mp) CBitSetArray(mp); ListCell *lc = nullptr; - CBitSet *current_result = GPOS_NEW(mp) CBitSet(mp); + + CBitSet *current_result = GPOS_NEW(mp) CBitSet(mp, num_cols); // Maintaining the order of grouping sets is essential because the // UnionAll operator matches each child's distribution with the // distribution of the first child - col_attnos_arr->Append(GPOS_NEW(mp) CBitSet(mp)); + col_attnos_arr->Append(GPOS_NEW(mp) CBitSet(mp, num_cols)); ForEach(lc, grouping_set->content) { GroupingSet *gs_current = (GroupingSet *) lfirst(lc); @@ -1152,8 +1153,9 @@ CTranslatorUtils::CreateGroupingSetsForCube(CMemoryPool *mp, GPOS_ASSERT(grouping_set->kind == GROUPING_SET_CUBE); CBitSetArray *col_attnos_arr = GPOS_NEW(mp) CBitSetArray(mp); - // add an empty set - col_attnos_arr->Append(GPOS_NEW(mp) CBitSet(mp)); + // add an empty set — vec_size must match what CreateAttnoSetForGroupingSet + // produces (num_cols), otherwise Union below leaves misaligned links. + col_attnos_arr->Append(GPOS_NEW(mp) CBitSet(mp, num_cols)); ListCell *lc = nullptr; ForEach(lc, grouping_set->content) diff --git a/src/backend/gporca/libgpos/src/common/CBitSet.cpp b/src/backend/gporca/libgpos/src/common/CBitSet.cpp index 1eb9d2c939f..c3d7703f59d 100644 --- a/src/backend/gporca/libgpos/src/common/CBitSet.cpp +++ b/src/backend/gporca/libgpos/src/common/CBitSet.cpp @@ -347,6 +347,8 @@ CBitSet::ExchangeClear(ULONG pos) void CBitSet::Union(const CBitSet *pbsOther) { + GPOS_ASSERT(m_vector_size == pbsOther->m_vector_size); + CBitSetLink *bsl = nullptr; CBitSetLink *bsl_other = nullptr; @@ -425,6 +427,10 @@ CBitSet::Intersection(const CBitSet *pbsOther) return; } + // See CBitSet::Union: link offsets depend on m_vector_size, so mixing + // bitsets with different sizes makes FindLinkByOffset miss links. + GPOS_ASSERT(m_vector_size == pbsOther->m_vector_size); + CBitSetLink *bsl_other = nullptr; CBitSetLink *bsl = m_bsllist.First(); diff --git a/src/common/stringinfo.c b/src/common/stringinfo.c index ec66210adb0..8a6e498b44d 100644 --- a/src/common/stringinfo.c +++ b/src/common/stringinfo.c @@ -361,16 +361,32 @@ enlargeStringInfo(StringInfo str, int needed) void replaceStringInfoString(StringInfo str, char *replace, char *replacement) { - char *ptr; + char *match_ptr = NULL; + char *start_ptr = str->data; + char *dup = NULL; + size_t replace_len = strlen(replace); - while ((ptr = strstr(str->data, replace)) != NULL) - { - char *dup = pstrdup(str->data); + // prevent empty loop, because strstr will always return start_ptr + if (replace_len == 0) + return; - resetStringInfo(str); - appendBinaryStringInfo(str, dup, ptr - str->data); + while ((match_ptr = strstr(start_ptr, replace)) != NULL) + { + if (dup == NULL) + { + dup = pstrdup(str->data); + start_ptr = dup; + match_ptr = dup + (match_ptr - str->data); + resetStringInfo(str); + } + + appendBinaryStringInfo(str, start_ptr, match_ptr - start_ptr); appendStringInfoString(str, replacement); - appendStringInfoString(str, dup + (ptr - str->data) + strlen(replace)); + start_ptr = match_ptr + replace_len; + } + if (dup != NULL) + { + appendStringInfoString(str, start_ptr); pfree(dup); } } diff --git a/src/test/regress/expected/gp_locale.out b/src/test/regress/expected/gp_locale.out index 0d916a93d70..fd2a65c01a3 100644 --- a/src/test/regress/expected/gp_locale.out +++ b/src/test/regress/expected/gp_locale.out @@ -87,4 +87,61 @@ SELECT * FROM hi_안녕세계 hi_안녕세계1, hi_안녕세계 hi_안녕세계2 1 | 안녕세계1 first UPDATE | 안녕세2 first | 안녕세계3 first | 1 | 안녕세계1 first UPDATE | 안녕세2 first | 안녕세계3 first (1 row) +-- ALIAS ON CONSTANTS, AGGREGATES AND SET OPERATIONS +-- These project elements are not Vars, so restoring the alias after a failed +-- wide character conversion must not depend on the Var-origin lookup. +SELECT '한글' AS "한글"; + 한글 +------ + 한글 +(1 row) + +SELECT 1+1 AS 안녕세계표현식; + 안녕세계표현식 +---------------- + 2 +(1 row) + +SELECT count(*) AS 안녕세계카운트 FROM hi_안녕세계; + 안녕세계카운트 +---------------- + 1 +(1 row) + +SELECT sum(a) AS 안녕세계합계, max(a) AS 안녕세계최대 FROM hi_안녕세계; + 안녕세계합계 | 안녕세계최대 +--------------+-------------- + 1 | 1 +(1 row) + +SELECT '안녕' AS 안녕세계유니온 UNION ALL SELECT 'x'; + 안녕세계유니온 +---------------- + 안녕 + x +(2 rows) + +-- SET RETURNING FUNCTION (ProjectSet can be the topmost plan node) +SELECT generate_series(1,2) AS 안녕세계SRF; + 안녕세계srf +------------- + 1 + 2 +(2 rows) + +-- The restore must take the name from the top-level target list entry at the +-- same position, never from a same-position entry inside a subquery. +SELECT EXISTS(SELECT a, 안녕세계1 FROM hi_안녕세계) AS c, 안녕세계1 AS 안녕세계별칭 FROM hi_안녕세계; + c | 안녕세계별칭 +---+------------------------ + t | 안녕세계1 first UPDATE +(1 row) + +-- A legitimate alias named "UNKNOWN" (no conversion failure) must survive. +SELECT EXISTS(SELECT a, 안녕세계1 FROM hi_안녕세계) AS c, a AS "UNKNOWN" FROM hi_안녕세계; + c | UNKNOWN +---+--------- + t | 1 +(1 row) + RESET optimizer_trace_fallback; diff --git a/src/test/regress/expected/groupingsets.out b/src/test/regress/expected/groupingsets.out index 5222cc22b70..a1cb32e2478 100644 --- a/src/test/regress/expected/groupingsets.out +++ b/src/test/regress/expected/groupingsets.out @@ -2470,4 +2470,32 @@ group by rollup (a,b) order by a; | | 6 (8 rows) +-- ORCA: rollup over a derived-expression group alias with a target-list SRF. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by rollup(a, ab) order by 1,2; + g | ab +---+---- + 1 | 2 + 1 | 4 + 1 | + 1 | + 2 | 4 + 2 | +(6 rows) + +-- Same shape with cube(): exercises additional grouping-set combinations. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by cube(a, ab) order by 1,2; + g | ab +---+---- + 1 | 2 + 1 | 4 + 1 | + 1 | + 2 | 4 + 2 | +(6 rows) + -- end diff --git a/src/test/regress/expected/groupingsets_optimizer.out b/src/test/regress/expected/groupingsets_optimizer.out index a07017eca32..f02364c362b 100644 --- a/src/test/regress/expected/groupingsets_optimizer.out +++ b/src/test/regress/expected/groupingsets_optimizer.out @@ -2645,4 +2645,32 @@ group by rollup (a,b) order by a; | | 6 (8 rows) +-- ORCA: rollup over a derived-expression group alias with a target-list SRF. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by rollup(a, ab) order by 1,2; + g | ab +---+---- + 1 | 2 + 1 | 4 + 1 | + 1 | + 2 | 4 + 2 | +(6 rows) + +-- Same shape with cube(): exercises additional grouping-set combinations. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by cube(a, ab) order by 1,2; + g | ab +---+---- + 1 | 2 + 1 | 4 + 1 | + 1 | + 2 | 4 + 2 | +(6 rows) + -- end diff --git a/src/test/regress/sql/gp_locale.sql b/src/test/regress/sql/gp_locale.sql index 444352c9edd..2053242105e 100644 --- a/src/test/regress/sql/gp_locale.sql +++ b/src/test/regress/sql/gp_locale.sql @@ -58,4 +58,27 @@ WITH cte(안녕세계x, こんにちわx) AS -- JOIN SELECT * FROM hi_안녕세계 hi_안녕세계1, hi_안녕세계 hi_안녕세계2 WHERE hi_안녕세계1.안녕세계1 LIKE '%UPDATE'; +-- ALIAS ON CONSTANTS, AGGREGATES AND SET OPERATIONS +-- These project elements are not Vars, so restoring the alias after a failed +-- wide character conversion must not depend on the Var-origin lookup. +SELECT '한글' AS "한글"; + +SELECT 1+1 AS 안녕세계표현식; + +SELECT count(*) AS 안녕세계카운트 FROM hi_안녕세계; + +SELECT sum(a) AS 안녕세계합계, max(a) AS 안녕세계최대 FROM hi_안녕세계; + +SELECT '안녕' AS 안녕세계유니온 UNION ALL SELECT 'x'; + +-- SET RETURNING FUNCTION (ProjectSet can be the topmost plan node) +SELECT generate_series(1,2) AS 안녕세계SRF; + +-- The restore must take the name from the top-level target list entry at the +-- same position, never from a same-position entry inside a subquery. +SELECT EXISTS(SELECT a, 안녕세계1 FROM hi_안녕세계) AS c, 안녕세계1 AS 안녕세계별칭 FROM hi_안녕세계; + +-- A legitimate alias named "UNKNOWN" (no conversion failure) must survive. +SELECT EXISTS(SELECT a, 안녕세계1 FROM hi_안녕세계) AS c, a AS "UNKNOWN" FROM hi_안녕세계; + RESET optimizer_trace_fallback; diff --git a/src/test/regress/sql/groupingsets.sql b/src/test/regress/sql/groupingsets.sql index 851a1eea6bb..f7bcba8a46a 100644 --- a/src/test/regress/sql/groupingsets.sql +++ b/src/test/regress/sql/groupingsets.sql @@ -723,4 +723,14 @@ select a, b, rank(b) within group (order by b nulls last) from (values (1,1),(1,4),(1,5),(3,1),(3,2)) v(a,b) group by rollup (a,b) order by a; +-- ORCA: rollup over a derived-expression group alias with a target-list SRF. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by rollup(a, ab) order by 1,2; + +-- Same shape with cube(): exercises additional grouping-set combinations. +select generate_series(1, a) g, a+b ab + from (values (1,1),(2,2)) t(a,b) + group by cube(a, ab) order by 1,2; + -- end diff --git a/src/test/singlenode_regress/expected/AOCO_Compression.out b/src/test/singlenode_regress/expected/AOCO_Compression.out index 74eca4dc3b1..7fd03ee1e6e 100644 --- a/src/test/singlenode_regress/expected/AOCO_Compression.out +++ b/src/test/singlenode_regress/expected/AOCO_Compression.out @@ -3571,6 +3571,7 @@ Access method: ao_column -- When I insert data insert into mpp17012_compress_test2 values('a',generate_series(1,250),'ksjdhfksdhfksdhfksjhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh','bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('mpp17012_compress_test2')), get_ao_compression_ratio('mpp17012_compress_test2'); pg_size_pretty | get_ao_compression_ratio @@ -3578,6 +3579,7 @@ get_ao_compression_ratio('mpp17012_compress_test2'); 712 bytes | 36.75 (1 row) +-- end_ignore -- Test that an AO/CO table with compresstype zlib and invalid compress level will error at create create table a_aoco_table_with_zlib_and_invalid_compression_level(col text) WITH (APPENDONLY=true, COMPRESSTYPE=zlib, compresslevel=-1, ORIENTATION=column); ERROR: value -1 out of bounds for option "compresslevel" diff --git a/src/test/singlenode_regress/sql/AOCO_Compression.sql b/src/test/singlenode_regress/sql/AOCO_Compression.sql index 120270886c9..14116b2fed3 100644 --- a/src/test/singlenode_regress/sql/AOCO_Compression.sql +++ b/src/test/singlenode_regress/sql/AOCO_Compression.sql @@ -1747,8 +1747,10 @@ get_ao_compression_ratio('mpp17012_compress_test2'); -- When I insert data insert into mpp17012_compress_test2 values('a',generate_series(1,250),'ksjdhfksdhfksdhfksjhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh','bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); -- Then the data will be compressed according to a consistent compression ratio +-- start_ignore select pg_size_pretty(pg_relation_size('mpp17012_compress_test2')), get_ao_compression_ratio('mpp17012_compress_test2'); +-- end_ignore -- Test that an AO/CO table with compresstype zlib and invalid compress level will error at create create table a_aoco_table_with_zlib_and_invalid_compression_level(col text) WITH (APPENDONLY=true, COMPRESSTYPE=zlib, compresslevel=-1, ORIENTATION=column); diff --git a/src/test/unit/mock/gpopt_mock.c b/src/test/unit/mock/gpopt_mock.c index ae16a4cab75..4f70eb03898 100644 --- a/src/test/unit/mock/gpopt_mock.c +++ b/src/test/unit/mock/gpopt_mock.c @@ -1,5 +1,6 @@ #include "postgres.h" +#include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h"