From a1b29dffe0a88243f7487ab4a8f888c175a5bdb9 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Mon, 23 Mar 2026 19:27:53 +0100 Subject: [PATCH] fix CI (hopefully) --- .github/actions/prepare-mlir-cache/action.yml | 54 +++++++++++++ .../actions/prepare-protobuf-cache/action.yml | 45 +++++++++++ .../restore-raptor-build-cache/action.yml | 26 +++++++ .../save-raptor-build-cache/action.yml | 16 ++++ .github/workflows/build_mlir_cache.yml | 50 ------------ .github/workflows/validate_operations.yml | 78 ++++++++----------- 6 files changed, 173 insertions(+), 96 deletions(-) create mode 100644 .github/actions/prepare-mlir-cache/action.yml create mode 100644 .github/actions/prepare-protobuf-cache/action.yml create mode 100644 .github/actions/restore-raptor-build-cache/action.yml create mode 100644 .github/actions/save-raptor-build-cache/action.yml delete mode 100644 .github/workflows/build_mlir_cache.yml diff --git a/.github/actions/prepare-mlir-cache/action.yml b/.github/actions/prepare-mlir-cache/action.yml new file mode 100644 index 0000000..7227e32 --- /dev/null +++ b/.github/actions/prepare-mlir-cache/action.yml @@ -0,0 +1,54 @@ +name: Prepare MLIR Cache +description: Restore or build the cached MLIR toolchain. + +inputs: + llvm-commit: + description: LLVM commit to build. + required: true + mold-linker-flags: + description: Linker flags used to force mold. + required: true + +runs: + using: composite + steps: + - name: Restore MLIR cache + id: restore-mlir-cache + uses: actions/cache/restore@v4 + with: + path: onnx-mlir/llvm-project + key: mlir-${{ runner.os }}-${{ inputs.llvm-commit }} + + - name: Clone LLVM + if: steps.restore-mlir-cache.outputs.cache-hit != 'true' + shell: bash + run: | + git clone --filter=blob:none --no-checkout https://github.com/llvm/llvm-project.git onnx-mlir/llvm-project + cd onnx-mlir/llvm-project + git fetch --depth 1 origin ${{ inputs.llvm-commit }} + git checkout FETCH_HEAD + + - name: Build MLIR + if: steps.restore-mlir-cache.outputs.cache-hit != 'true' + shell: bash + run: | + cmake -S onnx-mlir/llvm-project/llvm -B onnx-mlir/llvm-project/build -G Ninja \ + -DLLVM_ENABLE_PROJECTS="mlir;clang" \ + -DLLVM_ENABLE_RUNTIMES="openmp" \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON \ + -DENABLE_LIBOMPTARGET=OFF \ + -DLLVM_ENABLE_LIBEDIT=OFF \ + -DCMAKE_EXE_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" \ + -DCMAKE_MODULE_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" + cmake --build onnx-mlir/llvm-project/build + + - name: Save MLIR cache + if: steps.restore-mlir-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: onnx-mlir/llvm-project + key: mlir-${{ runner.os }}-${{ inputs.llvm-commit }} diff --git a/.github/actions/prepare-protobuf-cache/action.yml b/.github/actions/prepare-protobuf-cache/action.yml new file mode 100644 index 0000000..a92ced6 --- /dev/null +++ b/.github/actions/prepare-protobuf-cache/action.yml @@ -0,0 +1,45 @@ +name: Prepare Protobuf Cache +description: Restore or build the cached protobuf installation. + +inputs: + protobuf-commit: + description: Protobuf tag or commit to build. + required: true + mold-linker-flags: + description: Linker flags used to force mold. + required: true + +runs: + using: composite + steps: + - name: Restore protobuf cache + id: restore-protobuf-cache + uses: actions/cache/restore@v4 + with: + path: | + /usr/local/lib/libproto* + /usr/local/include/google/protobuf + key: protobuf-${{ runner.os }}-${{ inputs.protobuf-commit }} + + - name: Install protobuf + if: steps.restore-protobuf-cache.outputs.cache-hit != 'true' + shell: bash + run: | + git clone --depth 1 --branch ${{ inputs.protobuf-commit }} https://github.com/protocolbuffers/protobuf + cmake -S protobuf -B protobuf/build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_EXE_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" \ + -DCMAKE_MODULE_LINKER_FLAGS="${{ inputs.mold-linker-flags }}" + cmake --build protobuf/build + sudo cmake --install protobuf/build + rm -rf protobuf + + - name: Save protobuf cache + if: steps.restore-protobuf-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + /usr/local/lib/libproto* + /usr/local/include/google/protobuf + key: protobuf-${{ runner.os }}-${{ inputs.protobuf-commit }} diff --git a/.github/actions/restore-raptor-build-cache/action.yml b/.github/actions/restore-raptor-build-cache/action.yml new file mode 100644 index 0000000..e00a491 --- /dev/null +++ b/.github/actions/restore-raptor-build-cache/action.yml @@ -0,0 +1,26 @@ +name: Restore Raptor Build Cache +description: Restore the cached raptor build directory for incremental builds. + +inputs: + key: + description: Exact cache key to restore. + required: true + restore-keys: + description: Prefixes used to restore the most recent compatible cache. + required: false + +outputs: + cache-hit: + description: Whether the exact cache key was restored. + value: ${{ steps.restore-raptor-build-cache.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Restore raptor build cache + id: restore-raptor-build-cache + uses: actions/cache/restore@v4 + with: + path: build + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} diff --git a/.github/actions/save-raptor-build-cache/action.yml b/.github/actions/save-raptor-build-cache/action.yml new file mode 100644 index 0000000..66b6976 --- /dev/null +++ b/.github/actions/save-raptor-build-cache/action.yml @@ -0,0 +1,16 @@ +name: Save Raptor Build Cache +description: Save the raptor build directory after a successful incremental build. + +inputs: + key: + description: Cache key used to save the build directory. + required: true + +runs: + using: composite + steps: + - name: Save raptor build cache + uses: actions/cache/save@v4 + with: + path: build + key: ${{ inputs.key }} diff --git a/.github/workflows/build_mlir_cache.yml b/.github/workflows/build_mlir_cache.yml deleted file mode 100644 index f9a557e..0000000 --- a/.github/workflows/build_mlir_cache.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build MLIR Cache - -on: - workflow_call: - inputs: - llvm-commit: - required: true - type: string - -jobs: - build-mlir: - runs-on: ubuntu-latest - - steps: - - name: Cache MLIR build - id: cache-mlir - uses: actions/cache@v4 - with: - path: onnx-mlir/llvm-project - key: mlir-${{ runner.os }}-${{ inputs.llvm-commit }} - - - name: Install build dependencies - if: steps.cache-mlir.outputs.cache-hit != 'true' - run: | - sudo apt update - sudo apt install -y cmake ninja-build build-essential - - - name: Clone LLVM - if: steps.cache-mlir.outputs.cache-hit != 'true' - run: | - git clone --filter=blob:none --no-checkout https://github.com/llvm/llvm-project.git onnx-mlir/llvm-project - cd onnx-mlir/llvm-project - git fetch --depth 1 origin ${{ inputs.llvm-commit }} - git checkout FETCH_HEAD - - - name: Build MLIR - if: steps.cache-mlir.outputs.cache-hit != 'true' - run: | - mkdir -p onnx-mlir/llvm-project/build - cd onnx-mlir/llvm-project/build - cmake -G Ninja ../llvm \ - -DLLVM_ENABLE_PROJECTS="mlir;clang" \ - -DLLVM_ENABLE_RUNTIMES="openmp" \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_RTTI=ON \ - -DENABLE_LIBOMPTARGET=OFF \ - -DLLVM_ENABLE_LIBEDIT=OFF - cmake --build . diff --git a/.github/workflows/validate_operations.yml b/.github/workflows/validate_operations.yml index 014bdc8..87be0d5 100644 --- a/.github/workflows/validate_operations.yml +++ b/.github/workflows/validate_operations.yml @@ -4,29 +4,14 @@ env: LLVM_COMMIT: 0c2701fe7fa002e1befc5f86c268a7964f96d286 PROTOBUF_COMMIT: v34.0 CMAKE_VERSION: 4.3.0 + MOLD_LINKER_FLAGS: -fuse-ld=mold on: push: pull_request: jobs: - # Expose env vars as outputs so they can be passed to reusable workflows - config: - runs-on: ubuntu-latest - outputs: - llvm-commit: ${{ steps.set-vars.outputs.llvm-commit }} - steps: - - id: set-vars - run: echo "llvm-commit=$LLVM_COMMIT" >> "$GITHUB_OUTPUT" - - build-mlir-cache: - needs: config - uses: ./.github/workflows/build_mlir_cache.yml - with: - llvm-commit: ${{ needs.config.outputs.llvm-commit }} - - validate: - needs: build-mlir-cache + validate-operations: runs-on: ubuntu-latest steps: @@ -39,7 +24,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y ninja-build build-essential curl ca-certificates + sudo apt install -y cmake ninja-build build-essential mold curl ca-certificates - name: Install CMake run: | @@ -58,27 +43,17 @@ jobs: cmake --version which cmake - - name: Cache protobuf build - id: cache-protobuf - uses: actions/cache@v4 + - name: Prepare MLIR cache + uses: ./.github/actions/prepare-mlir-cache with: - path: | - /usr/local/lib/libproto* - /usr/local/include/google/protobuf - key: protobuf-${{ runner.os }}-${{ env.PROTOBUF_COMMIT }} + llvm-commit: ${{ env.LLVM_COMMIT }} + mold-linker-flags: ${{ env.MOLD_LINKER_FLAGS }} - - name: Install protobuf - if: steps.cache-protobuf.outputs.cache-hit != 'true' - run: | - git clone --depth 1 --branch ${{ env.PROTOBUF_COMMIT }} https://github.com/protocolbuffers/protobuf - cd protobuf - mkdir build - cd build - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release - ninja - sudo ninja install - cd ../.. - rm -rf protobuf + - name: Prepare protobuf cache + uses: ./.github/actions/prepare-protobuf-cache + with: + protobuf-commit: ${{ env.PROTOBUF_COMMIT }} + mold-linker-flags: ${{ env.MOLD_LINKER_FLAGS }} - name: Register installed libraries run: sudo ldconfig @@ -94,23 +69,34 @@ jobs: - name: Install Python dependencies run: pip install numpy onnx colorama - - name: Restore MLIR cache - uses: actions/cache/restore@v4 + - name: Restore raptor build cache + id: restore-raptor-build-cache + uses: ./.github/actions/restore-raptor-build-cache with: - path: onnx-mlir/llvm-project - key: mlir-${{ runner.os }}-${{ env.LLVM_COMMIT }} - fail-on-cache-miss: true + key: raptor-build-${{ runner.os }}-${{ github.ref_name }}-${{ env.LLVM_COMMIT }}-${{ env.PROTOBUF_COMMIT }}-${{ env.CMAKE_VERSION }}-${{ github.sha }} + restore-keys: | + raptor-build-${{ runner.os }}-${{ github.ref_name }}-${{ env.LLVM_COMMIT }}-${{ env.PROTOBUF_COMMIT }}-${{ env.CMAKE_VERSION }}- + raptor-build-${{ runner.os }}-${{ env.LLVM_COMMIT }}-${{ env.PROTOBUF_COMMIT }}-${{ env.CMAKE_VERSION }}- - name: Build Raptor + id: build-raptor run: | MLIR_DIR=$(pwd)/onnx-mlir/llvm-project/build/lib/cmake/mlir - mkdir -p build && cd build - cmake .. -G Ninja \ + cmake -S . -B build -G Ninja \ -DCMAKE_BUILD_TYPE=Debug \ -DONNX_MLIR_ACCELERATORS=PIM \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DMLIR_DIR=${MLIR_DIR} - cmake --build . + -DMLIR_DIR=${MLIR_DIR} \ + -DCMAKE_EXE_LINKER_FLAGS="${MOLD_LINKER_FLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${MOLD_LINKER_FLAGS}" \ + -DCMAKE_MODULE_LINKER_FLAGS="${MOLD_LINKER_FLAGS}" + cmake --build build + + - name: Save raptor build cache + if: steps.build-raptor.outcome == 'success' && steps.restore-raptor-build-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/save-raptor-build-cache + with: + key: raptor-build-${{ runner.os }}-${{ github.ref_name }}-${{ env.LLVM_COMMIT }}-${{ env.PROTOBUF_COMMIT }}-${{ env.CMAKE_VERSION }}-${{ github.sha }} - name: Run validation run: |