From 15513b0ada3ebc26a3e72414e4eb8078dea0ea2e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 1 Mar 2023 15:50:31 +0000 Subject: [PATCH] Clean up the way we build `xtask` in CI (#1600) Currently, the cache of the xtask binary isn't working terribly well: * we seem to build it on each run anyway, presumably because we don't cache any of the intermediate build artifacts. Running the binary directly rather than indirecting via "cargo" prevents this. * There is no sharing of the cache between the "rust" and "bindings" CI, because we use different cache keys. This PR addresses both problems, and hopefully speeds up CI a bit as a result. --- .github/workflows/bindings_ci.yml | 71 +++++--------------------- .github/workflows/ci.yml | 72 +++++++++++---------------- .github/workflows/xtask.yml | 82 +++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 104 deletions(-) create mode 100644 .github/workflows/xtask.yml diff --git a/.github/workflows/bindings_ci.yml b/.github/workflows/bindings_ci.yml index c468b6755..2d4113f2d 100644 --- a/.github/workflows/bindings_ci.yml +++ b/.github/workflows/bindings_ci.yml @@ -18,36 +18,12 @@ env: MATRIX_SDK_CRYPTO_JS_PATH: bindings/matrix-sdk-crypto-js jobs: - xtask-linux: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check xtask cache - uses: actions/cache@v3 - id: xtask-cache - with: - path: target/debug/xtask - key: xtask-linux-${{ hashFiles('Cargo.toml', 'xtask/**') }} - - - name: Install rust stable toolchain - if: steps.xtask-cache.outputs.cache-hit != 'true' - uses: dtolnay/rust-toolchain@stable - - - name: Build - if: steps.xtask-cache.outputs.cache-hit != 'true' - run: | - cargo build -p xtask + xtask: + uses: ./.github/workflows/xtask.yml test-uniffi-codegen: name: Test UniFFI bindings generation - needs: xtask-linux + needs: xtask if: github.event_name == 'push' || !github.event.pull_request.draft runs-on: ubuntu-latest @@ -67,10 +43,11 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-linux-${{ hashFiles('Cargo.toml', 'xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Build library & generate bindings run: target/debug/xtask ci bindings @@ -211,36 +188,9 @@ jobs: working-directory: ${{ env.MATRIX_SDK_CRYPTO_JS_PATH }} run: npm run doc - xtask-macos: - runs-on: macos-12 - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check xtask cache - uses: actions/cache@v3 - id: xtask-cache - with: - path: target/debug/xtask - key: xtask-macos-${{ hashFiles('Cargo.toml', 'xtask/**') }} - - - name: Install rust stable toolchain - if: steps.xtask-cache.outputs.cache-hit != 'true' - uses: dtolnay/rust-toolchain@stable - - - name: Build - if: steps.xtask-cache.outputs.cache-hit != 'true' - run: | - cargo build -p xtask - test-apple: name: matrix-rust-components-swift - needs: xtask-macos + needs: xtask runs-on: macos-12 if: github.event_name == 'push' || !github.event.pull_request.draft @@ -263,10 +213,11 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-macos-${{ hashFiles('Cargo.toml', 'xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-macos }}" + fail-on-cache-miss: true - name: Build library & bindings run: target/debug/xtask swift build-library @@ -276,4 +227,4 @@ jobs: run: swift test - name: Build Framework - run: cargo xtask swift build-framework --only-target=aarch64-apple-ios + run: target/debug/xtask swift build-framework --only-target=aarch64-apple-ios diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06db21d8d..dd6f282ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,31 +17,7 @@ env: jobs: xtask: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check xtask cache - uses: actions/cache@v3 - id: xtask-cache - with: - path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} - - - name: Install rust stable toolchain - if: steps.xtask-cache.outputs.cache-hit != 'true' - uses: dtolnay/rust-toolchain@stable - - - name: Build - if: steps.xtask-cache.outputs.cache-hit != 'true' - run: | - cargo build -p xtask + uses: ./.github/workflows/xtask.yml test-matrix-sdk-features: name: 🐧 [m], ${{ matrix.name }} @@ -76,14 +52,15 @@ jobs: uses: taiki-e/install-action@nextest - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Test run: | - cargo run -p xtask -- ci test-features ${{ matrix.name }} + target/debug/xtask ci test-features ${{ matrix.name }} test-matrix-sdk-examples: name: 🐧 [m]-examples @@ -105,14 +82,15 @@ jobs: uses: taiki-e/install-action@nextest - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Test run: | - cargo run -p xtask -- ci examples + target/debug/xtask ci examples test-matrix-sdk-crypto: name: 🐧 [m]-crypto @@ -134,14 +112,15 @@ jobs: uses: taiki-e/install-action@nextest - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Test run: | - cargo run -p xtask -- ci test-crypto + target/debug/xtask ci test-crypto test-all-crates: name: ${{ matrix.name }} @@ -261,18 +240,19 @@ jobs: uses: taiki-e/install-action@nextest - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Rust Check run: | - cargo run -p xtask -- ci wasm ${{ matrix.cmd }} + target/debug/xtask ci wasm ${{ matrix.cmd }} - name: Wasm-Pack test run: | - cargo run -p xtask -- ci wasm-pack ${{ matrix.cmd }} + target/debug/xtask ci wasm-pack ${{ matrix.cmd }} test-appservice: name: ${{ matrix.os-name }} [m]-appservice @@ -286,9 +266,11 @@ jobs: include: - os: ubuntu-latest os-name: 🐧 + xtask-cachekey: "${{ needs.xtask.outputs.cachekey-linux }}" - os: macos-latest os-name: 🍏 + xtask-cachekey: "${{ needs.xtask.outputs.cachekey-macos }}" steps: - name: Checkout @@ -304,14 +286,15 @@ jobs: uses: taiki-e/install-action@nextest - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ matrix.xtask-cachekey }}" + fail-on-cache-miss: true - name: Run checks run: | - cargo run -p xtask -- ci test-appservice + target/debug/xtask ci test-appservice formatting: name: Check Formatting @@ -367,14 +350,15 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Get xtask - uses: actions/cache@v3 + uses: actions/cache/restore@v3 with: path: target/debug/xtask - key: xtask-${{ hashFiles('xtask/**') }} + key: "${{ needs.xtask.outputs.cachekey-linux }}" + fail-on-cache-miss: true - name: Clippy run: | - cargo run -p xtask -- ci clippy + target/debug/xtask ci clippy integration-tests: name: Integration test diff --git a/.github/workflows/xtask.yml b/.github/workflows/xtask.yml new file mode 100644 index 000000000..fd9b95566 --- /dev/null +++ b/.github/workflows/xtask.yml @@ -0,0 +1,82 @@ +# A reusable github actions workflow that will build xtask, if it is not +# already cached. +# +# It will create a pair of GHA cache entries, if they do not already exist. +# The cache keys take the form `xtask-{os}-{hash}`, where "{os}" is "linux" +# or "macos", and "{hash}" is the hash of the xtask# directory. +# +# The cache keys are written to output variables named "cachekey-{os}". +# + +name: Build xtask if necessary + +on: + workflow_call: + outputs: + cachekey-linux: + description: "The cache key for the linux build artifact" + value: "${{ jobs.xtask.outputs.cachekey-linux }}" + cachekey-macos: + description: "The cache key for the macos build artifact" + value: "${{ jobs.xtask.outputs.cachekey-macos }}" + +env: + CARGO_TERM_COLOR: always + +jobs: + xtask: + name: "xtask-${{ matrix.os-name }}" + + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + os-name: 🐧 + cachekey-id: linux + + - os: macos-12 + os-name: 🍏 + cachekey-id: macos + + runs-on: "${{ matrix.os }}" + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Calculate cache key + id: cachekey + # set a step output variable "cachekey-{os}" that can be referenced in + # the job outputs below. + run: | + echo "cachekey-${{ matrix.cachekey-id }}=xtask-${{ matrix.cachekey-id }}-${{ hashFiles('Cargo.toml', 'xtask/**') }}" >> $GITHUB_OUTPUT + + - name: Check xtask cache + uses: actions/cache@v3 + id: xtask-cache + with: + path: target/debug/xtask + # use the cache key calculated in the step above. Bit of an awkard + # syntax + key: | + ${{ steps.cachekey.outputs[format('cachekey-{0}', matrix.cachekey-id)] }} + + - name: Install Protoc + if: steps.xtask-cache.outputs.cache-hit != 'true' + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install rust stable toolchain + if: steps.xtask-cache.outputs.cache-hit != 'true' + uses: dtolnay/rust-toolchain@stable + + - name: Build + if: steps.xtask-cache.outputs.cache-hit != 'true' + run: | + cargo build -p xtask + + outputs: + "cachekey-linux": "${{ steps.cachekey.outputs.cachekey-linux }}" + "cachekey-macos": "${{ steps.cachekey.outputs.cachekey-macos }}"