From ca639466e771d2e08dbbba7987d804e97a5953d3 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Wed, 17 Jun 2026 20:51:44 +0000 Subject: [PATCH] Run CI on self-hosted runners Mirror QuantConnect/Lean#9540 (commit c57fd18b): run the build/test workflows on self-hosted runners directly inside a job-level quantconnect/lean:foundation container (--cpus 12 --memory 12g) instead of the GitHub-hosted ubuntu runner + manual 'docker run --name test-container' + runInContainer helper. - build.yml and rename-build.yml: runs-on self-hosted + job container; drop the Start/Stop container steps; strip the runInContainer prefix from every command (they now run directly in the container); rename-build cds into the renamed copy per step since the job container's workdir is the repo root. build.yml gets a concurrency group (cancel-in-progress). - actions/setup composite: drop the now-unnecessary 'Liberate disk space', 'Pull foundation image' and 'Define docker helper' steps; keep the Lean branch-check/checkout/move. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/actions/setup/action.yml | 20 +-------------- .github/workflows/build.yml | 39 +++++++++++++----------------- .github/workflows/rename-build.yml | 20 ++++++--------- 3 files changed, 25 insertions(+), 54 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 92d4be6..e158a8a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,17 +1,9 @@ name: Setup -description: Liberate disk space, checkout Lean, pull foundation image, define docker helper +description: Checkout Lean runs: using: composite steps: - - name: Liberate disk space - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - large-packages: false - docker-images: false - swap-storage: false - - name: Check if Lean has same branch id: lean-branch-check run: | @@ -42,13 +34,3 @@ runs: - name: Move Lean run: mv Lean ../Lean shell: bash - - - name: Pull foundation image - run: docker pull quantconnect/lean:foundation - shell: bash - - - name: Define docker helper - run: | - echo 'runInContainer() { docker exec test-container "$@"; }' > $HOME/ci_functions.sh - echo "BASH_ENV=$HOME/ci_functions.sh" >> $GITHUB_ENV - shell: bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3599f13..4f840f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,9 +6,16 @@ on: pull_request: branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: - runs-on: ubuntu-24.04 + runs-on: self-hosted + container: + image: quantconnect/lean:foundation + options: --cpus 12 --memory 12g steps: - name: Checkout uses: actions/checkout@v5 @@ -19,50 +26,38 @@ jobs: - name: Copy sample data to LEAN Data folder run: cp -r ../Lean.DataSource.SDK/output/* ../Lean/Data/ - - name: Start container - run: | - docker run -d \ - --workdir /__w/Lean.DataSource.SDK/Lean.DataSource.SDK \ - -v /home/runner/work:/__w \ - --name test-container \ - quantconnect/lean:foundation \ - tail -f /dev/null - - name: Build DataSource - run: runInContainer dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Build Tests - run: runInContainer dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Run Tests - run: runInContainer dotnet test ./tests/bin/Release/net10.0/Tests.dll + run: dotnet test ./tests/bin/Release/net10.0/Tests.dll - name: Build LEAN Launcher - run: runInContainer dotnet build ../Lean/Launcher/QuantConnect.Lean.Launcher.csproj /p:Configuration=Release /p:OutputPath=bin/Release /v:quiet /p:WarningLevel=1 + run: dotnet build ../Lean/Launcher/QuantConnect.Lean.Launcher.csproj /p:Configuration=Release /p:OutputPath=bin/Release /v:quiet /p:WarningLevel=1 - name: Copy DataSource to LEAN Launcher - run: runInContainer cp ./tests/bin/Release/net10.0/QuantConnect.DataSource.MyCustomDataType.dll ../Lean/Launcher/bin/Release/ + run: cp ./tests/bin/Release/net10.0/QuantConnect.DataSource.MyCustomDataType.dll ../Lean/Launcher/bin/Release/ - name: Run C# Demo Algorithm run: | - output=$(runInContainer bash -c "cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language CSharp --algorithm-type-name CustomDataAlgorithm --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/tests/bin/Release/net10.0/Tests.dll --close-automatically true" 2>&1) + output=$(bash -c "cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language CSharp --algorithm-type-name CustomDataAlgorithm --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/tests/bin/Release/net10.0/Tests.dll --close-automatically true" 2>&1) echo "$output" echo "$output" | grep -q "STATISTICS:: Total Orders [^0]" || { echo "Algorithm placed no trades"; exit 1; } - name: Run Python Demo Algorithm run: | - output=$(runInContainer bash -c 'cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language Python --algorithm-type-name CustomDataAlgorithm --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/Demonstration.py --close-automatically true' 2>&1) || true + output=$(bash -c 'cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language Python --algorithm-type-name CustomDataAlgorithm --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/Demonstration.py --close-automatically true' 2>&1) || true echo "$output" echo "$output" | grep -q "STATISTICS:: Total Orders [^0]" || { echo "Algorithm placed no trades"; exit 1; } - name: Run C# Universe Demo Algorithm - run: runInContainer bash -c "cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language CSharp --algorithm-type-name CustomDataUniverse --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/tests/bin/Release/net10.0/Tests.dll --close-automatically true" + run: bash -c "cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language CSharp --algorithm-type-name CustomDataUniverse --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/tests/bin/Release/net10.0/Tests.dll --close-automatically true" - name: Run Python Universe Demo Algorithm - run: runInContainer bash -c 'cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language Python --algorithm-type-name CustomDataUniverse --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/DemonstrationUniverse.py --close-automatically true || [ $? -eq 134 ]' - - - name: Stop container - run: docker rm -f test-container + run: bash -c 'cd ../Lean/Launcher/bin/Release && dotnet ./QuantConnect.Lean.Launcher.dll --data-folder /__w/Lean.DataSource.SDK/Lean/Data/ --algorithm-language Python --algorithm-type-name CustomDataUniverse --algorithm-location /__w/Lean.DataSource.SDK/Lean.DataSource.SDK/DemonstrationUniverse.py --close-automatically true || [ $? -eq 134 ]' rename: needs: build diff --git a/.github/workflows/rename-build.yml b/.github/workflows/rename-build.yml index cf72370..41d324b 100644 --- a/.github/workflows/rename-build.yml +++ b/.github/workflows/rename-build.yml @@ -6,7 +6,10 @@ on: jobs: rename-build: if: github.repository == 'QuantConnect/Lean.DataSource.SDK' - runs-on: ubuntu-24.04 + runs-on: self-hosted + container: + image: quantconnect/lean:foundation + options: --cpus 12 --memory 12g steps: - name: Checkout uses: actions/checkout@v5 @@ -20,20 +23,11 @@ jobs: cd ../Lean.DataSource.Rename python renameDataset.py Rename Dataset - - name: Start container (renamed) - run: | - docker run -d \ - --workdir /__w/Lean.DataSource.SDK/Lean.DataSource.Rename \ - -v /home/runner/work:/__w \ - --name test-container \ - quantconnect/lean:foundation \ - tail -f /dev/null - - name: Build Renamed DataSource - run: runInContainer dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + run: cd ../Lean.DataSource.Rename && dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Build Renamed Tests - run: runInContainer dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + run: cd ../Lean.DataSource.Rename && dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Run Renamed Tests - run: runInContainer dotnet test ./tests/bin/Release/net10.0/Tests.dll + run: cd ../Lean.DataSource.Rename && dotnet test ./tests/bin/Release/net10.0/Tests.dll