Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 69 additions & 12 deletions cmd/mapt/cmd/params/params.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ const (
TagsDesc string = "tags to add on each resource (--tags name1=value1,name2=value2)"
GHActionsRunnerTokenDesc string = "Token needed for registering the Github Actions Runner token"
GHActionsRunnerRepoDesc string = "Full URL of the repository where the Github Actions Runner should be registered"
GHActionsRunnerOrgDesc string = "GitHub organization name where the runner should be registered (mutually exclusive with --ghactions-runner-repo)"
GHActionsRunnerLabelsDesc string = "List of labels separated by comma to be added to the self-hosted runner"
GHActionsRunnerEphemeralDesc string = "Configure the runner as ephemeral: it executes a single job then deregisters automatically"

// Compute request
memory string = "memory"
Expand All @@ -75,9 +77,18 @@ const (
CreateCmdName string = "create"
DestroyCmdName string = "destroy"

ghActionsRunnerToken string = "ghactions-runner-token"
ghActionsRunnerRepo string = "ghactions-runner-repo"
ghActionsRunnerLabels string = "ghactions-runner-labels"
ghActionsRunnerToken string = "ghactions-runner-token"
ghActionsRunnerRepo string = "ghactions-runner-repo"
ghActionsRunnerOrg string = "ghactions-runner-org"
ghActionsRunnerLabels string = "ghactions-runner-labels"
ghActionsRunnerEphemeral string = "ghactions-runner-ephemeral"

ghActionsAppID string = "ghactions-app-id"
ghActionsAppIDDesc string = "GitHub App ID used to authenticate and generate a runner registration token"
ghActionsAppInstallationID string = "ghactions-app-installation-id"
ghActionsAppInstallationIDDesc string = "GitHub App installation ID for the target organization or repository"
ghActionsAppPrivateKey string = "ghactions-app-private-key"
ghActionsAppPrivateKeyDesc string = "Path to the GitHub App RSA private key PEM file"

cirrusPWToken string = "it-cirrus-pw-token"
cirrusPWTokenDesc string = "Add mapt target as a cirrus persistent worker. The value will hold a valid token to be used by cirrus cli to join the project."
Expand Down Expand Up @@ -293,27 +304,71 @@ func AddDebugFlags(fs *pflag.FlagSet) {
func AddGHActionsFlags(fs *pflag.FlagSet) {
fs.StringP(ghActionsRunnerToken, "", "", GHActionsRunnerTokenDesc)
fs.StringP(ghActionsRunnerRepo, "", "", GHActionsRunnerRepoDesc)
fs.StringP(ghActionsRunnerOrg, "", "", GHActionsRunnerOrgDesc)
fs.StringSlice(ghActionsRunnerLabels, nil, GHActionsRunnerLabelsDesc)
fs.StringP(ghActionsAppID, "", "", ghActionsAppIDDesc)
fs.StringP(ghActionsAppInstallationID, "", "", ghActionsAppInstallationIDDesc)
fs.StringP(ghActionsAppPrivateKey, "", "", ghActionsAppPrivateKeyDesc)
fs.Bool(ghActionsRunnerEphemeral, false, GHActionsRunnerEphemeralDesc)
}

func GithubRunnerArgs(arch *github.Arch) *github.GithubRunnerArgs {
token := viper.GetString(ghActionsRunnerToken)
repoURL := viper.GetString(ghActionsRunnerRepo)
org := viper.GetString(ghActionsRunnerOrg)
appID := viper.GetString(ghActionsAppID)
installationID := viper.GetString(ghActionsAppInstallationID)
privateKeyPath := viper.GetString(ghActionsAppPrivateKey)
ephemeral := viper.GetBool(ghActionsRunnerEphemeral)
pat := os.Getenv("GITHUB_TOKEN")

if token == "" && pat == "" {
if token == "" && appID == "" && pat == "" {
return nil
}

if repoURL != "" && org != "" {
logging.Error("--ghactions-runner-repo and --ghactions-runner-org are mutually exclusive")
return nil
}

if repoURL == "" {
logging.Error("--ghactions-runner-repo is required for GitHub Actions runner setup")
if repoURL == "" && org == "" {
logging.Error("one of --ghactions-runner-repo or --ghactions-runner-org is required for GitHub Actions runner setup")
return nil
}

// App auth: validate required companion flags; token is fetched later inside
// the Pulumi context via github.SetupRunner.
if appID != "" {
if installationID == "" {
logging.Error("--ghactions-app-installation-id is required when --ghactions-app-id is set")
return nil
}
if privateKeyPath == "" {
logging.Error("--ghactions-app-private-key is required when --ghactions-app-id is set")
return nil
}
return &github.GithubRunnerArgs{
RepoURL: repoURL,
Org: org,
Labels: viper.GetStringSlice(ghActionsRunnerLabels),
Platform: &github.Linux,
Arch: arch,
AppID: appID,
InstallationID: installationID,
PrivateKeyPath: privateKeyPath,
Ephemeral: ephemeral,
}
}

// PAT path: auto-generate registration token before Pulumi runs.
if token == "" {
logging.Info("no --ghactions-runner-token provided, auto-generating from GITHUB_TOKEN")
var err error
token, err = github.GenerateRegistrationToken(pat, repoURL)
if org != "" {
token, err = github.GenerateOrgRegistrationToken(pat, org)
} else {
token, err = github.GenerateRegistrationToken(pat, repoURL)
}
if err != nil {
logging.Errorf("failed to auto-generate runner registration token: %v", err)
return nil
Expand All @@ -322,11 +377,13 @@ func GithubRunnerArgs(arch *github.Arch) *github.GithubRunnerArgs {
}

return &github.GithubRunnerArgs{
Token: token,
RepoURL: repoURL,
Labels: viper.GetStringSlice(ghActionsRunnerLabels),
Platform: &github.Linux,
Arch: arch,
Token: token,
RepoURL: repoURL,
Org: org,
Labels: viper.GetStringSlice(ghActionsRunnerLabels),
Platform: &github.Linux,
Arch: arch,
Ephemeral: ephemeral,
}
}

Expand Down
100 changes: 63 additions & 37 deletions docs/self-hosted-runner.md
Original file line number Diff line number Diff line change
@@ -1,59 +1,85 @@
# Overview

This feature of `mapt` allows to setup hosts deployed by it as a GitHub Self Hosted Runner, which can then be directly used for running GitHub actions jobs.
This feature of `mapt` allows to setup hosts deployed by it as a GitHub Self Hosted Runner, which can then be directly used for running GitHub Actions jobs.
It benefits from all the existing features that `mapt` already provides, allowing to create self-hosted runners that can be used for different QE scenarios.

## Providers and Platforms

Currently, it allows to create self-hosted runners on AWS (Windows Server, RHEL) and Azure (Windows Desktop)
Currently, it allows to create self-hosted runners on:

### Prerequisite
* AWS: Windows Server, RHEL, Fedora, macOS
* Azure: Windows Desktop, RHEL
* IBM Cloud: IBM Power (ppc64le), IBM Z (s390x)

To register a Self Hosted Runner for a repository or a GitHub organization, the runner program needs a registration token, which can be obtained by requesting the
GitHub API.
## Authentication

* [Information for requesting a token to register a runner for an Organization](https://docs.github.com/en/rest/actions/self-hosted-runners#create-a-registration-token-for-an-organization)
* [Information for requesting a token to register a runner for a repository](https://docs.github.com/en/rest/actions/self-hosted-runners#create-a-registration-token-for-a-repository)
Registering a Self Hosted Runner requires a short-lived registration token from GitHub. `mapt` supports three ways to supply it. Precedence order: GitHub App credentials → explicit registration token → GITHUB_TOKEN (PAT auto-generate).

After obtaining the token we can invoke `mapt` with it to deploy a VM as a Self hosted runner.
### Option A — GitHub App (recommended)

For example to add a runner to this repository, we can use the following `curl` command to request a token:
Using a GitHub App avoids long-lived credentials. `mapt` exchanges the App's private key for a short-lived installation access token inside the Pulumi deployment, then uses it to fetch a fresh runner registration token automatically.

```
% curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer <github_personal_access_token>" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/redhat-developer/mapt/actions/runners/registration-token
```
The Response from this `POST` request will be:
**Prerequisites:**

```
{
"token": "ACDZL3QXEIC73UXBDGSEYEI",
"expires_at": "2024-07-12T19:01:48.478+05:30"
}
```
1. Create a GitHub App with the `administration:write` permission on the target repository.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For org-level runners (--ghactions-runner-org), the App needs the organization_self_hosted_runners:write permission instead of repo-level administration:write. Since the flags table and code already support the org path, consider something like:

Suggested change
1. Create a GitHub App with the `administration:write` permission on the target repository.
1. Create a GitHub App with the `administration:write` permission (for repository runners) or `organization_self_hosted_runners:write` permission (for organization runners).

2. Install the App on the target repository and note the **Installation ID**.
3. Generate and download the App's **private key** (`.pem` file).

### Operations
**Flags:**

After getting the required token, we need to also decide what we are going to call this runner, the desired name can be passed to the `mapt` command using the
`--ghactions-runner-name` flag.
| Flag | Description |
|------|-------------|
| `--ghactions-app-id` | GitHub App ID (numeric, shown on the App settings page) |
| `--ghactions-app-installation-id` | Installation ID for the target org or repository |
| `--ghactions-app-private-key` | Path to the App RSA private key PEM file |
| `--ghactions-runner-repo` | Full URL of the repository (mutually exclusive with `--ghactions-runner-org`) |
| `--ghactions-runner-org` | GitHub organization name for an org-level runner (mutually exclusive with `--ghactions-runner-repo`) |
| `--ghactions-runner-labels` | Comma-separated labels to attach to the runner (optional) |

The full URL of the repository or the GitHub organization also needs to be passed using the `--ghactions-runner-repo` flag.
**Example:**

```bash
mapt aws rhel create \
--ghactions-runner-repo "https://github.com/redhat-developer/mapt" \
--ghactions-app-id "123456" \
--ghactions-app-installation-id "789012" \
--ghactions-app-private-key "/path/to/app-private-key.pem" \
--project-name mapt-rhel-aws \
--backed-url file:///workspace/state \
--conn-details-output /workspace/conn-details
```

To deploy a Windows runner on the Azure provider, we can use the following command:
### Option B — Personal Access Token (PAT)

Set the `GITHUB_TOKEN` environment variable to a PAT with `repo` admin scope. `mapt` will call the GitHub API to generate a registration token automatically before deployment.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only covers the repo case. For org-level runners (--ghactions-runner-org), the PAT needs admin:org scope instead of repo admin. The error message in api.go:58 already tells the user this on failure, but it'd be better to document it upfront.

Suggested change
Set the `GITHUB_TOKEN` environment variable to a PAT with `repo` admin scope. `mapt` will call the GitHub API to generate a registration token automatically before deployment.
Set the `GITHUB_TOKEN` environment variable to a PAT with `repo` scope (for repository runners) or `admin:org` scope (for organization runners). `mapt` will call the GitHub API to generate a registration token automatically before deployment.


```bash
export GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx
mapt aws rhel create \
--ghactions-runner-repo "https://github.com/redhat-developer/mapt" \
--project-name mapt-rhel-aws \
--backed-url file:///workspace/state \
--conn-details-output /workspace/conn-details
```
% mapt azure windows create --spot \
--install-ghactions-runner \
--ghcations-runner-token="ACDZL3QXEIC73UXBDGSEYEI" \
--ghcations-runner-name "az-win-11" \
--ghcations-runner-repo "https://github.com/redhat-developer/mapt" \

### Option C — Pre-generated registration token

Pass a registration token obtained manually from the GitHub API directly via `--ghactions-runner-token`. Tokens expire after 1 hour.

```bash
# Obtain a token via the GitHub API:
curl -L -X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer <pat>" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/redhat-developer/mapt/actions/runners/registration-token

mapt azure windows create --spot \
--ghactions-runner-repo "https://github.com/redhat-developer/mapt" \
--ghactions-runner-token "ACDZL3QXEIC73UXBDGSEYEI" \
--project-name mapt-windows-azure \
--backed-url file:///Users/tester/workspace \
--conn-details-output /Users/tester/workspace/conn-details
--backed-url file:///workspace/state \
--conn-details-output /workspace/conn-details
```
> *NOTE:* additional _labels_ can be added to the runner using the flag `--ghactions-runner-labels`, e.g `--ghactions-runner-lables="azure,mapt,windows"`

> **Note:** Additional labels can be added to the runner with `--ghactions-runner-labels`, e.g. `--ghactions-runner-labels="azure,mapt,windows"`.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ require (
github.com/pulumi/pulumi-azure-native-sdk/network/v3 v3.20.0
github.com/pulumi/pulumi-azure-native-sdk/resources/v3 v3.20.0
github.com/pulumi/pulumi-azure-native-sdk/storage/v3 v3.20.0
github.com/pulumi/pulumi-github/sdk/v6 v6.14.0
github.com/pulumi/pulumi-gitlab/sdk/v9 v9.11.1
github.com/pulumi/pulumi-tls/sdk/v5 v5.5.0
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,8 @@ github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild v0.0.20 h1:8VJt/iUb1ur3
github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild v0.0.20/go.mod h1:iRiNhXmySLlREgx1zeoD4Es62WU2jTpd5YXdZ/0AQB4=
github.com/pulumi/pulumi-docker/sdk/v4 v4.5.8 h1:rik9L2SIpsoDenY51MkogR6GWgu/0Sy/XmyQmKWNUqU=
github.com/pulumi/pulumi-docker/sdk/v4 v4.5.8/go.mod h1:eph7BPNPkEIIK882/Ll4dbeHl5wZEc/UvTcUW0CK1UY=
github.com/pulumi/pulumi-github/sdk/v6 v6.14.0 h1:49TS7PctMDGQpOAD6vu0On+k6L3t0Rtrzmwd/fDcbVk=
github.com/pulumi/pulumi-github/sdk/v6 v6.14.0/go.mod h1:aSCgSWErJwJujq1CKHe71wArYAKjWKHs+REB5GcM0es=
github.com/pulumi/pulumi-gitlab/sdk/v9 v9.11.1 h1:DPDQD9FamQ3myy0EQOYuzUFvcJpQDr1H3GNwz/scDgA=
github.com/pulumi/pulumi-gitlab/sdk/v9 v9.11.1/go.mod h1:eS9gkxGvdzprE767heyMrPrMY/6FX9UbdPYeI4pbkE8=
github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.33.0 h1:fFP72Q6/GfyI7BII3hUS7vn45Eg16zKIIS3I3R9bNEY=
Expand Down
3 changes: 3 additions & 0 deletions oci/Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ ARG PULUMI_TLS_VERSION=v5.5.0
ARG PULUMI_RANDOM_VERSION=v4.21.0
# renovate: datasource=github-releases depName=pulumi/pulumi-aws-native
ARG PULUMI_AWS_NATIVE_VERSION=v1.71.0
# renovate: datasource=github-releases depName=pulumi/pulumi-github
ARG PULUMI_GITHUB_VERSION=v6.14.0
# renovate: datasource=github-releases depName=pulumi/pulumi-gitlab
ARG PULUMI_GITLAB_VERSION=v10.0.0
# renovate: datasource=github-releases depName=mapt-oss/pulumi-ibmcloud
Expand All @@ -53,6 +55,7 @@ RUN mkdir -p ${PULUMI_HOME} \
&& pulumi plugin install resource random ${PULUMI_RANDOM_VERSION} \
&& pulumi plugin install resource awsx ${PULUMI_AWSX_VERSION} \
&& pulumi plugin install resource aws-native ${PULUMI_AWS_NATIVE_VERSION} \
&& pulumi plugin install resource github ${PULUMI_GITHUB_VERSION} \
&& pulumi plugin install resource gitlab ${PULUMI_GITLAB_VERSION}

# Stage 2: Red Hat Hardened minimal runtime (glibc + coreutils, no toolchain)
Expand Down
69 changes: 60 additions & 9 deletions pkg/integrations/github/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,73 @@ type registrationTokenResponse struct {
ExpiresAt string `json:"expires_at"`
}

// GenerateRegistrationToken calls the GitHub API to create a short-lived
// runner registration token for the given repository.
// pat is a Personal Access Token with repo admin scope.
// repoURL is in the form "owner/repo" or "https://github.com/owner/repo".
func GenerateRegistrationToken(pat, repoURL string) (string, error) {
ownerRepo := repoURL
ownerRepo = strings.TrimPrefix(ownerRepo, "https://github.com/")
// splitOwnerRepo parses a repo URL in the form "owner/repo" or
// "https://github.com/owner/repo" and returns the two components.
func splitOwnerRepo(repoURL string) (owner, repo string, err error) {
ownerRepo := strings.TrimPrefix(repoURL, "https://github.com/")
ownerRepo = strings.TrimPrefix(ownerRepo, "http://github.com/")
ownerRepo = strings.TrimSuffix(ownerRepo, "/")

parts := strings.Split(ownerRepo, "/")
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", fmt.Errorf("invalid repo format %q, expected owner/repo", repoURL)
return "", "", fmt.Errorf("invalid repo format %q, expected owner/repo", repoURL)
}
return parts[0], parts[1], nil
}

// GenerateOrgRegistrationToken calls the GitHub API to create a short-lived
// runner registration token for the given organization.
// pat is a Personal Access Token with admin:org scope.
func GenerateOrgRegistrationToken(pat, org string) (string, error) {
url := fmt.Sprintf("https://api.github.com/orgs/%s/actions/runners/registration-token", org)

req, err := http.NewRequest(http.MethodPost, url, nil)
if err != nil {
return "", fmt.Errorf("creating request: %w", err)
}
req.Header.Set("Authorization", "token "+pat)
req.Header.Set("Accept", "application/vnd.github+json")
req.Header.Set("X-GitHub-Api-Version", "2022-11-28")

client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("calling GitHub API: %w", err)
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("reading response: %w", err)
}

if resp.StatusCode != http.StatusCreated {
return "", fmt.Errorf("GitHub API returned %d: %s (ensure GITHUB_TOKEN has admin:org scope)", resp.StatusCode, string(body))
}

var tokenResp registrationTokenResponse
if err := json.Unmarshal(body, &tokenResp); err != nil {
return "", fmt.Errorf("parsing response: %w", err)
}

if tokenResp.Token == "" {
return "", fmt.Errorf("empty token in GitHub API response")
}

return tokenResp.Token, nil
}

// GenerateRegistrationToken calls the GitHub API to create a short-lived
// runner registration token for the given repository.
// pat is a Personal Access Token with repo admin scope.
// repoURL is in the form "owner/repo" or "https://github.com/owner/repo".
func GenerateRegistrationToken(pat, repoURL string) (string, error) {
owner, repo, err := splitOwnerRepo(repoURL)
if err != nil {
return "", err
}

url := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/runners/registration-token", parts[0], parts[1])
url := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/runners/registration-token", owner, repo)

req, err := http.NewRequest(http.MethodPost, url, nil)
if err != nil {
Expand Down
Loading