diff --git a/.dockerignore b/.dockerignore index 85f712af..03113260 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,5 +2,17 @@ /create-a-container/certs/* !/create-a-container/certs/.gitignore /create-a-container/.env +/create-a-container/client/dist /mie-opensource-landing/build +/mie-opensource-landing/site +/mie-opensource-landing/.venv +/mie-opensource-landing/.cache **/node_modules +**/__pycache__ + +# packaging build artifacts (rebuilt inside the builder image) +**/.pkg +/dist +**/*.deb +**/*.rpm +**/*.apk diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index abcb07d9..67d29eb8 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -1,25 +1,21 @@ name: Build and Push Images on: + # Build on pushes to the default branch (merges) and on every pull request + # update. Feature-branch pushes are covered by their PR, avoiding duplicate + # builds and registry churn. No paths filter, so doc-only changes still + # rebuild the images. push: branches: - - '**' - tags: - - '**' - paths: - - 'images/**' - - 'mie-opensource-landing/**' - - 'Makefile' + - main pull_request: types: [opened, synchronize, reopened, closed] - paths: - - 'images/**' - - 'mie-opensource-landing/**' - - 'Makefile' schedule: # Run weekly on Sunday at 11:00 PM UTC (Sunday-Monday night depending on timezone) - cron: '0 23 * * 0' workflow_dispatch: + release: + types: [published] env: REGISTRY: ghcr.io @@ -35,6 +31,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: + fetch-depth: 0 persist-credentials: false - name: Set up Docker Buildx @@ -53,12 +50,13 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/base bake-target: base + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=pr type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Docker Meta (NodeJS) id: meta-nodejs @@ -66,12 +64,13 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/nodejs bake-target: nodejs + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=pr type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Docker Meta (Docs) id: meta-docs @@ -79,12 +78,13 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/docs bake-target: docs + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=pr type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Docker Meta (Agent) id: meta-agent @@ -92,11 +92,12 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/agent bake-target: agent + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Docker Meta (Manager) id: meta-manager @@ -104,11 +105,12 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/manager bake-target: manager + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Docker Meta (Proxmox VE) id: meta-proxmox-ve @@ -116,11 +118,12 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}/proxmox-ve bake-target: proxmox-ve + flavor: latest=false tags: | type=sha type=ref,event=branch type=ref,event=tag - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} - name: Build and push uses: docker/bake-action@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..43360594 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release Packages + +# Triggered by publishing a GitHub release (full or prerelease). Builds the +# three Debian packages and uploads them to the release that triggered the +# workflow, together with flat APT repository metadata (Packages, Packages.gz) +# so the release can be used directly as an apt source: +# +# deb [trusted=yes] https://github.com/mieweb/opensource-server/releases/latest/download/ ./ +# +# The release URLs (releases/latest/download/) serve the flat repo; the +# images ship this source so `apt upgrade` tracks future releases. This +# workflow never creates or modifies the release itself — create the release +# (and choose full vs prerelease) in GitHub first, then this attaches assets. + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build-and-release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build packages + uses: docker/bake-action@v5 + with: + workdir: ./images + targets: builder + files: | + ./docker-bake.hcl + set: | + builder.output=type=local,dest=../dist + + - name: Generate flat APT repository metadata + run: | + cd dist/dist + # Scan only the versioned packages so the apt index has no duplicates. + dpkg-scanpackages --multiversion . > Packages + gzip -k9f Packages + # Stable-name aliases for one-off downloads (the _latest URLs resolve + # via releases/latest/download for the newest non-prerelease release). + for pkg in opensource-server opensource-docs opensource-agent; do + f=$(ls ${pkg}_*.deb | head -1) + [ -n "$f" ] && cp -f "$f" "${pkg}_latest.deb" + done + ls -l + + - name: Upload assets to the release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + run: | + gh release upload "$TAG" \ + dist/dist/*.deb \ + dist/dist/Packages \ + dist/dist/Packages.gz \ + --repo "${{ github.repository }}" \ + --clobber diff --git a/.gitignore b/.gitignore index 1176a0f3..d650d1a5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ node_modules .env .tmp-verify/ .playwright-mcp/ + +# packaging build artifacts +/dist/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index 40d4aeb3..430907f9 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,53 @@ -.PHONY: install install-create-container install-pull-config install-docs help +.DEFAULT_GOAL := help + +COMPONENTS := pull-config mie-opensource-landing create-a-container +PACKAGER ?= deb + +# Forwarded to every component Makefile. +MAKE_VARS = $(if $(PREFIX),PREFIX=$(PREFIX),) \ + $(if $(DESTDIR),DESTDIR=$(DESTDIR),) + +.PHONY: help deps build install deb rpm apk clean help: - @echo "opensource-server installation" + @echo "opensource-server — delegates to each component's Makefile." @echo "" - @echo "Available targets:" - @echo " make install - Install all components" - @echo " make install-create-container - Install create-a-container web application" - @echo " make install-pull-config - Install pull-config system" - @echo " make install-docs - Install documentation server" + @echo "Targets (run across all components):" + @echo " deps install build/runtime dependencies" + @echo " build build all components" + @echo " install stage component files into DESTDIR (default /)" + @echo " deb build .deb packages, collected into ./dist" + @echo " rpm build .rpm packages, collected into ./dist" + @echo " apk build .apk packages, collected into ./dist" + @echo " clean remove build artifacts, staging, packages and ./dist" + @echo " help show this message" @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + @echo "The package version is derived from git by ./package-version." -install: install-create-container install-pull-config install-docs - -SYSTEMD_DIR := create-a-container/systemd -SERVICES := $(wildcard $(SYSTEMD_DIR)/*.service) -install-create-container: - cd create-a-container && npm install --omit=dev - cd create-a-container/client && npm install && npm run build - install -m 644 -o root -g root $(SERVICES) /etc/systemd/system/ - systemctl daemon-reload || true - @for service in $(notdir $(SERVICES)); do \ - systemctl enable $$service; \ +deps build install: + @for c in $(COMPONENTS); do \ + echo "==> $$c: $@"; \ + $(MAKE) -C $$c $@ $(MAKE_VARS) || exit $$?; \ done -install-pull-config: - cd pull-config && bash install.sh +# Clean each component (which removes its built packages) and the dist/ +# collection directory. +clean: + @for c in $(COMPONENTS); do \ + echo "==> $$c: clean"; \ + $(MAKE) -C $$c clean $(MAKE_VARS) || exit $$?; \ + done + rm -rf dist -install-docs: - cd mie-opensource-landing && uv run zensical build +# Package every component, then collect the artifacts into ./dist. +deb rpm apk: + @mkdir -p dist + @for c in $(COMPONENTS); do \ + echo "==> $$c: $@"; \ + $(MAKE) -C $$c $@ $(MAKE_VARS) || exit $$?; \ + cp -f $$c/*.$@ dist/; \ + done + @echo "" + @echo "Packages collected in dist/:" + @ls -1 dist/ diff --git a/compose.yml b/compose.yml index d5fe6a70..429538d8 100644 --- a/compose.yml +++ b/compose.yml @@ -37,7 +37,11 @@ services: volumes: - ./:/opt/opensource-server working_dir: /opt/opensource-server/create-a-container - command: npm ci --no-audit --no-fund + entrypoint: ["/bin/sh", "-c"] + command: + - | + npm ci --omit=dev --no-audit --no-fund + exec npm --prefix client ci --no-audit --no-fund # Watches the React client and rebuilds the production bundle on change. The # Manager LXC inside Proxmox serves create-a-container/client/dist statically @@ -49,6 +53,7 @@ services: # # Like zensical, this is a development convenience and not a hard dependency: # it does an initial build so a dist always exists, then watches for changes. + # The server itself runs inside Proxmox, so this only runs the client watcher. client: image: node:24-trixie-slim volumes: diff --git a/create-a-container/.fpm b/create-a-container/.fpm new file mode 100644 index 00000000..baa9b50a --- /dev/null +++ b/create-a-container/.fpm @@ -0,0 +1,20 @@ +--name opensource-server +--architecture amd64 +--license Apache-2.0 +--maintainer "Medical Informatics Engineering " +--vendor "Medical Informatics Engineering" +--url "https://github.com/mieweb/opensource-server" +--category admin +--description "MIE opensource-server cluster manager (create-a-container): web UI and REST API for self-service LXC container hosting on Proxmox VE, with automated DNS and reverse-proxy configuration, LDAP authentication and ACME TLS orchestration." +--depends opensource-agent +--depends opensource-docs +--depends nodejs +--depends sudo +--depends libc6 +--depends libgcc-s1 +--depends libstdc++6 +--deb-suggests postgresql +--config-files /etc/logrotate.d/opensource-server +--deb-no-default-config-files +--after-install contrib/postinstall.sh +--before-remove contrib/preremove.sh diff --git a/create-a-container/.gitignore b/create-a-container/.gitignore index d83eef34..ca7e5ca8 100644 --- a/create-a-container/.gitignore +++ b/create-a-container/.gitignore @@ -1,3 +1,9 @@ .env node_modules data/ + +# packaging build artifacts +/.pkg/ +/*.deb +/*.rpm +/*.apk diff --git a/create-a-container/Makefile b/create-a-container/Makefile new file mode 100644 index 00000000..0d52453b --- /dev/null +++ b/create-a-container/Makefile @@ -0,0 +1,77 @@ +.DEFAULT_GOAL := help + +PREFIX ?= /opt/opensource-server +DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/create-a-container +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system + +# Staging directory for packaging +STAGE := $(CURDIR)/.pkg/buildroot + +INSTALL := install +INSTALL_DATA := $(INSTALL) -m 0644 + +# Server runtime files and directories. Client sources, dev tooling and repo +# metadata are intentionally excluded; only the built client/dist ships. +APP_FILES := server.js job-runner.js package.json package-lock.json openapi.v1.yaml +APP_DIRS := bin config data middlewares migrations models node_modules \ + public routers seeders utils views + +.PHONY: help deps build dev install deb rpm apk package clean + +help: + @echo "create-a-container — builds the opensource-server package." + @echo "" + @echo "Targets:" + @echo " deps install dependencies (npm ci, server + client)" + @echo " build build the web client bundle" + @echo " install stage the app into DESTDIR (default /)" + @echo " dev run server + client watch (npm run dev)" + @echo " deb build the .deb package" + @echo " rpm build the .rpm package" + @echo " apk build the .apk package" + @echo " clean remove the staging dir, build output and built packages" + @echo " help show this message" + @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + +deps: + npm ci --omit=dev + npm --prefix client ci + +build: deps + npm --prefix client run build + +# Full local development: server (nodemon) + client (vite watch) together. +dev: deps + npm run dev + +install: build + $(INSTALL) -d $(DESTBIN) + $(INSTALL_DATA) $(APP_FILES) $(DESTBIN)/ + cp -a $(APP_DIRS) $(DESTBIN)/ + $(INSTALL) -d $(DESTBIN)/client + cp -a client/dist $(DESTBIN)/client/ + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/container-creator.service $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/job-runner.service $(UNIT_DIR)/ + $(INSTALL) -d $(DESTDIR)/etc/logrotate.d + $(INSTALL_DATA) contrib/opensource-server.logrotate $(DESTDIR)/etc/logrotate.d/opensource-server + +PACKAGER ?= deb +package: + rm -rf $(STAGE) + $(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX) + fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \ + -C $(STAGE) -p $(CURDIR) -f + +deb: + $(MAKE) package PACKAGER=deb +rpm: + $(MAKE) package PACKAGER=rpm +apk: + $(MAKE) package PACKAGER=apk + +clean: + rm -rf $(STAGE) node_modules client/node_modules client/dist + rm -f *.deb *.rpm *.apk diff --git a/create-a-container/compose.ldap.env b/create-a-container/compose.ldap.env deleted file mode 100644 index 397bf5d2..00000000 --- a/create-a-container/compose.ldap.env +++ /dev/null @@ -1,10 +0,0 @@ -LOG_LEVEL=debug -DIRECTORY_BACKEND=sql -REQUIRE_AUTH_FOR_SEARCH=false -AUTH_BACKENDS=sql -LDAP_COMMON_NAME=ldap -LDAP_BASE_DN=dc=docker,dc=internal -SQL_QUERY_ALL_USERS='SELECT "uid" AS username, "uidNumber" AS uid_number, "gidNumber" AS gid_number, "cn" AS full_name, "sn" AS last_name, "mail", "homeDirectory" AS home_directory, "userPassword" AS password, "givenName" as first_name FROM "Users"' -SQL_QUERY_ONE_USER='SELECT "uid" AS username, "uidNumber" AS uid_number, "gidNumber" AS gid_number, "cn" AS full_name, "sn" AS last_name, "mail", "homeDirectory" AS home_directory, "userPassword" AS password, "givenName" as first_name FROM "Users" WHERE "uid" = ?' -SQL_QUERY_ALL_GROUPS='SELECT g."cn" AS name, g."gidNumber" AS gid_number FROM "Groups" g' -SQL_QUERY_GROUPS_BY_MEMBER='SELECT g."cn" AS name, g."gidNumber" AS gid_number FROM "Groups" g INNER JOIN "UserGroups" ug ON g."gidNumber" = ug."gidNumber" INNER JOIN "Users" u ON ug."uidNumber" = u."uidNumber" WHERE u."uid" = ?' diff --git a/images/manager/opensource-server.logrotate b/create-a-container/contrib/opensource-server.logrotate similarity index 100% rename from images/manager/opensource-server.logrotate rename to create-a-container/contrib/opensource-server.logrotate diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh new file mode 100755 index 00000000..e92c7acf --- /dev/null +++ b/create-a-container/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +UNITS="container-creator.service job-runner.service" + +# Nothing to do without systemctl (non-systemd container/chroot). +command -v systemctl >/dev/null 2>&1 || exit 0 + +# `systemctl enable` only creates static symlinks, so it works during an image +# build too +systemctl enable $UNITS + +# daemon-reload and restart need a running systemd; skip them at build time. +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + systemctl restart $UNITS +fi + +exit 0 diff --git a/create-a-container/contrib/preremove.sh b/create-a-container/contrib/preremove.sh new file mode 100755 index 00000000..a3ebf879 --- /dev/null +++ b/create-a-container/contrib/preremove.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +UNITS="container-creator.service job-runner.service" + +# $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final +# removal). Only act on a real removal, not an upgrade. +case "${1:-}" in + upgrade|1) + # rpm upgrade ("1") / deb upgrade: leave units in place. + exit 0 + ;; +esac + +# Nothing to do without systemctl +command -v systemctl >/dev/null 2>&1 || exit 0 + +# Stopping needs a running systemd; disabling (symlink removal) does not. +if [ -d /run/systemd/system ]; then + systemctl stop $UNITS +fi +systemctl disable $UNITS + +exit 0 diff --git a/create-a-container/systemd/container-creator.service b/create-a-container/contrib/systemd/container-creator.service similarity index 93% rename from create-a-container/systemd/container-creator.service rename to create-a-container/contrib/systemd/container-creator.service index 3325e919..f1ed7eba 100644 --- a/create-a-container/systemd/container-creator.service +++ b/create-a-container/contrib/systemd/container-creator.service @@ -12,6 +12,7 @@ Restart=on-failure Environment=NODE_ENV=production Environment=ACCESS_LOG=/var/log/opensource-server/access.log EnvironmentFile=/etc/default/container-creator +LogsDirectory=opensource-server [Install] WantedBy=multi-user.target diff --git a/create-a-container/systemd/job-runner.service b/create-a-container/contrib/systemd/job-runner.service similarity index 100% rename from create-a-container/systemd/job-runner.service rename to create-a-container/contrib/systemd/job-runner.service diff --git a/images/agent/Dockerfile b/images/agent/Dockerfile index df3cf944..c46446d9 100644 --- a/images/agent/Dockerfile +++ b/images/agent/Dockerfile @@ -1,14 +1,15 @@ +# syntax=docker/dockerfile:1 FROM nodejs -# Install all APT packages -RUN apt-get update && \ - apt-get install -y \ - libnginx-mod-stream \ - libnginx-mod-http-modsecurity \ - ssl-cert \ - make \ - dnsmasq \ - && \ +# Stage the release APT source so `apt upgrade` pulls future releases of the +# opensource-* packages automatically +COPY images/opensource-server.sources /etc/apt/sources.list.d/opensource-server.sources + +# Install the agent package (and its dependencies: nginx with stream + +# ModSecurity modules, dnsmasq, cron, ...) built by the builder image. +RUN --mount=from=builder,source=/dist,target=/dist \ + apt-get update && \ + apt-get install -y /dist/opensource-agent_*.deb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -19,9 +20,6 @@ RUN sed -i \ && sed -i -e 's/IncludeOptional/Include/' /usr/share/modsecurity-crs/owasp-crs.load \ && sed -i -e 's/^SecRuleEngine .*$/SecRuleEngine DetectionOnly/' /etc/nginx/modsecurity.conf -# Create the auth cache directory for NGINX forward auth -RUN mkdir -p /var/cache/nginx/auth_cache - # Logrotate overrides for NGINX and ModSecurity to work around a logrotate # repoen bug at https://github.com/owasp-modsecurity/ModSecurity-nginx/issues/351 COPY ./images/agent/nginx.logrotate /etc/logrotate.d/nginx @@ -31,7 +29,7 @@ COPY ./images/agent/nginx.logrotate /etc/logrotate.d/nginx COPY ./images/agent/crs-setup.conf /etc/modsecurity/crs/crs-setup.conf COPY ./images/agent/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf /etc/modsecurity/crs/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf -# Install DNSMasq and configure it to only get it's config from our pull-config +# Configure DNSMasq to only get its config from our pull-config RUN sed -i \ -e 's/^CONFIG_DIR=\(.*\)$/#CONFIG_DIR=\1/' \ -e 's/^#IGNORE_RESOLVCONF=\(.*\)$/IGNORE_RESOLVCONF=\1/' \ @@ -41,18 +39,6 @@ RUN sed -i \ RUN curl -fsSL https://get.acme.sh | sh \ && /root/.acme.sh/acme.sh --upgrade --auto-upgrade -# Install uv for building the docs -ARG UV_VERSION=0.11.14 -RUN curl -fsSL "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ - | tar -xzf - --strip-components=1 -C /usr/local/bin \ - uv-x86_64-unknown-linux-gnu/uv uv-x86_64-unknown-linux-gnu/uvx - -# Install the software. We include the .git directory so that the software can -# update itself without replacing the entire container. -COPY . /opt/opensource-server -RUN cd /opt/opensource-server \ - && make install-pull-config - # Tag the exposed ports for services handled by the container # NGINX (http, https, quic) EXPOSE 80 diff --git a/images/builder/Dockerfile b/images/builder/Dockerfile new file mode 100644 index 00000000..a44ce17d --- /dev/null +++ b/images/builder/Dockerfile @@ -0,0 +1,52 @@ +# syntax=docker/dockerfile:1 +# Builds the three opensource-server Debian packages with the component +# Makefiles + fpm. The final stage contains only the built .deb artifacts so +# the runtime images can install them with: +# +# COPY --from=builder /dist/*.deb /tmp/debs/ +# +# and CI can export them with: +# +# docker buildx bake builder --set builder.output=type=local,dest=dist +FROM debian:13 AS build + +# Build toolchain: make/git for the component Makefiles, Node.js (with npm) +# from NodeSource for the manager build, uv for the docs build, and fpm for +# packaging. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + make \ + ruby \ + ruby-dev \ + build-essential \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG NODE_MAJOR_VERSION=24 +RUN curl -fsSL https://deb.nodesource.com/setup_$NODE_MAJOR_VERSION.x | bash - && \ + apt-get update && \ + apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG UV_VERSION=0.11.14 +RUN curl -fsSL "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + | tar -xzf - --strip-components=1 -C /usr/local/bin \ + uv-x86_64-unknown-linux-gnu/uv uv-x86_64-unknown-linux-gnu/uvx + +ARG FPM_VERSION=1.16.0 +RUN gem install --no-document fpm -v "${FPM_VERSION}" + +COPY . /usr/src/opensource-server +WORKDIR /usr/src/opensource-server + +# Build all three packages into ./dist. +RUN make deb + +# Artifact-only stage. +FROM scratch +COPY --from=build /usr/src/opensource-server/dist /dist diff --git a/images/docker-bake.hcl b/images/docker-bake.hcl index 7e8f1b28..0977e607 100644 --- a/images/docker-bake.hcl +++ b/images/docker-bake.hcl @@ -13,11 +13,18 @@ target "nodejs" { } } +# Builds the Debian packages consumed by the other images. +target "builder" { + context = "../" + dockerfile = "images/builder/Dockerfile" +} + target "docs" { context = "../" dockerfile = "images/docs/Dockerfile" contexts = { base = "target:base" + builder = "target:builder" } } @@ -26,6 +33,7 @@ target "agent" { dockerfile = "images/agent/Dockerfile" contexts = { nodejs = "target:nodejs" + builder = "target:builder" } } @@ -34,6 +42,7 @@ target "manager" { dockerfile = "images/manager/Dockerfile" contexts = { agent = "target:agent" + builder = "target:builder" } } diff --git a/images/docs/Dockerfile b/images/docs/Dockerfile index 8af6f6af..dcaa8b45 100644 --- a/images/docs/Dockerfile +++ b/images/docs/Dockerfile @@ -1,21 +1,16 @@ +# syntax=docker/dockerfile:1 FROM base -# Install prerequisites -RUN apt-get update && \ - apt-get install -y make nginx && \ +# Stage the release APT source so `apt upgrade` pulls future releases. +COPY images/opensource-server.sources /etc/apt/sources.list.d/opensource-server.sources + +# Install nginx and the docs package built by the builder image +RUN --mount=from=builder,source=/dist,target=/dist \ + apt-get update && \ + apt-get install -y nginx /dist/opensource-docs_*.deb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Install uv for building the docs -ARG UV_VERSION=0.11.6 -RUN curl -fsSL "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ - | tar -xzf - --strip-components=1 -C /usr/local/bin \ - uv-x86_64-unknown-linux-gnu/uv uv-x86_64-unknown-linux-gnu/uvx - -# Build docs -COPY . /opt/opensource-server -RUN cd /opt/opensource-server && make install-docs - # Replace the default nginx site with our docs config COPY ./images/docs/docs-site.conf /etc/nginx/sites-enabled/default diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index cdb0be99..4eb624eb 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM agent # Default environment for pull-config. The manager runs the app on localhost @@ -8,9 +9,9 @@ ENV MANAGER_URL=http://localhost:3000 # Install Postgres 18 from the PGDG repository and enable remote access only # for TLS encrypted connections. Debian creates a self-signed cert for this so # we don't have to make our own. -RUN apt update && apt -y install postgresql-common \ +RUN apt-get update && apt-get -y install postgresql-common \ && /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y \ - && apt -y install postgresql-18 \ + && apt-get -y install postgresql-18 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && echo 'hostssl all all 0.0.0.0/0 scram-sha-256' >>/etc/postgresql/18/main/pg_hba.conf \ @@ -21,14 +22,16 @@ RUN apt update && apt -y install postgresql-common \ # override prevents a race condition at boot. COPY images/manager/wait-proxmox-regenerate-snakeoil.conf /etc/systemd/system/postgresql@.service.d/ -# Directory for the create-a-container access log written by morgan -# (see Environment=ACCESS_LOG in container-creator.service) and its -# logrotate config. -RUN mkdir -p /var/log/opensource-server -COPY images/manager/opensource-server.logrotate /etc/logrotate.d/opensource-server +# First-boot database initialization. +COPY images/manager/container-creator-init.service /etc/systemd/system/container-creator-init.service -# Install the software. We include the .git directory so that the software can -# update itself without replacing the entire container. -COPY . /opt/opensource-server -RUN cd /opt/opensource-server \ - && make install +# Install the manager and docs packages built by the builder image (the agent +# package is already present from the parent image; apt resolves the +# opensource-server -> opensource-docs dependency), then enable the init +# service. +RUN --mount=from=builder,source=/dist,target=/dist \ + apt-get update && \ + apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-server_*.deb && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + systemctl enable container-creator-init.service diff --git a/create-a-container/systemd/container-creator-init.service b/images/manager/container-creator-init.service similarity index 89% rename from create-a-container/systemd/container-creator-init.service rename to images/manager/container-creator-init.service index ca09a474..8c9feac2 100644 --- a/create-a-container/systemd/container-creator-init.service +++ b/images/manager/container-creator-init.service @@ -1,6 +1,8 @@ [Unit] Description=Initialize PostgreSQL for Container Creator +Requires=postgresql.service After=postgresql.service +Before=container-creator.service ConditionPathExists=!/etc/default/container-creator [Service] @@ -21,7 +23,8 @@ ExecStart=/bin/bash -e -c '\ echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> /etc/default/container-creator; \ echo "POSTGRES_USER=$${POSTGRES_USER}" >> /etc/default/container-creator; \ echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator; \ - npm run db:migrate;' + node_modules/.bin/sequelize db:migrate; \ + node_modules/.bin/sequelize db:seed:all;' [Install] WantedBy=multi-user.target diff --git a/images/opensource-server.sources b/images/opensource-server.sources new file mode 100644 index 00000000..97a80bc4 --- /dev/null +++ b/images/opensource-server.sources @@ -0,0 +1,4 @@ +Types: deb +URIs: https://github.com/mieweb/opensource-server/releases/latest/download/ +Suites: ./ +Trusted: yes diff --git a/mie-opensource-landing/.fpm b/mie-opensource-landing/.fpm new file mode 100644 index 00000000..df34938a --- /dev/null +++ b/mie-opensource-landing/.fpm @@ -0,0 +1,9 @@ +--name opensource-docs +--architecture all +--license Apache-2.0 +--maintainer "Medical Informatics Engineering " +--vendor "Medical Informatics Engineering" +--url "https://github.com/mieweb/opensource-server" +--category doc +--description "MIE opensource-server documentation site (static content). Ships the prebuilt site under /opt/opensource-server/mie-opensource-landing; serving it is a deployment concern." +--deb-suggests nginx diff --git a/mie-opensource-landing/.gitignore b/mie-opensource-landing/.gitignore index 102b6feb..b7d8e8e4 100644 --- a/mie-opensource-landing/.gitignore +++ b/mie-opensource-landing/.gitignore @@ -14,3 +14,9 @@ __pycache__/ # Misc .DS_Store tmp/ + +# packaging build artifacts +/.pkg/ +/*.deb +/*.rpm +/*.apk diff --git a/mie-opensource-landing/Makefile b/mie-opensource-landing/Makefile new file mode 100644 index 00000000..75ab5417 --- /dev/null +++ b/mie-opensource-landing/Makefile @@ -0,0 +1,58 @@ +.DEFAULT_GOAL := help + +PREFIX ?= /opt/opensource-server +DESTDIR ?= / + +PKG_NAME := opensource-docs +SITE_DEST := $(DESTDIR)$(PREFIX)/mie-opensource-landing/site + +STAGE := $(CURDIR)/.pkg/buildroot + +.PHONY: help deps build dev install deb rpm apk package clean + +help: + @echo "mie-opensource-landing — builds the opensource-docs package." + @echo "" + @echo "Targets:" + @echo " deps install dependencies (uv sync)" + @echo " build build the documentation site (zensical build)" + @echo " install stage the site into DESTDIR (default /)" + @echo " dev run the live-reloading docs server (zensical serve)" + @echo " deb build the .deb package" + @echo " rpm build the .rpm package" + @echo " apk build the .apk package" + @echo " clean remove the staging dir, build output and built packages" + @echo " help show this message" + @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + +deps: + uv sync --frozen + +build: deps + uv run --frozen zensical build + +dev: deps + uv run zensical serve + +install: build + install -d $(SITE_DEST) + cp -a site/. $(SITE_DEST)/ + +PACKAGER ?= deb +package: + rm -rf $(STAGE) + $(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX) + fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \ + -C $(STAGE) -p $(CURDIR) -f + +deb: + $(MAKE) package PACKAGER=deb +rpm: + $(MAKE) package PACKAGER=rpm +apk: + $(MAKE) package PACKAGER=apk + +clean: + rm -rf $(STAGE) site .venv + rm -f *.deb *.rpm *.apk diff --git a/mie-opensource-landing/docs/developers/docker-images.md b/mie-opensource-landing/docs/developers/docker-images.md index 84a086b0..7162f02f 100644 --- a/mie-opensource-landing/docs/developers/docker-images.md +++ b/mie-opensource-landing/docs/developers/docker-images.md @@ -22,13 +22,13 @@ Extends base with Node.js 24 from NodeSource. Inherits LDAP authentication. ### Agent (`agent`) -Extends nodejs with nginx (+ ModSecurity/OWASP CRS), dnsmasq, and [lego](https://github.com/go-acme/lego) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). +Extends nodejs with the `opensource-agent` package (pull-config, nginx with ModSecurity/OWASP CRS, dnsmasq) and [acme.sh](https://github.com/acmesh-official/acme.sh) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). **Registry:** `ghcr.io/mieweb/opensource-server/agent` · **Source:** [`images/agent/`](https://github.com/mieweb/opensource-server/tree/main/images/agent) ### Manager (`manager`) -Extends agent with PostgreSQL 18. Runs the full management application. See [Installation Guide](../admins/installation.md). +Extends agent with PostgreSQL 18 and the `opensource-server` + `opensource-docs` packages. Runs the full management application. See [Installation Guide](../admins/installation.md). **Registry:** `ghcr.io/mieweb/opensource-server/manager` · **Source:** [`images/manager/`](https://github.com/mieweb/opensource-server/tree/main/images/manager) @@ -36,6 +36,8 @@ Extends agent with PostgreSQL 18. Runs the full management application. See [Ins Images use **Docker Bake** (`docker buildx bake`) with [`images/docker-bake.hcl`](https://github.com/mieweb/opensource-server/blob/main/images/docker-bake.hcl) defining build order and dependencies. The `contexts` attribute ensures proper ordering (e.g., nodejs depends on base). +The application payloads are **not** copied into the images. Instead, the `builder` target compiles the three Debian packages (`opensource-server`, `opensource-docs`, `opensource-agent`) and the docs/agent/manager images install them via a `builder` context. See [Release Pipeline](release-pipeline.md) for the packaging details. + ``` images/ ├── docker-bake.hcl # Build config with dependency ordering @@ -45,10 +47,14 @@ images/ │ └── ldapusers ├── nodejs/ │ └── Dockerfile # Extends base image +├── builder/ +│ └── Dockerfile # Builds the .deb packages (artifact-only image) +├── docs/ +│ └── Dockerfile # Extends base (nginx + opensource-docs) ├── agent/ -│ └── Dockerfile # Extends nodejs (nginx, dnsmasq, lego) +│ └── Dockerfile # Extends nodejs (opensource-agent, ModSecurity, acme.sh) └── manager/ - └── Dockerfile # Extends agent (PostgreSQL, full app) + └── Dockerfile # Extends agent (PostgreSQL + opensource-server/docs) ``` ## CI Workflow diff --git a/mie-opensource-landing/docs/developers/pull-config.md b/mie-opensource-landing/docs/developers/pull-config.md index 1481e1ed..7620107f 100644 --- a/mie-opensource-landing/docs/developers/pull-config.md +++ b/mie-opensource-landing/docs/developers/pull-config.md @@ -21,7 +21,7 @@ pull-config/ │ ├── dnsmasq-hosts │ ├── dnsmasq-dhcp-opts │ └── dnsmasq-servers -└── install.sh # Copies scripts to /etc/ +└── Makefile # Builds the opensource-agent package (see Release Pipeline) ``` ## Environment Variables diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md new file mode 100644 index 00000000..533996f9 --- /dev/null +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -0,0 +1,119 @@ +# Release Pipeline + +The three deployable components are packaged as Debian packages and published +to GitHub Releases as a flat APT repository. The same component build commands +are reused by local development, the container images, and CI. + +## Components and packages + +| Directory | Package | Arch | Contents | +|---|---|---|---| +| [`create-a-container/`](https://github.com/mieweb/opensource-server/tree/main/create-a-container) | `opensource-server` | amd64 | Manager web app, job runner, systemd units | +| [`mie-opensource-landing/`](https://github.com/mieweb/opensource-server/tree/main/mie-opensource-landing) | `opensource-docs` | all | Prebuilt documentation site | +| [`pull-config/`](https://github.com/mieweb/opensource-server/tree/main/pull-config) | `opensource-agent` | all | pull-config engine, instances, error pages | + +Everything installs under the `/opt/opensource-server` prefix, matching the +paths referenced by the systemd units, the pull-config instances, and the +manager-rendered nginx configuration. `opensource-server` depends on +`opensource-agent` and `opensource-docs` because the manager's nginx config +serves the agent's error pages and the docs site. + +## The component Makefile contract + +Each component directory has a self-contained `Makefile` with the same targets. +The default goal is `help`, which lists the available targets. + +| Target | Description | +|---|---| +| `help` | List the available targets (default goal) | +| `deps` | Install build/runtime dependencies (`npm ci`, `uv sync`, or nothing) | +| `build` | Compile the component; depends on `deps` | +| `install` | Stage built files into `DESTDIR` at their final paths; depends on `build` | +| `dev` | Run the development watch loop; depends on `deps` | +| `deb` / `rpm` / `apk` | Stage and package with [fpm](https://fpm.readthedocs.io/); depend on `install` | + +Variables (overridable): + +| Variable | Default | Meaning | +|---|---|---| +| `PREFIX` | `/opt/opensource-server` | Vendor install prefix | +| `DESTDIR` | `/` | Staging root for `install` | + +The package version is derived from git by [`./package-version`](https://github.com/mieweb/opensource-server/blob/main/package-version), +which composes a format-appropriate string per packager from +`git describe --tags --long --dirty`: an exact tag `2026.6.3` is used as-is; +commits after a tag become a snapshot that sorts above the tag and below the +next release (e.g. deb `2026.6.3+.g`); a prerelease tag `2026.6.3-rc1` +sorts below the eventual `2026.6.3`. A leading `v` on the tag is optional — it +is stripped if present (`v2026.6.3` and `2026.6.3` are equivalent). + +```bash +# Build and stage a component anywhere: +make -C pull-config install DESTDIR=/tmp/agent-root + +# Build one package: +make -C create-a-container deb # -> create-a-container/*.deb + +# Build all three and collect them into ./dist: +make deb +``` + +The top-level `Makefile` simply forwards these targets to every component and collects the packages into `dist/`. + +## Development + +`make dev` runs the long-running watch loops: + +```bash +make -C create-a-container dev # server (nodemon) + client (vite watch) +make -C mie-opensource-landing dev # docs live server +``` + +## Packaging with fpm + +Each component has a `.fpm` options file holding the static package metadata (name, architecture, dependencies, description, scripts, config files). The Makefile's `package` target stages the component into a `.pkg/buildroot` and runs [fpm](https://fpm.readthedocs.io/) with the dynamic options on the command line — output type, version (composed per format by `./package-version`), and the staging dir. fpm's `dir` input copies the staged tree verbatim from `-C .pkg/buildroot`, preserving symlinks (e.g. `node_modules/.bin/sequelize`) and the directory layout. The same definition produces deb, rpm, and apk, so `make rpm` and `make apk` also work. + +- `opensource-server` ships the `container-creator` and `job-runner` systemd units and enables them via an `after-install` script (`before-remove` disables them on real removal). The log directory is created on demand by the unit's `LogsDirectory`, not shipped in the package. The logrotate drop-in is the only config file. The `container-creator-init` unit (which provisions a *local* PostgreSQL) is **not** in the package — it is part of the manager image, since the package only suggests postgresql and works with a remote database too. +- `opensource-agent` ships the pull-config engine, instances, cron schedule and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. +- `opensource-docs` ships content only. + +Config files are marked explicitly with `--config-files`; `--deb-no-default-config-files` stops fpm/dpkg from auto-marking everything under `/etc` as a conffile. + +## Container images + +The [`builder`](https://github.com/mieweb/opensource-server/blob/main/images/builder/Dockerfile) image runs `make deb` (Node.js, uv, and fpm) to produce all three packages, exported as an artifact-only stage. The `docs`, `agent`, and `manager` images install those packages via a Docker Bake `builder` context instead of copying the repository. The packages are bind-mounted (no extra image layer): + +```dockerfile +RUN --mount=from=builder,source=/dist,target=/dist \ + apt-get update && \ + apt-get install -y /dist/opensource-agent_*.deb +``` + +Each leaf image also stages the release APT source (`/etc/apt/sources.list.d/opensource-server.sources`) so a running container picks up future releases with `apt upgrade`. The source stays in place for the whole build which is normally not an issue since the local builder would have built a newer version than what's available in the repo. + +## Releases + +To cut a release, **publish a GitHub release** (full or prerelease) for a semver tag — the leading `v` is optional (e.g. `2026.6.3` or `v2026.6.3`, and `2026.6.3-rc1` for a prerelease). Publishing the release triggers [`release.yml`](https://github.com/mieweb/opensource-server/blob/main/.github/workflows/release.yml), which builds the packages, generates flat APT repository metadata (`Packages`, `Packages.gz`) with `dpkg-scanpackages`, and uploads the debs and metadata to that release. The workflow never creates or modifies the release itself — you choose full vs prerelease when creating it. Because GitHub serves `releases/latest/download/` for the newest non-prerelease release, the release doubles as an apt source: + +```text +deb [trusted=yes] https://github.com/mieweb/opensource-server/releases/latest/download/ ./ +``` + +Image tagging follows the same rule: `:latest` is published only when a **non-prerelease** release is published, keeping the `:latest` image channel aligned with the `releases/latest` package channel. Pre-releases publish their own assets and `:X.Y.Z` image tags without moving `:latest`. + +## Installing and updating on a host + +```bash +# One-off install (stable URL): +curl -fsSLO https://github.com/mieweb/opensource-server/releases/latest/download/opensource-agent_latest.deb +apt install -y ./opensource-agent_latest.deb + +# Or add the apt source once, then use apt normally: +cat >/etc/apt/sources.list.d/opensource-server.sources <<'EOF' +Types: deb +URIs: https://github.com/mieweb/opensource-server/releases/latest/download/ +Suites: ./ +Trusted: yes +EOF +apt update && apt install opensource-server +``` diff --git a/mie-opensource-landing/zensical.toml b/mie-opensource-landing/zensical.toml index a61f3210..261e1c18 100644 --- a/mie-opensource-landing/zensical.toml +++ b/mie-opensource-landing/zensical.toml @@ -51,6 +51,7 @@ nav = [ { "Core Technologies" = "developers/core-technologies.md" }, { "Development Workflow" = "developers/development-workflow.md" }, { "Docker Images" = "developers/docker-images.md" }, + { "Release Pipeline" = "developers/release-pipeline.md" }, { "Database Schema" = "developers/database-schema.md" }, { "Pull Config" = "developers/pull-config.md" }, { "Contributing" = "developers/contributing.md" }, diff --git a/package-version b/package-version new file mode 100755 index 00000000..77b54abc --- /dev/null +++ b/package-version @@ -0,0 +1,97 @@ +#!/bin/sh +# Print the package version string for a given packaging format, derived from +# the current git state. +# +# Usage: ./package-version +# +# The version is parsed from `git describe --tags --long --dirty`: +# VERSION base version, leading 'v' stripped, 0.0.0 if there is no tag +# PRERELEASE prerelease label (e.g. rc1), empty otherwise +# COMMITS commits since the tag (0 on an exact tag) +# HASH short commit hash (g) +# DIRTY whether the working tree has uncommitted changes +# +# Each format has a different version grammar, so the separators differ: +# prerelease : deb/rpm use ~
 (sorts below the release); apk uses _
+#   snapshot   : deb uses +. (build metadata); rpm uses
+#                ^. (post-release, sorts above); apk uses
+#                _git (apk cannot carry the hash)
+#   dirty      : appended as .dirty for deb/rpm; ignored for apk
+set -eu
+
+packager=${1:-}
+case "$packager" in
+    deb | rpm | apk) ;;
+    *)
+        echo "usage: $0 " >&2
+        exit 2
+        ;;
+esac
+
+# parse version parts from git
+describe=$(git describe --tags --long --dirty 2>/dev/null || true)
+
+dirty=0
+case "$describe" in
+    *-dirty)
+        dirty=1
+        describe=${describe%-dirty}
+        ;;
+esac
+
+if [ -n "$describe" ]; then
+    # describe is --g
+    hash=${describe##*-}            # g
+    rest=${describe%-*}             # -
+    commits=${rest##*-}            # 
+    tag=${rest%-*}                 # 
+    base=${tag#v}                  # strip a leading v
+    case "$base" in
+        *-*)
+            version=${base%%-*}
+            prerelease=${base#*-}
+            ;;
+        *)
+            version=$base
+            prerelease=""
+            ;;
+    esac
+else
+    version="0.0.0"
+    prerelease=""
+    commits=0
+    hash=""
+fi
+
+# compose the format-specific version string
+case "$packager" in
+    deb)
+        v=$version
+        [ -n "$prerelease" ] && v="$v~$prerelease"
+        if [ "$commits" != "0" ]; then
+            v="$v+$commits.$hash"
+            [ "$dirty" = "1" ] && v="$v.dirty"
+        elif [ "$dirty" = "1" ]; then
+            v="$v+dirty"
+        fi
+        ;;
+    rpm)
+        v=$version
+        [ -n "$prerelease" ] && v="$v~$prerelease"
+        if [ "$commits" != "0" ]; then
+            v="$v^$commits.$hash"
+            [ "$dirty" = "1" ] && v="$v.dirty"
+        elif [ "$dirty" = "1" ]; then
+            v="$v^dirty"
+        fi
+        ;;
+    apk)
+        # apk grammar is restrictive: digits(.digits)*[_suffix[digits]]* since it
+        # can't embed the commit hash snapshots use _git.
+        v=$version
+        [ -n "$prerelease" ] && v="${v}_$prerelease"
+        [ "$commits" != "0" ] && v="${v}_git$commits"
+        ;;
+esac
+
+printf '%s\n' "$v"
diff --git a/pull-config/.fpm b/pull-config/.fpm
new file mode 100644
index 00000000..eb462614
--- /dev/null
+++ b/pull-config/.fpm
@@ -0,0 +1,17 @@
+--name opensource-agent
+--architecture all
+--license Apache-2.0
+--maintainer "Medical Informatics Engineering "
+--vendor "Medical Informatics Engineering"
+--url "https://github.com/mieweb/opensource-server"
+--category admin
+--description "MIE opensource-server edge agent (pull-config): cron-driven distribution of nginx and dnsmasq configuration to edge nodes. Also ships the static error pages referenced by the manager-rendered nginx config."
+--depends nginx
+--depends libnginx-mod-stream
+--depends libnginx-mod-http-modsecurity
+--depends modsecurity-crs
+--depends ssl-cert
+--depends dnsmasq
+--depends curl
+--depends cron
+--deb-no-default-config-files
diff --git a/pull-config/.gitignore b/pull-config/.gitignore
new file mode 100644
index 00000000..1c09fed7
--- /dev/null
+++ b/pull-config/.gitignore
@@ -0,0 +1,5 @@
+# packaging build artifacts
+/.pkg/
+/*.deb
+/*.rpm
+/*.apk
diff --git a/pull-config/Makefile b/pull-config/Makefile
new file mode 100644
index 00000000..e3cfd106
--- /dev/null
+++ b/pull-config/Makefile
@@ -0,0 +1,61 @@
+.DEFAULT_GOAL := help
+
+PREFIX  ?= /opt/opensource-server
+DESTDIR ?= /
+
+STAGE := $(CURDIR)/.pkg/buildroot
+
+INSTALL      := install
+INSTALL_DATA := $(INSTALL) -m 0644
+INSTALL_PROG := $(INSTALL) -m 0755
+
+ERROR_PAGES := ../error-pages
+
+.PHONY: help deps build dev install deb rpm apk package clean
+
+help:
+	@echo "pull-config — builds the opensource-agent package."
+	@echo ""
+	@echo "Targets:"
+	@echo "  deps     install dependencies (none; plain bash)"
+	@echo "  build    build the component (nothing to compile)"
+	@echo "  install  stage files into DESTDIR (default /)"
+	@echo "  deb      build the .deb package"
+	@echo "  rpm      build the .rpm package"
+	@echo "  apk      build the .apk package"
+	@echo "  clean    remove the staging dir and built packages"
+	@echo "  help     show this message"
+	@echo ""
+	@echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)."
+
+# Plain bash; nothing to install, compile, or watch.
+deps:
+build: deps
+dev: deps
+
+install: build
+	$(INSTALL) -D -m 0755 bin/pull-config $(DESTDIR)$(PREFIX)/pull-config/bin/pull-config
+	$(INSTALL) -d $(DESTDIR)/etc/pull-config.d
+	$(INSTALL_PROG) etc/pull-config.d/* $(DESTDIR)/etc/pull-config.d/
+	$(INSTALL) -D -m 0644 etc/cron.d/pull-config $(DESTDIR)/etc/cron.d/pull-config
+	$(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages
+	$(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/
+	$(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache
+
+PACKAGER ?= deb
+package:
+	rm -rf $(STAGE)
+	$(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX)
+	fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \
+		-C $(STAGE) -p $(CURDIR) -f
+
+deb:
+	$(MAKE) package PACKAGER=deb
+rpm:
+	$(MAKE) package PACKAGER=rpm
+apk:
+	$(MAKE) package PACKAGER=apk
+
+clean:
+	rm -rf $(STAGE)
+	rm -f *.deb *.rpm *.apk
diff --git a/pull-config/install.sh b/pull-config/install.sh
deleted file mode 100755
index cee74a55..00000000
--- a/pull-config/install.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-# Installation script for pull-config
-# This script copies configuration files into system directories
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-echo "Installing pull-config..."
-
-# Copy cron job
-echo "Installing cron job to /etc/cron.d/pull-config..."
-install -m 644 "${SCRIPT_DIR}/etc/cron.d/pull-config" /etc/cron.d/pull-config
-
-# Create instance configuration directory
-echo "Creating /etc/pull-config.d/..."
-mkdir -p /etc/pull-config.d
-
-# Copy instance scripts
-echo "Installing instance scripts to /etc/pull-config.d/..."
-for script in "${SCRIPT_DIR}/etc/pull-config.d"/*; do
-  if [[ -f "${script}" ]]; then
-    instance=$(basename "${script}")
-    echo "  - Installing ${instance} instance"
-    install -m 755 "${script}" "/etc/pull-config.d/${instance}"
-  fi
-done
-
-# Make sure the main binary is executable
-echo "Setting executable permission on /opt/opensource-server/pull-config/bin/pull-config..."
-chmod +x "${SCRIPT_DIR}/bin/pull-config"
-
-echo ""
-echo "Installation complete!"
-echo ""
-echo "Installed instances:"
-for script in /etc/pull-config.d/*; do
-  if [[ -f "${script}" && -x "${script}" ]]; then
-    echo "  - $(basename "${script}")"
-  fi
-done
-echo ""
-echo "Instance scripts can be customized in /etc/pull-config.d/"
-echo "Each instance runs independently via cron every minute using run-parts"
-echo ""
-echo "To add a new instance:"
-echo "  1. Create an executable script in /etc/pull-config.d/"
-echo "  2. Set required environment variables (CONF_FILE, CONF_URL)"
-echo "  3. Call: exec /opt/opensource-server/pull-config/bin/pull-config"
-echo ""
-echo "To test an instance manually:"
-echo "  sudo /etc/pull-config.d/nginx"
-
-