diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..1258312 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,61 @@ +name: bridge + +# IPCR Forge's bridge image (bridge/): Gitea → Radicle mirror + the forge's page. +on: + push: + branches: [main] + tags: ["bridge-v*"] + paths: ["bridge/**", ".github/workflows/bridge.yml"] + pull_request: + paths: ["bridge/**", ".github/workflows/bridge.yml"] + workflow_dispatch: + +permissions: + contents: read + packages: write + +env: + IMAGE: ghcr.io/yundera/ipcr-forge-bridge + +jobs: + vet: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: bridge/go.mod + - run: test -z "$(gofmt -l .)" && go vet ./... && go test ./... + working-directory: bridge + + image: + needs: vet + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + if: github.event_name != 'pull_request' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + tags: | + type=match,pattern=bridge-v(.*),group=1 + type=ref,event=branch + type=sha + # amd64 only: the Radicle CLI is fetched as its x86_64 release, and the Forge listing is + # amd64 only. + - uses: docker/build-push-action@v6 + with: + context: bridge + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: VERSION=${{ steps.meta.outputs.version }} + cache-from: type=gha,scope=bridge + cache-to: type=gha,mode=max,scope=bridge diff --git a/Apps/IPCR-Forge/docker-compose.yml b/Apps/IPCR-Forge/docker-compose.yml index 675894e..a846f93 100644 --- a/Apps/IPCR-Forge/docker-compose.yml +++ b/Apps/IPCR-Forge/docker-compose.yml @@ -1,77 +1,88 @@ name: ipcr-forge -# IPCR Forge — the whole decentralised forge in one app: +# IPCR Forge — a forge whose images are served from IPFS and whose code is mirrored to Radicle: # -# git push rad main v1.2.3 (developer) -# → radicle-node / radicle-api source of truth, peer-to-peer -# → rad-actions (cib + act) the node's own CI: GitHub-style workflows, run on the -# commit straight from the node's storage -# → rad-actions-registry staging: localhost:5000/:1.2.3 (internal only) -# → ipcr-gateway follows that registry imports every tag into IPFS, publishes IPNS +# git push origin main v1.2.3 (developer, plain git) +# → gitea the source of truth: repositories, PRs, issues, users +# → forge-runner (act_runner) Gitea Actions: .gitea/workflows/ for the tag +# → forge-registry staging: localhost:5000//:1.2.3 (internal only) +# → ipcr-gateway follows that registry imports every tag into IPFS, publishes IPNS # → ipcr-kubo / ipcr-registry -# docker pull ipcr.localhost:4767/ipns/:1.2.3 (anyone) +# docker pull ipcr.localhost:4767/ipns///:1.2.3 (anyone) # -# Radicle is the only place anything originates. There is no mirror, no second forge and no -# bridge: the CI broker reads the node's events and its storage, and writes each result back -# into the repository as a job COB signed by the node. The CI's pages (cib's reports and the run -# logs) are static files the landing page serves under /ci/. +# gitea ──system webhook──▶ ipcr-forge (bridge) ──rad init / git push rad──▶ radicle-node +# every PUBLIC repository, one way: Radicle is a distribution mirror signed by this node. # -# The tile opens ipcr-forge-: a static landing page (nginx) that explains the pipeline, -# links Radicle and the CI runs, and lists every published image with the exact `image:` line to -# use (IPCR's published.json, served as /images.json). is the IPNS key IPCR minted for the -# repository (k51…), not the repository's name, which is why the page lists it. +# The tile opens Gitea. ipcr-forge- is the forge's own page, served by the bridge: how to +# publish, the published images with the `image:` line to use, and the Radicle mirrors. # -# Radicle and IPCR services are copied verbatim from their standalone listings (Apps/Radicle, -# Apps/IPCR), so each keeps its own comments and rationale.md. Only the data folders move, one -# subfolder per part: $AppID/{radicle,ci,ipcr}/. +# Gitea, Radicle and IPCR services are copied from their standalone listings (Apps/Gitea, +# Apps/Radicle, Apps/IPCR), so each keeps its own comments and rationale.md; the differences are +# marked "Forge:". Data folders move, one subfolder per part: $AppID/{gitea,radicle,bridge,ci,ipcr}/. # -# Replaces the Radicle and IPCR apps; it cannot sit beside them. It uses the same container -# names, domains (radicle-, radicle-api-) and host ports (8776, 4768, 127.0.0.1:4767). +# Replaces the Gitea, Radicle and IPCR apps; it cannot sit beside them. It uses the same container +# names, domains (gitea-, radicle-, radicle-api-) and host ports (8776, 4768, 127.0.0.1:4767). # -# The handoff from CI to IPFS is a pull, run by IPCR: jobs run in the CI's own Docker daemon, +# The handoff from CI to IPFS is a pull, run by IPCR: jobs run in the forge's own Docker daemon, # which cannot reach ipcr.localhost (the *host's* loopback). So a workflow pushes to the staging # registry, which shares that daemon's network namespace and is therefore `localhost:5000` to # every job, and ipcr-gateway polls it over ci-internal. services: - # ═══ Landing page — the tile's web UI ══════════════════════════════════════════ - # Static: the page, its icon, IPCR's list of published images and the CI's pages (read-only). - # It shows only links, documentation, image names that are public on IPFS anyway, and build - # logs of public repositories (the adapter does not run private ones), so it is not behind - # the SSO gate; Radicle is public by design. + # ═══ Bridge — the forge's page, and Gitea → Radicle ════════════════════════════ + # ipcr-forge-bridge (bridge/ in the IPCR repository). Receives Gitea's system webhook on :8081 + # (internal: no Caddy route) and mirrors every public repository to the Radicle node below, + # with the stock `rad` CLI against the node's own home. Serves the forge's page on :8080. + # Its install step (forge-setup, below) mints its Gitea token and registers the webhook. ipcr-forge: - image: nginxinc/nginx-unprivileged:1.29.3-alpine + image: ghcr.io/yundera/ipcr-forge-bridge:0.1.0 container_name: ipcr-forge user: $PUID:$PGID read_only: true cap_drop: [ALL] security_opt: ["no-new-privileges:true"] restart: unless-stopped - cpu_shares: 80 + command: ["ipcr-forge-bridge", "serve"] + cpu_shares: 30 deploy: resources: limits: - memory: 32M + memory: 256M environment: TZ: $TZ + GITEA_URL: http://gitea:3000 + GITEA_OWNER: gitea_admin + SECRETS_DIR: /secrets + STATE_DIR: /bridge + RAD_HOME: /radicle-home + IPCR_PUBLISHED: /srv/ipcr-state/published.json + RECONCILE: 10m tmpfs: - # nginx-unprivileged keeps its pid file and temp paths here. + # HOME for git and rad. - /tmp volumes: - - ${DATA_ROOT:-/DATA}/AppData/$AppID/site:/usr/share/nginx/html:ro - - ${DATA_ROOT:-/DATA}/AppData/$AppID/web/default.conf:/etc/nginx/conf.d/default.conf:ro - # published.json → /images.json. Read-only; the location in default.conf serves that file only. + # The node's home: its storage (where repositories are created and pushed), its control + # socket (announces) and its key, which signs everything it publishes. The same mount + # radicle-api has. + - ${DATA_ROOT:-/DATA}/AppData/$AppID/radicle/home/:/radicle-home/ + # repos.json (Gitea repository → Radicle ID) and one bare mirror per repository. + - ${DATA_ROOT:-/DATA}/AppData/$AppID/bridge/state:/bridge + # The Gitea token and the webhook secret the install step writes. Read-only here. + - ${DATA_ROOT:-/DATA}/AppData/$AppID/bridge/secrets:/secrets:ro + # published.json → /images.json. Read-only. - ${DATA_ROOT:-/DATA}/AppData/$AppID/ipcr/state:/srv/ipcr-state:ro - # The CI broker's report pages and the run logs → /ci/. Read-only. - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/html:/srv/ci:ro + depends_on: + - gitea + - radicle-node healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/"] + test: ["CMD", "ipcr-forge-bridge", "health"] interval: 30s timeout: 5s retries: 3 start_period: 10s expose: - 8080 + - 8081 labels: caddy_0: ipcr-forge-${APP_DOMAIN} caddy_0.import: gateway_tls @@ -84,8 +95,199 @@ services: networks: - pcs - # ═══ Radicle — the source of truth ═════════════════════════════════════════════ - # Verbatim from Apps/Radicle; only its data folder moves to $AppID/radicle/. + # ═══ Gitea — the source of truth ═══════════════════════════════════════════════ + # From Apps/Gitea; its data folder moves to $AppID/gitea/. + + # Gitea itself — git hosting, the Actions controller, and the OCI package + # registry, all in one process. SQLite: a personal forge never has the write + # concurrency that would justify a second container for Postgres. + gitea: + image: gitea/gitea:1.27.3-rootless + container_name: gitea + user: $PUID:$PGID + restart: unless-stopped + cpu_shares: 70 + deploy: + resources: + limits: + memory: 1G + environment: + TZ: $TZ + # Every URL Gitea hands out — clone URLs, the registry host, and the + # server_url the CI runner passes to jobs — is derived from ROOT_URL, so + # it has to be the address that works from outside as well as inside. + GITEA__server__ROOT_URL: https://gitea-${APP_DOMAIN}/ + GITEA__server__DOMAIN: gitea-${APP_DOMAIN} + GITEA__server__HTTP_PORT: "3000" + GITEA__server__PROTOCOL: http + # The address Gitea's own CLI uses to reach the running server's internal + # API. It defaults to localhost, which is right for the server itself and + # useless from the init container that runs + # `gitea actions generate-runner-token`. It has to be set here rather than + # there: the CLI reads it out of the shared app.ini, and only this + # container's entrypoint renders environment into app.ini. + GITEA__server__LOCAL_ROOT_URL: http://gitea:3000/ + # Git over SSH would need a published host port of its own. Everything + # here — clone, push, CI, registry — goes over HTTPS instead. + GITEA__server__DISABLE_SSH: "true" + GITEA__server__START_SSH_SERVER: "false" + # Skips the first-run install wizard: every value it would ask for is set + # here, and the admin account is created by an init step below. + GITEA__security__INSTALL_LOCK: "true" + GITEA__database__DB_TYPE: sqlite3 + GITEA__database__PATH: /var/lib/gitea/data/gitea.db + GITEA__service__DISABLE_REGISTRATION: "true" + GITEA__service__REQUIRE_SIGNIN_VIEW: "true" + GITEA__actions__ENABLED: "true" + # Where a bare `uses: actions/checkout@v4` is resolved from. + GITEA__actions__DEFAULT_ACTIONS_URL: github + # Forge: off. Images go to the staging registry that IPCR follows; a second registry would + # be a second place an image could be published from. + GITEA__packages__ENABLED: "false" + # Forge: webhooks may reach the bridge (ipcr-forge, on this server's network) and, as by + # default, any public address. Not `private`: that would let any user's webhook reach every + # container on the shared network. + GITEA__webhook__ALLOWED_HOST_LIST: external,ipcr-forge + volumes: + - ${DATA_ROOT:-/DATA}/AppData/$AppID/gitea/data:/var/lib/gitea + - ${DATA_ROOT:-/DATA}/AppData/$AppID/gitea/config:/etc/gitea + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:3000/api/healthz"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + expose: + - 3000 + labels: + caddy_0: gitea-${APP_DOMAIN} + caddy_0.import: gateway_tls + caddy_0.reverse_proxy: "{{upstreams 3000}}" + caddy_1: gitea-${APP_PUBLIC_IP_DASH}.nip.io + caddy_1.import: gateway_tls + caddy_1.reverse_proxy: "{{upstreams 3000}}" + caddy_2: gitea-${APP_PUBLIC_IP_DASH}.sslip.io + caddy_2.reverse_proxy: "{{upstreams 3000}}" + networks: + - pcs + + # ═══ CI — Gitea Actions ════════════════════════════════════════════════════════ + + # The runner: act_runner (plain, not its -dind variant) driving the separate daemon below over a + # unix socket. Runs unprivileged; whoever controls it controls that daemon, which is the point. + # Its config (seed/gitea/runner-config/config.yaml) puts jobs on the daemon's host network, where + # the staging registry is localhost:5000. + forge-runner: + image: gitea/act_runner:0.6.1 + container_name: forge-runner + user: $PUID:$PGID + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + # The stock run.sh reads the registration token once and gives up after 10 tries; the token is + # minted by an install step that runs after the stack is up. So wait for it (or for an + # existing registration) first. + entrypoint: + - /sbin/tini + - -- + - sh + - -c + - | + until [ -s /data/token ] || [ -s /data/.runner ]; do echo "waiting for the runner registration token"; sleep 2; done + until [ -S /run/dind/docker.sock ]; do sleep 1; done + exec run.sh + cpu_shares: 20 + deploy: + resources: + limits: + memory: 256M + environment: + TZ: $TZ + HOME: /data + DOCKER_HOST: unix:///run/dind/docker.sock + # Handed to every job as GITHUB_SERVER_URL, and jobs run on the daemon's network where + # `gitea` does not resolve: checkout goes out and back over HTTPS. + GITEA_INSTANCE_URL: https://gitea-${APP_DOMAIN} + GITEA_RUNNER_REGISTRATION_TOKEN_FILE: /data/token + GITEA_RUNNER_NAME: forge-runner + # Registration-time labels; runner-config/config.yaml repeats them and wins on every start. + GITEA_RUNNER_LABELS: ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04 + CONFIG_FILE: /config/config.yaml + volumes: + # .runner (this runner's registration) and the registration token the init step leaves. + - ${DATA_ROOT:-/DATA}/AppData/$AppID/gitea/runner:/data + - ${DATA_ROOT:-/DATA}/AppData/$AppID/gitea/runner-config:/config:ro + - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/socket:/run/dind + depends_on: + - gitea + - forge-docker + networks: + # Egress: it polls Gitea through its public address, like the jobs. + - ci-internal + + # The daemon jobs run in: Docker-in-Docker, so jobs and the images they build never touch the + # host's daemon. See rationale.md for what `privileged` buys and what it costs. No TCP listener: + # its only API is a unix socket in a folder shared with forge-runner, owned by $PGID. + forge-docker: + image: docker:29.1-dind + container_name: forge-docker + privileged: true + restart: unless-stopped + cpu_shares: 20 + command: ["--host=unix:///run/dind/docker.sock", "--group=$PGID"] + deploy: + resources: + limits: + memory: 2G + environment: + TZ: $TZ + # No certificates: with an explicit --host the entrypoint adds no TCP listener to protect. + DOCKER_TLS_CERTDIR: "" + volumes: + - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/socket:/run/dind + # The daemon's image store. Without it every restart re-pulls every job image. + - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/docker:/var/lib/docker + healthcheck: + test: ["CMD", "docker", "-H", "unix:///run/dind/docker.sock", "info"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + networks: + - ci-internal + + # The staging registry, in the daemon's network namespace: `localhost:5000` to the daemon and to + # every job (act runs jobs on the host network, which is this namespace), so a plain + # `docker push` or buildx with `network=host` works without TLS or credentials — Docker allows + # plain HTTP to localhost only. ipcr-gateway reaches it as forge-docker:5000. Nothing + # outside ci-internal can. + forge-registry: + image: registry:3.0.0 + container_name: forge-registry + user: $PUID:$PGID + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + network_mode: service:forge-docker + cpu_shares: 20 + deploy: + resources: + limits: + memory: 128M + environment: + TZ: $TZ + REGISTRY_HTTP_ADDR: 0.0.0.0:5000 + # Staging only: IPCR has its own copy once imported, so old tags may be deleted. + REGISTRY_STORAGE_DELETE_ENABLED: "true" + OTEL_TRACES_EXPORTER: none + volumes: + - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/registry:/var/lib/registry + depends_on: + - forge-docker + + # ═══ Radicle — the public mirror ═══════════════════════════════════════════════ + # Verbatim from Apps/Radicle; only its data folder moves to $AppID/radicle/. The bridge + # creates and pushes repositories in this node's storage; the node replicates them. # The web UI: the Radicle Explorer bundle served as static files by Caddy. # It is a pure client-side app — every request it makes goes from the visitor's @@ -210,120 +412,6 @@ services: networks: - radicle-internal - # ═══ CI — Radicle CI broker + act ═════════════════════════════════════════════════ - - # cib (radicle-ci-broker) follows the node's events over its control socket, keeps the ones its - # filters allow (changes signed by a delegate, in a repository with a workflow folder — see - # ci/ci-broker.yaml in the IPCR repository), and runs the rad-actions adapter for each. The - # adapter checks the commit out of the node's storage and runs the matching GitHub-style - # workflows with act, against the Docker daemon below. cib writes HTML reports, and the result - # of every run back into the repository as a job COB signed by this node. - rad-actions: - image: ghcr.io/yundera/rad-actions:0.1.0 - container_name: rad-actions - user: $PUID:$PGID - # Reaps act's and git's children: cib is not an init. - init: true - cap_drop: [ALL] - security_opt: ["no-new-privileges:true"] - restart: unless-stopped - cpu_shares: 20 - deploy: - resources: - limits: - memory: 512M - environment: - TZ: $TZ - # The daemon jobs run in. Its socket is the only thing shared with it. - DOCKER_HOST: unix:///run/dind/docker.sock - # The images act runs a `runs-on:` label in. act's own medium images (Ubuntu with Node and - # the Docker CLI), the ones act users run on their laptops. - RAD_ACTIONS_PLATFORMS: ubuntu-latest=catthehacker/ubuntu:act-latest,ubuntu-24.04=catthehacker/ubuntu:act-24.04,ubuntu-22.04=catthehacker/ubuntu:act-22.04 - # `vars` every workflow sees. IMAGE = IMAGE_PREFIX/, the name IPCR imports. - RAD_ACTIONS_VARS: REGISTRY=localhost:5000,IMAGE_PREFIX=localhost:5000 - # cib's pages link each run to its log there (served by ipcr-forge). - RAD_ACTIONS_LOG_URL: /ci/logs/ - # The repository's web page, as the adapter's GitHub API stand-in reports it (html_url, which - # docker/metadata-action turns into the image's org.opencontainers.image.source label). - RAD_ACTIONS_REPO_URL: https://radicle-${APP_DOMAIN}/nodes/radicle-api-${APP_DOMAIN}/{rid} - volumes: - # The node's home: its control socket (events), its storage (checkouts, job COBs) and its - # key, which signs the job COBs. The same mount radicle-api has. - - ${DATA_ROOT:-/DATA}/AppData/$AppID/radicle/home/:/radicle-home/ - # cib's database, its pages and the run logs, act's action cache, and scratch checkouts. - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/state:/ci/state - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/html:/ci/html - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/cache:/ci/cache - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/home:/ci/home - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/work:/ci/work - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/socket:/run/dind - depends_on: - - radicle-node - - rad-actions-docker - networks: - # Egress only: act fetches actions (actions/checkout, docker/*) from GitHub. - - ci-internal - - # The daemon jobs run in: Docker-in-Docker, so jobs and the images they build never touch the - # host's daemon. See rationale.md for what `privileged` buys and what it costs. No TCP listener: - # its only API is a unix socket in a folder shared with rad-actions, owned by $PGID. - rad-actions-docker: - image: docker:29.1-dind - container_name: rad-actions-docker - privileged: true - restart: unless-stopped - cpu_shares: 20 - command: ["--host=unix:///run/dind/docker.sock", "--group=$PGID"] - deploy: - resources: - limits: - memory: 2G - environment: - TZ: $TZ - # No certificates: with an explicit --host the entrypoint adds no TCP listener to protect. - DOCKER_TLS_CERTDIR: "" - volumes: - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/socket:/run/dind - # The daemon's image store. Without it every restart re-pulls every job image. - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/docker:/var/lib/docker - healthcheck: - test: ["CMD", "docker", "-H", "unix:///run/dind/docker.sock", "info"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - networks: - - ci-internal - - # The staging registry, in the daemon's network namespace: `localhost:5000` to the daemon and to - # every job (act runs jobs on the host network, which is this namespace), so a plain - # `docker push` or buildx with `network=host` works without TLS or credentials — Docker allows - # plain HTTP to localhost only. ipcr-gateway reaches it as rad-actions-docker:5000. Nothing - # outside ci-internal can. - rad-actions-registry: - image: registry:3.0.0 - container_name: rad-actions-registry - user: $PUID:$PGID - cap_drop: [ALL] - security_opt: ["no-new-privileges:true"] - restart: unless-stopped - network_mode: service:rad-actions-docker - cpu_shares: 20 - deploy: - resources: - limits: - memory: 128M - environment: - TZ: $TZ - REGISTRY_HTTP_ADDR: 0.0.0.0:5000 - # Staging only: IPCR has its own copy once imported, so old tags may be deleted. - REGISTRY_STORAGE_DELETE_ENABLED: "true" - OTEL_TRACES_EXPORTER: none - volumes: - - ${DATA_ROOT:-/DATA}/AppData/$AppID/ci/registry:/var/lib/registry - depends_on: - - rad-actions-docker - # ═══ IPCR — the IPFS container registry ════════════════════════════════════════ # Verbatim from Apps/IPCR; only its data folder moves to $AppID/ipcr/. @@ -353,7 +441,7 @@ services: # a tag is re-imported when it moves. Empty = off. IPCR_IMPORT_AUTH is user:token, # a read-only (read:package) token is enough. # Forge: watch the CI's staging registry, over ci-internal. Internal, so no credentials. - IMPORT_WATCH: ${IPCR_IMPORT_WATCH:-http://rad-actions-docker:5000} + IMPORT_WATCH: ${IPCR_IMPORT_WATCH:-http://forge-docker:5000} IMPORT_AUTH: ${IPCR_IMPORT_AUTH:-} IMPORT_REPOS: ${IPCR_IMPORT_REPOS:-} # Forge: every repository this node builds is published under one IPNS key, "forge", as @@ -477,7 +565,7 @@ networks: # From Apps/Radicle: egress only, for the node. radicle-internal: driver: bridge - # CI: rad-actions ↔ its daemon (and staging registry) ↔ ipcr-gateway; egress for builds. + # CI: the runner ↔ its daemon (and staging registry) ↔ ipcr-gateway; egress for builds. ci-internal: driver: bridge # From Apps/IPCR: gateway ↔ registry ↔ kubo, nothing else. @@ -487,7 +575,7 @@ networks: x-casaos: architectures: - amd64 - main: ipcr-forge + main: gitea author: Yundera developer: Yundera category: Developer @@ -500,102 +588,127 @@ x-casaos: title: en_us: IPCR Forge tagline: - en_us: A decentralised forge in one install — Radicle, GitHub-style CI and IPFS images. - fr_fr: Une forge décentralisée en une installation — Radicle, CI au format GitHub et images IPFS. + en_us: Gitea with Actions, images served from IPFS, code mirrored to Radicle. + fr_fr: Gitea avec Actions, images servies depuis IPFS, code répliqué sur Radicle. description: en_us: | - IPCR Forge is the full path from code to running container, without a central service: + IPCR Forge is a git forge whose releases do not depend on it staying online: - • **Radicle** holds your repositories on a peer-to-peer network. You push with `git push rad`. - • **Radicle CI** on this node builds them with GitHub-style workflows, triggered by a release tag. - • **IPCR** serves the resulting images from IPFS, so anyone can `docker pull` them by name. + • **Gitea** is where you work: repositories, pull requests, issues, and GitHub-style Actions. + Push with plain `git`. + • **IPCR** publishes every image your workflows build on IPFS, so any server running IPCR + can `docker pull` it by name, from whichever peers hold it. + • **Radicle** receives a copy of every public repository, signed by this server, on its + peer-to-peer network: anyone can clone it, even when this server is down. - Radicle is the single source of truth: builds read the commit straight from this node, and - their results go back into the repository as Radicle jobs. There is no mirror to keep in sync. + A repository publishes images by committing a workflow under `.gitea/workflows/` that + pushes to `localhost:5000`. The install creates an example, `ipcr-hello`: create a tag on it + in Gitea and watch the whole chain run. - A repository joins the pipeline by committing a workflow under `.github/workflows/`, as on - GitHub: there is nothing to configure per project. + The forge's own page explains how to publish, lists every published image with the line to + put in a compose file, and every repository mirrored to Radicle. - Its own page explains how to publish a project, links every build and its log, and lists - every published image with the line to put in a compose file. - - This app bundles the Radicle and IPCR apps. Install it instead of them, not beside them. + This app bundles the Gitea, Radicle and IPCR apps. Install it instead of them, not beside them. fr_fr: | - IPCR Forge couvre tout le chemin du code au conteneur en production, sans service central : + IPCR Forge est une forge git dont les versions publiées ne dépendent pas de sa disponibilité : - • **Radicle** garde vos dépôts sur un réseau pair-à-pair. Vous poussez avec `git push rad`. - • **Radicle CI** sur ce nœud les construit avec des workflows au format GitHub, déclenchés par un tag. - • **IPCR** sert les images produites depuis IPFS : n'importe qui peut les `docker pull`. + • **Gitea** est l'endroit où l'on travaille : dépôts, pull requests, tickets, et Actions au + format GitHub. On pousse avec `git`, tout simplement. + • **IPCR** publie sur IPFS chaque image construite par vos workflows : tout serveur équipé + d'IPCR peut la `docker pull` par son nom, depuis les pairs qui la détiennent. + • **Radicle** reçoit une copie de chaque dépôt public, signée par ce serveur, sur son réseau + pair-à-pair : n'importe qui peut la cloner, même quand ce serveur est arrêté. - Radicle est l'unique source de vérité : les builds lisent le commit directement sur ce nœud, - et leurs résultats retournent dans le dépôt sous forme de jobs Radicle. Aucun miroir à - synchroniser. + Un dépôt publie des images en ajoutant un workflow sous `.gitea/workflows/` qui pousse vers + `localhost:5000`. L'installation crée un exemple, `ipcr-hello` : créez-y un tag dans Gitea et + regardez toute la chaîne s'exécuter. - Un dépôt rejoint la chaîne en ajoutant un workflow sous `.github/workflows/`, comme sur - GitHub : rien à configurer par projet. + La page de la forge explique comment publier, liste chaque image publiée avec la ligne à + mettre dans un fichier compose, et chaque dépôt répliqué sur Radicle. - Sa page explique comment publier un projet, donne accès à chaque build et à son journal, et - liste chaque image publiée avec la ligne à mettre dans un fichier compose. - - Cette application regroupe les applications Radicle et IPCR. Installez-la à leur place, pas à - côté. + Cette application regroupe les applications Gitea, Radicle et IPCR. Installez-la à leur + place, pas à côté. tips: before_install: en_us: | - **Replaces two apps.** Uninstall Radicle and IPCR first: this app uses the same addresses - and ports. + **Start here: [the forge's page](https://ipcr-forge-$APP_DOMAIN)**: how to publish, the + published images, the Radicle mirrors. + + **Replaces three apps.** Uninstall Gitea, Radicle and IPCR first: this app uses the same + addresses and ports. | Address | What | | --- | --- | - | `https://ipcr-forge-$APP_DOMAIN` | This app: how to publish, every published image | - | `https://ipcr-forge-$APP_DOMAIN/ci/` | CI runs and their logs (public) | + | `https://gitea-$APP_DOMAIN` | Gitea (opens from the tile). Sign in as `gitea_admin` with your default app password | + | `https://ipcr-forge-$APP_DOMAIN` | The forge's page (public) | | `https://radicle-$APP_DOMAIN` | Radicle explorer (public) | | `https://radicle-api-$APP_DOMAIN` | Radicle API and read-only git (public) | | `ipcr.localhost:4767` | IPCR registry, this server's Docker only | **Ports.** TCP 8776 (Radicle peers) and 4768 TCP+UDP (IPFS peers) are published. - **Who can build.** Only changes signed by one of a repository's delegates trigger a build: - release tags, the default branch and their patches. Builds run in their own Docker daemon. + **Public repositories go to Radicle.** Every public Gitea repository is copied there, signed + by this server, within seconds of each push. That cannot be undone: making a repository + private later stops the copy, and what was published stays on the network. Private + repositories are never copied. - **Images reach IPFS on their own.** A workflow pushes to `vars.IMAGE` (this node's - staging registry); about a minute later the image can be pulled as - `ipcr.localhost:4767/ipns/:`, where `` is the IPNS name (`k51…`) IPCR - creates for the repository. The app's page lists each one with the line to copy. + **Who can publish images.** Anyone who can push to a repository here: a build can push any + image name to the staging registry, and IPCR publishes what it finds there under this + forge's name. Images are public once on IPFS, even from a private repository. Read a + fork's workflow before approving its run. fr_fr: | - **Remplace deux applications.** Désinstallez d'abord Radicle et IPCR : cette application - utilise les mêmes adresses et ports. + **Commencez ici : [la page de la forge](https://ipcr-forge-$APP_DOMAIN)** : publier, les + images publiées, les copies Radicle. + + **Remplace trois applications.** Désinstallez d'abord Gitea, Radicle et IPCR : cette + application utilise les mêmes adresses et ports. | Adresse | Quoi | | --- | --- | - | `https://ipcr-forge-$APP_DOMAIN` | Cette application : publier, toutes les images publiées | - | `https://ipcr-forge-$APP_DOMAIN/ci/` | Builds et leurs journaux (public) | + | `https://gitea-$APP_DOMAIN` | Gitea (s'ouvre depuis la tuile). Connexion : `gitea_admin` et votre mot de passe d'applications par défaut | + | `https://ipcr-forge-$APP_DOMAIN` | La page de la forge (publique) | | `https://radicle-$APP_DOMAIN` | Explorateur Radicle (public) | | `https://radicle-api-$APP_DOMAIN` | API Radicle et git en lecture seule (public) | | `ipcr.localhost:4767` | Registre IPCR, pour le Docker de ce serveur uniquement | **Ports.** TCP 8776 (pairs Radicle) et 4768 TCP+UDP (pairs IPFS) sont publiés. - **Qui peut lancer un build.** Seuls les changements signés par un délégué du dépôt - déclenchent un build : tags de version, branche par défaut et leurs patches. Les builds - tournent dans leur propre démon Docker. + **Les dépôts publics partent sur Radicle.** Chaque dépôt public de Gitea y est copié, signé + par ce serveur, quelques secondes après chaque push. C'est irréversible : rendre un dépôt + privé ensuite arrête la copie, et ce qui a été publié reste sur le réseau. Les dépôts privés + ne sont jamais copiés. - **Les images arrivent sur IPFS d'elles-mêmes.** Un workflow pousse vers `vars.IMAGE` - (le registre de transit de ce nœud) ; environ une minute plus tard, l'image se tire avec - `ipcr.localhost:4767/ipns/:`, où `` est le nom IPNS (`k51…`) qu'IPCR crée - pour le dépôt. La page de l'application les liste, prêts à copier. + **Qui peut publier des images.** Toute personne qui peut pousser sur un dépôt d'ici : un + build peut pousser n'importe quel nom d'image vers le registre de transit, et IPCR publie ce + qu'il y trouve sous le nom de cette forge. Une image est publique une fois sur IPFS, même + si elle vient d'un dépôt privé. Lisez le workflow d'un fork avant d'approuver son exécution. index: / - webui_port: 8080 + webui_port: 3000 store_app_id: ipcr-forge x-compose-app: schema_version: 2 folders: - # ── Landing page ── - - path: /DATA/AppData/$AppID/site + # ── Bridge ── + - path: /DATA/AppData/$AppID/bridge/state user: $PUID group: $PGID - - path: /DATA/AppData/$AppID/web + - path: /DATA/AppData/$AppID/bridge/secrets + user: $PUID + group: $PGID + mode: "0700" + # ── Gitea ── + - path: /DATA/AppData/$AppID/gitea/data + user: $PUID + group: $PGID + - path: /DATA/AppData/$AppID/gitea/config + user: $PUID + group: $PGID + - path: /DATA/AppData/$AppID/gitea/runner + user: $PUID + group: $PGID + mode: "0700" + - path: /DATA/AppData/$AppID/gitea/runner-config user: $PUID group: $PGID # ── Radicle ── @@ -605,25 +718,7 @@ x-compose-app: - path: /DATA/AppData/$AppID/radicle/caddy user: $PUID group: $PGID - # ── CI ── - - path: /DATA/AppData/$AppID/ci/state - user: $PUID - group: $PGID - mode: "0700" - - path: /DATA/AppData/$AppID/ci/html/logs - user: $PUID - group: $PGID - - path: /DATA/AppData/$AppID/ci/cache - user: $PUID - group: $PGID - - path: /DATA/AppData/$AppID/ci/home - user: $PUID - group: $PGID - mode: "0700" - - path: /DATA/AppData/$AppID/ci/work - user: $PUID - group: $PGID - mode: "0700" + # ── CI daemon ── # The daemon's socket: created by root, group $PGID (dockerd --group), mode 0660. - path: /DATA/AppData/$AppID/ci/socket user: $PUID @@ -646,21 +741,12 @@ x-compose-app: user: $PUID group: $PGID files: - # The landing page follows store updates: re-rendered on every up. None of the three holds a - # dollar-brace sequence, so rendering cannot fail on an unresolved variable. - - path: /DATA/AppData/$AppID/site/index.html - from: site/index.html + # Each entry and its reason are in the standalone listing it comes from. All follow store + # updates (re-rendered on every up). + - path: /DATA/AppData/$AppID/gitea/runner-config/config.yaml + from: gitea/runner-config/config.yaml ensure: always mode: "0644" - - path: /DATA/AppData/$AppID/site/icon.png - from: site/icon.png - ensure: always - mode: "0644" - - path: /DATA/AppData/$AppID/web/default.conf - from: web/default.conf - ensure: always - mode: "0644" - # Each entry and its reason are in the standalone listing it comes from. - path: /DATA/AppData/$AppID/radicle/home/config.json from: radicle/home/config.json.tmpl ensure: always @@ -674,7 +760,8 @@ x-compose-app: ensure: always mode: "0755" init: - # Radicle's identity before its node starts. The CI needs nothing: it uses the node's. + # In order: Radicle's identity before its node starts, then Gitea's admin, then the runner + # token and the bridge's setup, which both need that admin to exist. - name: radicle-identity image: dirk1980/radicle-seed-node@sha256:78ccee526ad4b4f4d7c79c2323b032a7209858a48e1dc9c077272ff1660050e2 user: $PUID:$PGID @@ -682,5 +769,54 @@ x-compose-app: - /DATA/AppData/$AppID/radicle/home:/radicle-home command: ["sh", "-c", "yes '' | rad auth --alias radicle-${APP_PUBLIC_IP_DASH} --stdin"] when: absent:/DATA/AppData/$AppID/radicle/home/keys/radicle - webui-host: ipcr-forge-${domain} + - name: gitea-admin + image: gitea/gitea:1.27.3-rootless + user: $PUID:$PGID + phase: post_up + volumes: + - /DATA/AppData/$AppID/gitea/data:/var/lib/gitea + - /DATA/AppData/$AppID/gitea/config:/etc/gitea + command: + - sh + - -c + - | + timeout 240 sh -c 'until gitea admin user list >/dev/null 2>&1; do sleep 3; done' + gitea admin user create --admin --username gitea_admin \ + --password "$APP_DEFAULT_PASSWORD" --email "$APP_EMAIL" \ + --must-change-password=false \ + || echo "admin account already present, leaving it alone" + when: always + - name: gitea-runner-token + image: gitea/gitea:1.27.3-rootless + user: $PUID:$PGID + phase: post_up + network: $APP_NET + volumes: + - /DATA/AppData/$AppID/gitea/data:/var/lib/gitea + - /DATA/AppData/$AppID/gitea/config:/etc/gitea + - /DATA/AppData/$AppID/gitea/runner:/runner + command: + - sh + - -c + - | + timeout 240 sh -c 'until gitea actions generate-runner-token > /runner/token.new 2>/dev/null; do sleep 3; done' + mv /runner/token.new /runner/token + chmod 600 /runner/token + when: absent:/DATA/AppData/$AppID/gitea/runner/.runner + # The bridge's Gitea token (public-only, read:repository), the webhook secret, the system + # webhook, and the example repository. Idempotent, so on every start: it repairs the webhook. + # The admin password is used here only, never stored. + - name: forge-setup + image: ghcr.io/yundera/ipcr-forge-bridge:0.1.0 + user: $PUID:$PGID + phase: post_up + network: $APP_NET + volumes: + - /DATA/AppData/$AppID/bridge/secrets:/secrets + command: + - sh + - -c + - GITEA_PASSWORD="$APP_DEFAULT_PASSWORD" GITEA_URL=http://gitea:3000 ipcr-forge-bridge setup + when: always + webui-host: gitea-${domain} webui-path: / diff --git a/Apps/IPCR-Forge/rationale.md b/Apps/IPCR-Forge/rationale.md index 5d60065..f29934a 100644 --- a/Apps/IPCR-Forge/rationale.md +++ b/Apps/IPCR-Forge/rationale.md @@ -1,112 +1,148 @@ # IPCR Forge — Rationale -IPCR Forge bundles two store apps (Radicle, IPCR), the node's own CI (`rad-actions`) and a static -landing page. The Radicle and IPCR services are copied verbatim, so each keeps the deviations its -standalone listing already argues. This document lists them, then covers only what the bundle adds. +IPCR Forge bundles three store apps (Gitea, Radicle, IPCR), a CI runner with its own Docker daemon, +and the bridge (`ipcr-forge-bridge`), which mirrors public Gitea repositories to Radicle and +serves the forge's page. The Gitea, Radicle and IPCR services are copied from their listings, +so each keeps the deviations its standalone listing already argues. This document lists them, +then covers only what the bundle adds. -Earlier versions built with Gitea Actions, fed by a bridge that copied every Radicle repository -into Gitea. That made two places where state could start (Gitea held a second copy of each -repository, with its own PRs, issues and registry) and needed a glue service to keep them in step. -Gitea, its runner and the bridge are gone; the CI now runs on the Radicle node itself. +Gitea is the source of truth. Earlier versions made Radicle the source: first with a read-only +Gitea fed from it (v1), then with CI on the Radicle node alone (v2). Users found both hard to work +with. See docs/forge.md. ## What deviation / exception is being requested -**Inherited, unchanged** (the full argument is in each listing's `rationale.md`): +**Inherited, unchanged.** The full argument for each is in that listing's `rationale.md`. | From | Deviation | | --- | --- | +| Gitea | An admin account (`gitea_admin`) created with the server's default app password | | Radicle | No authentication gate on `radicle-` and `radicle-api-`; host port `8776/tcp` | | IPCR | `ipcr-gateway` and `ipcr-registry` run as root (no capabilities); `/etc/docker/certs.d` mounted; no authentication; ports `127.0.0.1:4767` and `4768/tcp+udp` | **Added by the bundle:** -1. **`rad-actions-docker` runs `privileged`** (Docker-in-Docker), as Gitea's runner did. -2. **`rad-actions` mounts the Radicle node's home read-write**, including the node's key. -3. **A cross-app network link.** `ipcr-gateway` joins `ci-internal`, where the staging registry is. -4. **A public landing page, including the CI's pages.** `ipcr-forge-` and its `/ci/` - (run reports and logs) are not behind the SSO gate. -5. **It replaces two apps** and cannot be installed beside them: same container names, domains - and host ports. +1. **`forge-docker` runs `privileged`** (Docker-in-Docker), as Gitea's runner did. +2. **The bridge (`ipcr-forge`) mounts the Radicle node's home read-write**, including the node's + key. It publishes repositories signed by that key. +3. **A system webhook and its allow-list.** Gitea may deliver webhooks to `ipcr-forge` on the + server's network (`GITEA__webhook__ALLOWED_HOST_LIST: external,ipcr-forge`). +4. **An install step that uses the admin password** to mint the bridge's token and to register + that webhook. +5. **A cross-app network link.** `ipcr-gateway` joins `ci-internal`, where the staging registry + is. +6. **A public page.** `ipcr-forge-` is not behind the SSO gate. +7. **Public repositories are published to Radicle automatically**, and that cannot be undone. +8. **It replaces three apps** and cannot be installed beside them: it uses the same container names, + domains and host ports. ## Why it is necessary -**1. Privileged Docker-in-Docker.** Building an image needs a Docker daemon. The two ways to give -a CI job one are the host's socket (root on the PCS for every workflow) or a daemon of its own, -which needs `privileged`. The second confines a build to its own container and image store. The -rootless variant does not start on this platform (`kernel.apparmor_restrict_unprivileged_userns=1`; -see Apps/Gitea/rationale.md, where it was tried first). +**1. Privileged Docker-in-Docker.** Building an image needs a Docker daemon. A CI job can get one +in two ways: +- the host's socket, which gives every workflow root on the PCS; +- a daemon of its own, which needs `privileged`. -**2. The node's home.** The CI broker (`cib`, upstream `radicle-ci-broker`) is a client of the -node: it subscribes to the node's events over the control socket in that folder, reads repository -storage to check out a commit, and writes each run's result back into the repository as a job COB, -which the node's key signs. This is the mount `radicle-api` already has. It is what removes the -mirror: the build reads the commit where it lives, so there is no copy to drift. +The second keeps a build inside its own container and image store. The rootless variant does +not start on this platform (`kernel.apparmor_restrict_unprivileged_userns=1`; see +Apps/Gitea/rationale.md, where it was tried first). The runner itself is a separate, unprivileged +container that reaches the daemon over a unix socket. -**3. The network link.** IPCR publishes images by *pulling* them: it polls a registry and imports -every new tag into IPFS. Jobs run in the CI's own daemon, which cannot reach `ipcr.localhost` (that -name is the host's loopback), so a workflow pushes to a staging registry instead. That registry -shares the daemon's network namespace, so it is `localhost:5000` to every job — the only address -Docker and BuildKit accept plain HTTP for, so no TLS or credentials are involved. `ipcr-gateway` -reaches it as `rad-actions-docker:5000` over `ci-internal`. +**2. The node's home.** Publishing to Radicle means three things: creating a repository in the +node's storage, signing its refs with the node's key, and announcing them through the node's +control socket. Those live in that folder. This is the mount `radicle-api` already has. The bridge +uses the stock `rad` CLI (1.10.1, the node's version) rather than reimplementing any of it. -**4. The public pages.** The landing page holds links, documentation and image names that are -public on IPFS anyway. `/ci/` shows `cib`'s report pages and the run logs. Radicle repositories are -public by design, and the adapter refuses to run a private repository, so a log never shows code -that is not already public. A login in front would stand between a developer and their build log. +**3. The webhook.** Pushes reach Radicle within seconds instead of at the next 10-minute +reconcile. Gitea's default allow-list (`external`) refuses private addresses, and the bridge sits +on one. The list adds exactly one host name. `private` would let any user's webhook reach every +container on the shared network. -**5. Replacing two apps.** The bundle is the same services, not a different product. Renaming every -container, domain and port to coexist would break the defaults the services rely on -(`radicle-api:8080`) and the documented addresses users type. +**4. The admin password at install.** Gitea's token API and its system-webhook API accept only an +administrator signing in. The step uses the password once per start, never stores it, and +leaves the running service holding nothing but a `public-only`, `read:repository` token. If the +password has been changed, the step keeps the existing token and webhook instead of failing. + +**5. The network link.** IPCR publishes images by *pulling* them: it polls a registry and +imports every new tag into IPFS. Jobs run in the CI's own daemon, which cannot reach +`ipcr.localhost`, because that name is the host's loopback. So a workflow pushes to a staging +registry instead. That registry shares the daemon's network namespace, so every job sees it as +`localhost:5000`. That is the only address Docker and BuildKit accept plain HTTP for, so no TLS +or credentials are involved. `ipcr-gateway` reaches it as `forge-docker:5000` over `ci-internal`. + +**6. The public page.** It holds links, documentation, image names that are public on IPFS anyway, +and the list of repositories already public on Radicle. A login in front of it would stand between +a stranger and the instructions for pulling an image. + +**7. Publishing to Radicle.** That is the point of the mirror: the code of a released image stays +available without this server. Only public repositories go. A public Gitea repository is meant to +be read by anyone, and Radicle is one more place to read it. + +**8. Replacing three apps.** The bundle runs the same services, not a different product. Renaming +every container, domain and port so they could coexist would break the defaults the services rely +on (`radicle-api:8080`, `gitea:3000`) and the addresses users type. ## Security mitigations in place -- **Only delegates can start a build.** `cib`'s filter (ci/ci-broker.yaml) runs CI only for changes - that come from a delegate of the repository (`AnyDelegate`): release tags, the default branch and - their patches. A stranger's patch is never run. This matters because a build can publish: what a - job pushes to the staging registry ends up on IPFS under the repository's IPNS name. A build - therefore gets exactly the trust of a release. -- **The privileged daemon has no network listener.** `dockerd` gets an explicit - `--host=unix:///run/dind/docker.sock` and `DOCKER_TLS_CERTDIR` is empty, so the entrypoint adds no - TCP socket. The unix socket sits in a folder shared only with `rad-actions`, group `$PGID`. -- **`rad-actions` itself runs unprivileged:** `$PUID`, every capability dropped, - `no-new-privileges`, 512 MB. Whoever controls it controls the daemon (that is the point of the - socket), so it accepts no input but the node's events and a workflow from a delegate. -- **The node's key never enters a job.** Jobs run in the nested daemon from a self-contained, - depth-1 checkout that the adapter fetched; the Radicle home is not mounted into them. -- **The staging registry is unreachable from outside.** No port, no Caddy route, on `ci-internal` - only (`rad-actions`, the daemon and `ipcr-gateway`). It runs as `$PUID` with no capabilities. -- **Bounded.** One run at a time (`concurrent_adapters: 1`), 60 minutes per run, 2 GB for the daemon. -- **The landing page cannot change anything.** `nginx-unprivileged` runs as `$PUID` with a - read-only root filesystem, no capabilities, `no-new-privileges` and 32 MB. It mounts the page, - its config, IPCR's `state/` and the CI's `html/`, all read-only, and serves static files only. - From `state/` it maps exactly one file (`/images.json` → `published.json`). -- **The CI is severable.** Stopping `rad-actions` and `rad-actions-docker` removes the privileged - container and leaves a working Radicle node and IPCR registry. -- **Smaller than what it replaces.** Gone: an admin account with the default password, two minted - tokens, the `DAC_READ_SEARCH` capability IPCR needed to read one of them, and a second git server. +**The bridge** +- **It sees public repositories only.** Its Gitea token is scoped `public-only` and + `read:repository`. Gitea hides every private or limited repository from it, and it can write + nothing in Gitea. The code checks visibility a second time. Forks and pull mirrors are skipped. +- **Webhooks are authenticated and carry no content.** Each delivery must pass the HMAC-SHA256 + check against a random secret written at install (`bridge/secrets`, 0700). A delivery only says + *which* repository changed. The bridge re-reads that repository from Gitea, so a forged payload + can at most cause a pointless sync. Port 8081 has no Caddy route. +- **It runs unprivileged:** `$PUID`, every capability dropped, a read-only root filesystem, + `no-new-privileges`, 256 MB. One worker does one sync at a time. +- **The node's key stays in the bridge and the node.** No CI job mounts the Radicle home. + +**The CI daemon** +- **No network listener.** `dockerd` gets an explicit `--host=unix:///run/dind/docker.sock` and an + empty `DOCKER_TLS_CERTDIR`. The unix socket sits in a folder shared only with `forge-runner`, + group `$PGID`. +- **`forge-runner` runs unprivileged:** `$PUID`, no capabilities, `no-new-privileges`. Jobs run in + the nested daemon, never on the host's. +- **The staging registry is unreachable from outside.** It has no port and no Caddy route and sits + on `ci-internal` only. It runs as `$PUID` with no capabilities. +- **Bounded.** One job at a time, at most 1 hour per job, 2 GB for the daemon. +- **Severable.** Stopping `forge-runner` and `forge-docker` removes the privileged container and + leaves a working Gitea, Radicle node and IPCR registry. + +**Gitea** +- **Fork pull requests wait for approval.** In Gitea 1.27, a run from a user without write access + waits for "Approve and run". +- **Registration is off.** Every account is created by the admin. +- **Who can publish is stated** in the tips and on the page. Write access to any repository means + being able to publish images under the forge's name. Images from private repositories become + public on IPFS. + +**The page** +- **It cannot change anything.** It is served by the bridge's own read-only HTTP handler: the + embedded page, IPCR's `published.json` (mounted read-only) and the bridge's repository list. ## Alternatives considered and rejected | Alternative | Why not | | --- | --- | -| Keep Gitea, fed by a bridge (previous version) | Two sources of state (a second copy of each repository, with its own PRs and registry) and a glue service to keep them in step. | -| Gitea as a strict read-only mirror | Still a second git server and a sync to babysit, for one thing it adds: a web view of builds, which `cib`'s pages now provide. | -| Gitea's `act_runner` in `exec` mode | Same engine as `act` (a fork), but built for Gitea; nektos `act` is what people run on their laptops. | -| `cib`'s native adapter, or Ambient | Neither runs GitHub-style workflows; Ambient needs KVM. | -| `ipcrd` accepting pushes itself | A push endpoint would be a second registry implementation. The staging registry plus IPCR's existing import is byte-identical to `nerdctl push` and needed no change. | -| Host Docker socket for jobs | Any workflow would get root on the host. | -| Run patches from anyone | Their build could publish under the repository's IPNS name. Needs a separate daemon with no registry first. | +| Radicle as the source, Gitea read-only (v1) | Users had to work in Radicle and found a read-only Gitea confusing | +| Radicle only, CI on the node (v2) | Not user-friendly enough: every step went through `rad` | +| Gitea push mirror to Radicle | Gitea cannot push to `rad://` (no `git-remote-rad`), and mirror pushes run on a timer | +| Publishing to Radicle from a workflow job | It would put the node's key inside CI jobs | +| A per-repository webhook | Every repository would need setup; one system webhook covers all of them, including new ones | +| Gitea's package registry as the staging registry (v1) | Needs tokens for the workflow and for IPCR, plus the `DAC_READ_SEARCH` capability for IPCR to read its token. It would also be a second place images get published from | +| Host Docker socket for jobs | Any workflow would get root on the host | +| A long-lived admin token for the bridge | The webhook is repaired by the install step on every start instead, so the running service never holds admin rights | ## Known limitations -- **No job COB for tag builds.** cib 0.32.1 keys a job by the event's tip, which for an annotated - tag is the tag object rather than a commit, and creating that job fails (silently; the log only - shows `NoJob` when the run finishes). Branch and patch builds get their job COB. Tag builds still - appear in `/ci/` with their log. Upstream fix: peel the tag before creating the job. -- **Tags arrive as pushes.** cib sends a tag event as `event_type: push` with the tag's name in - `branch`. The adapter tells them apart by looking for that name under `refs/tags/` in storage. -- **`HasFile` matches files only**, not folders, so cib cannot pre-filter on a workflow folder; the - adapter answers "no workflow" itself. +- **Radicle cannot unpublish.** A repository made private or deleted in Gitea is *frozen* (no + further sync). What already reached the network stays there. +- **A deleted tag stays in Radicle's canonical `refs/tags/`.** It is removed from the node's own + namespace only. +- **Radicle signatures are the forge's, not the developer's.** The node is each repository's only + delegate. +- **Image names are not tied to repositories.** A build may push any name to the staging registry. + Per-repository credentials, issued by the bridge, are a planned follow-up. ## Data protection @@ -114,12 +150,15 @@ Everything lives under `/DATA/AppData/ipcr-forge/`, one folder per part: | Folder | Holds | | --- | --- | -| `radicle/` | node identity (key pair), seeded repositories (including job COBs), Caddy config | -| `ci/` | `cib`'s database (`state/`), report pages and run logs (`html/`), act's action cache, the daemon's image store (`docker/`), the staging registry (`registry/`, disposable) | +| `gitea/` | repositories, the database (SQLite), configuration, the runner's registration and config | +| `radicle/` | node identity (key pair), seeded repositories, Caddy config | +| `bridge/` | `state/`: the Gitea → Radicle mapping and one bare mirror per repository (disposable); `secrets/` (0700): its Gitea token and the webhook secret | +| `ci/` | the daemon's socket, its image store (`docker/`), the staging registry (`registry/`, disposable) | | `ipcr/` | Kubo repo (node identity, IPNS keys, pinned images), TLS CA, watcher state | -| `site/`, `web/` | the landing page and its nginx config, re-rendered on every start | No user directory (`/DATA/Documents`, `Media`, …) is mounted. The one file outside `/DATA` is IPCR's `/etc/docker/certs.d/ipcr.localhost:4767/ca.crt`, as in the standalone app. Identities that -matter on the network (Radicle's node key, Kubo's node key and IPNS keys) survive uninstall and -reinstall, so published repositories and image names stay valid. +matter on the network survive uninstall and reinstall, so published repositories and image names +stay valid: +- Radicle's node key; +- Kubo's node key and IPNS keys. diff --git a/Apps/IPCR-Forge/seed/gitea/runner-config/config.yaml b/Apps/IPCR-Forge/seed/gitea/runner-config/config.yaml new file mode 100644 index 0000000..8e8f42f --- /dev/null +++ b/Apps/IPCR-Forge/seed/gitea/runner-config/config.yaml @@ -0,0 +1,42 @@ +# act_runner configuration for IPCR Forge. Only the settings that differ from upstream's defaults +# are written out — `act_runner generate-config` prints the full set. From Apps/Gitea's, except +# where marked "Forge:". + +log: + level: info + +runner: + # The file the runner's own registration is kept in, relative to /data. + file: .runner + # One job at a time. A PCS shares its CPU with everything else installed on + # it, and an image build is the least polite thing on the box. + capacity: 1 + timeout: 1h + fetch_timeout: 5s + fetch_interval: 2s + # Overrides the labels stored at registration (.runner). Gitea's own job images, which carry + # node, bash, git and the docker CLI with buildx, so marketplace actions (actions/checkout, + # docker/build-push-action, ...) run as they do on GitHub. + labels: + - "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest" + - "ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04" + - "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04" + +cache: + # The cache server binds an address inside the runner container, which the + # job containers — created by the separate Docker daemon — cannot route to. + # Left on, every `actions/cache` step fails noisily instead of being skipped. + enabled: false + +container: + # Forge: jobs run on the daemon's host network, which is the forge-docker container's own + # namespace — where the staging registry listens as localhost:5000. Plain HTTP to localhost is + # the one thing Docker and BuildKit accept without TLS, so a workflow needs no login and no + # certificate. Jobs still reach the internet (actions, base images) and Gitea through its public + # address. + network: host + privileged: false + # Empty: the runner's own daemon (DOCKER_HOST) is mounted into each job, so `docker build` + # and buildx work in a job, against the same daemon (and the same localhost:5000). + docker_host: "" + force_pull: false diff --git a/Apps/IPCR-Forge/seed/web/default.conf b/Apps/IPCR-Forge/seed/web/default.conf deleted file mode 100644 index be820a5..0000000 --- a/Apps/IPCR-Forge/seed/web/default.conf +++ /dev/null @@ -1,29 +0,0 @@ -# IPCR Forge landing page. Static, plus IPCR's published.json as /images.json and the CI broker's -# report pages (and the adapter's run logs) under /ci/. -# No nginx variables on purpose: this file is also safe to render as a Maison template. -server { - listen 8080; - server_tokens off; - root /usr/share/nginx/html; - index index.html; - - location = /images.json { - alias /srv/ipcr-state/published.json; - default_type application/json; - add_header Cache-Control "no-store"; - } - - # cib rewrites these pages as runs progress; logs are plain text. - location /ci/ { - alias /srv/ci/; - index index.html; - types { text/html html; text/css css; application/json json; text/plain log; } - charset utf-8; - add_header Cache-Control "no-cache"; - } - - # Only the page and its icon; nothing else under the state directory is reachable. - location / { - add_header Cache-Control "no-cache"; - } -} diff --git a/docs/forge.md b/docs/forge.md index 5961e94..6e01576 100644 --- a/docs/forge.md +++ b/docs/forge.md @@ -1,185 +1,176 @@ # IPCR Forge — design notes How the forge in [Apps/IPCR-Forge/](../Apps/IPCR-Forge/) goes from `git push` to `docker pull`, why it -is shaped this way, and what was learned building it (2026-10-04). The listing's own security -argument is in [rationale.md](../Apps/IPCR-Forge/rationale.md). +is shaped this way, and what was learned building it. The listing's own security argument is in +[rationale.md](../Apps/IPCR-Forge/rationale.md). ## Goal -A decentralised forge that works out of the box: **Radicle is the single source of truth**, builds -use GitHub-style workflows (the developer comfort worth keeping), and images are served from IPFS -through IPCR. Nothing in the chain is a second copy that can drift from Radicle. +A forge people actually want to use, whose releases do not depend on it staying online: -## Why Gitea was removed +- **Gitea is the source of truth.** Repositories, pull requests, issues, users and CI live there. + Developers use plain git and a familiar web UI. +- **Images go to IPFS** through IPCR, under one name for the whole forge, so any server running IPCR + can pull them from any peer that holds them. +- **Code goes to Radicle.** Every public repository is mirrored, one way, to the Radicle network, + so it can be cloned without this server. -The first version built with Gitea Actions. A bridge service copied every Radicle repository into -Gitea, Gitea's runner built it, and IPCR imported from Gitea's registry. It worked end to end, but: +## History: three versions -- **Two places where state could originate.** Gitea held a full copy of each repository, with its - own PRs, issues and registry. A copy that is only ever *derived* is a cache; one where things can - also *start* will desync. -- **A glue service** (the bridge) to keep the two in step, plus its credentials: an admin account, - two minted tokens, and a capability (`DAC_READ_SEARCH`) IPCR needed only to read one of them. -- What Gitea actually contributed was narrow: running Actions YAML, a web view of runs, and a - staging registry. None of those needs a forge. +| Version | Source of truth | CI | Feedback | +| --- | --- | --- | --- | +| v1 (`afb1072`) | Radicle | Gitea Actions on a read-only Gitea fed by a Radicle → Gitea bridge | Radicle as the place to work was not user-friendly, and a read-only Gitea was confusing | +| v2 (`734a33f`) | Radicle | Radicle CI broker + act on the node (`ci/`, removed since) | Not user-friendly enough: everything went through `rad` | +| **v3** (this) | **Gitea** | **Gitea Actions** | — | + +v3 keeps what worked in both: v1's Gitea + Actions setup, and v2's staging registry inside the +CI's own Docker daemon (no tokens, no registry credentials). The bridge is v1's glue reversed and +rewritten in Go ([bridge/](../bridge/)). + +The objection that removed Gitea in v2 was "two places where state can originate". It does not apply +here: state originates in Gitea only. Radicle receives a copy that nobody edits; patches or issues +opened on the Radicle side are not read. ## Architecture ``` -developer: git push rad main v1.2.3 (tag signed by a repository delegate) +developer: git push origin main v1.2.3 (or: create a tag in Gitea's UI) │ ▼ -radicle-node ──node events──▶ rad-actions container - ├─ cib (radicle-ci-broker 0.32.1, unmodified): filter, queue, reports, job COBs - └─ rad-actions adapter (ci/adapter, ours): - · fetch the commit from the node's storage (depth 1) - · apply GitHub's `on:` filters - · run the workflow with act 0.2.89 (unmodified) - │ DOCKER_HOST = unix socket - ▼ - rad-actions-docker (dind, privileged, no TCP listener) - jobs run on its host network → localhost:5000 = - rad-actions-registry (registry:3, shares dind's network namespace) - ▲ polled every minute over ci-internal - ipcr-gateway (ipcrd): import → IPFS → publish under IPNS +gitea ─────────── system webhook (push/create/delete/repository) ──▶ ipcr-forge (bridge) + │ · git fetch from Gitea + │ Actions · rad init (first time) + ▼ · git push rad (signed by the node) +forge-runner (act_runner 0.6.1) ──unix socket──▶ forge-docker (dind) ▼ + jobs on the daemon's host network → localhost:5000 = radicle-node ──▶ Radicle network +forge-registry (registry:3, shares dind's network namespace) + ▲ polled every minute over ci-internal +ipcr-gateway (ipcrd): import → IPFS → publish under the forge's IPNS key │ ▼ -anyone: docker pull ipcr.localhost:4767/ipns/metadec.eth/:1.2.3 +anyone: docker pull ipcr.localhost:4767/ipns///:1.2.3 ``` -- **No mirror.** The build reads the commit where it lives. Results go back into the repository as - job COBs (`xyz.radworks.job`) signed by the node (see limitations: not for tags yet). -- **CI pages.** `cib` writes static HTML reports; the adapter writes one plain-text log per run. The - landing page serves both at `ipcr-forge-/ci/`. -- **Handoff to IPFS is a pull.** Jobs cannot reach `ipcr.localhost` (the host's loopback), so they - push to the staging registry and `ipcrd` imports from it. `ipcrd` needed no change for this. +The two legs do not depend on each other: a repository can be built without being mirrored +(private repositories), and mirrored without being built (no workflow). -## Trust model +## The bridge -- `cib` runs CI only for changes from a **delegate** of the repository (`!AnyDelegate`): release - tags, the default branch and patches. A build can publish an image under the forge's IPNS name, - so a build gets the trust of a release. Patches from non-delegates are never run. -- The node's key (which signs job COBs) never enters a job: jobs get a self-contained checkout. -- The adapter refuses private repositories, because run logs are public. -- To run outsiders' patches safely, they would need a separate daemon with no route to the staging - registry. Not built. +[bridge/](../bridge/): one Go binary, `ipcr-forge-bridge`, image `ghcr.io/yundera/ipcr-forge-bridge`. + +- **`setup`**, an install step (`post_up`, on every start, idempotent). It signs in once as + `gitea_admin` with the default app password and does four things: + - mints the daemon's token, `public-only` + `read:repository`; + - generates the webhook secret; + - creates or repairs one **system webhook** (`POST /api/v1/admin/hooks`, `is_system_webhook`), + matched by URL so it is never duplicated; + - creates the example `gitea_admin/ipcr-hello`. +- **`serve`**: + - `:8081` takes the webhook. It is internal, with no Caddy route. It checks the HMAC, queues the + repository ID and answers 202 at once, because Gitea gives a delivery 5 seconds. + - One worker, with a coalescing queue: a burst of deliveries is one sync. + - A **reconcile** every 10 minutes catches what no event announces. In Gitea 1.27 that is renames + and visibility changes, plus any lost delivery. + - `:8080` serves the forge's page (`/`, `/images.json`, `/repos.json`). It is the place for future + admin pages, such as Radicle settings. + +### Mirror semantics + +| Gitea | Radicle | +| --- | --- | +| public repository, first sight | `rad init --public` from a bare mirror; this node is the only delegate; then a `xyz.radicle.crefs` rule `refs/tags/*` (threshold 1) so tags are canonical | +| push, force push, branch delete | `git push --prune rad +refs/heads/*:… +refs/tags/*:…`, then `rad sync --announce` | +| tag created (git or web UI) | same; becomes canonical `refs/tags/` | +| tag deleted | removed from the node's namespace; **the canonical `refs/tags/` stays** (see findings) | +| default branch changed | `rad id update … defaultBranch` | +| rename / transfer | same RID (state is keyed by Gitea's repository ID) | +| made private, deleted | **frozen**: no further sync. Nothing is retracted; it cannot be | +| private, fork, pull mirror | never mirrored (forks and mirrors did not originate here) | +| empty | waits for its first push | + +The token cannot see a private repository at all (`public-only`), so a bug cannot publish one. The +code checks visibility again anyway. + +## Trust + +- **Radicle signatures mean "this forge published it"**, not "this developer wrote it". Radicle is a + distribution channel here, not the trust root. The `/rad//…` image-naming idea in + [naming.md](naming.md) is weaker for it. +- **Anyone who can push to a repository can publish images.** A job can push any name to the + unauthenticated `localhost:5000`, and IPCR publishes whatever it finds there under the forge's IPNS + key, including images built from private repositories. Registration is off, so every user was + created by the admin. +- **Fork pull requests** from users without write access wait for "Approve and run" in Gitea 1.27. +- The page, the tips and `rationale.md` say all of this plainly. +- **Follow-up:** per-repository push credentials, with the bridge acting as a registry token + server, would tie an image name to its repository. ## Workflow contract -A repository opts in by committing a workflow under `.github/workflows/` (`.gitea/` and -`.forgejo/` are also read). The template on the landing page: +A repository opts in by committing a workflow under `.gitea/workflows/`. The template is +[bridge/example/.gitea/workflows/build.yml](../bridge/example/.gitea/workflows/build.yml): -```yaml -on: - push: - tags: ['v*'] -jobs: - image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: docker/metadata-action@v5 - id: meta - with: - images: ${{ vars.IMAGE }} - tags: | - type=semver,pattern={{version}} - type=raw,value=latest - - uses: docker/setup-buildx-action@v3 - with: - driver-opts: network=host - - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} -``` - -- `vars.IMAGE` = `localhost:5000/`; `vars.REGISTRY` = `localhost:5000`. No secret. -- Two lines differ from a GitHub-hosted workflow: `context: .` and `driver-opts: network=host` - (both explained below). -- `github.repository` is `rad/`; `RADICLE_RID` is set as an env var and a var. +- `images: localhost:5000/${{ github.repository }}`: the staging registry. No login, no secret. + `docker/metadata-action` lowercases the name. +- `context: .` and `driver-opts: network=host`: see findings 2 and 3. +- The image is published as `/ipns///:`. ## Findings -Each of these silently produced the wrong result rather than an error. +Carried over from v1 and v2; each one silently produced the wrong result rather than an error. -### act +1. **Jobs must share the daemon's network namespace.** `localhost:5000` is the registry only on + forge-docker's own network, and Docker/BuildKit accept plain HTTP for `localhost` only. The runner + config sets `container.network: host`. As a consequence, compose service names do not resolve + inside jobs, so `GITEA_INSTANCE_URL` is Gitea's public address. +2. **`docker/build-push-action` defaults to a Git context** (`/.git#`), which needs + credentials here. Hence `context: .`. +3. **buildx's builder container** does not share the job's network. `driver-opts: network=host` puts + it on the daemon's, where `localhost:5000` is the registry. +4. **act_runner's `run.sh` reads the registration token once** and gives up after 10 tries. The token + is minted by an install step that runs after the stack is up, so the compose entrypoint waits for + it first. +5. **Radicle has no shared `refs/tags/`** unless the identity carries a canonical-refs rule. Without + one, a tag lives under its pusher's namespace, where `rad clone` and the explorer do not show it. + The bridge adds the rule right after `rad init`. -1. **act does not apply `on:` branch/tag filters.** A workflow with `on: push: tags: ['v*']` ran for - a push to `main`. GitHub and Gitea filter server-side; the adapter does it here - ([filter.go](../ci/adapter/filter.go), GitHub's pattern syntax, tested). -2. **Without a git remote, `github.repository` is `nektos/act`;** with a non-GitHub remote it is the - whole URL. The adapter passes `--env GITHUB_REPOSITORY=rad/`. -3. **`actions/checkout` is skipped** by act, which copies the working directory into the job. So the - checkout must be a self-contained repository. A `git clone --shared` of storage left a `.git` - whose alternates pointed outside the job (`unable to normalize alternate object path`). The - adapter now fetches the one commit, depth 1, with `uploadpack.allowAnySHA1InWant` on the serving - side — what `actions/checkout` does by default. -4. **Jobs run on the daemon's host network** (`--network host`, act's default). That is what makes - `localhost:5000` reach the staging registry. It is also why compose service names do not - resolve inside jobs (`getaddrinfo ENOTFOUND rad-actions`): the adapter advertises its own IP - address instead. +New in v3 (tested locally with Gitea 1.27.3 and rad 1.10.1, 2026-10-04): -### GitHub-isms in common actions +6. **`rad init` works on a bare repository** (it uses `workdir().unwrap_or(path)`), so the bridge + keeps bare mirrors only. It also seeds the repository on the node and adds the `rad` remote. +7. **A deleted tag stays canonical.** `git push --prune` removes it from the node's namespace, but + `refs/tags/` keeps the last target. Tags are meant to be immutable, so this was left alone. +8. **Gitea's default webhook allow-list (`external`) blocks the bridge.** It is on the server's + private network. `ALLOWED_HOST_LIST: external,ipcr-forge` allows exactly that one container. + `private` would let any user's webhook reach every container on the shared network. +9. **`rad sync --announce` takes its RID positionally** (`rad sync --announce`); there is no + `--rid` flag. +10. **`rad sync` blocks for its timeout when the node has no peers.** The bridge pushes with + `-o no-sync` and announces separately with a 30-second bound, so a push is never held up by the + network. -5. **`docker/metadata-action` calls the GitHub REST API** (`GET /repos/{owner}/{repo}`) and fails - with `Bad credentials` without it. The adapter serves that one call for the length of a run, from - what the node knows, and sets `GITHUB_API_URL` to it ([api.go](../ci/adapter/api.go)). `html_url` - is the Radicle explorer page, so the image's `org.opencontainers.image.source` label points there. -6. **`docker/build-push-action` defaults to a Git context** (`https://github.com/.git#`), - which does not exist here. Hence `context: .`. -7. **buildx's builder container** does not share the job's network; `driver-opts: network=host` - puts it on the daemon's, where `localhost:5000` is the registry. Docker and BuildKit accept - plain HTTP for `localhost` only, which is why the registry shares the daemon's namespace. +## Verified -### radicle-ci-broker (cib) 0.32.1 +Local integration (Gitea 1.27.3-rootless, `radicle-seed-node` 1.10.1, the bridge image, one Docker +network): -8. **Tags arrive as pushes.** A tag event reaches the adapter as `event_type: "push"` with the tag's - name in `branch` (src/msg.rs), so the first tag build ran as if `v1.1.1` were a branch. The - adapter looks the name up in storage (`refs/namespaces/*/refs/tags/`). -9. **`!HasFile` matches files only,** never directories (it checks `into_blob()`), despite its doc - comment. A filter on `.github/workflows` therefore never matched and nothing ran. It was dropped; - the adapter answers "no workflow" in about a second. -10. **No job COB for tag builds.** The job is keyed by the event's tip, which for an annotated tag is - the tag object; creating it fails silently and the run ends with `NoJob`. Branch builds get their - COB. Upstream fix: peel the tag before creating the job. -11. **Default log level is trace:** every gossip message the node sees. The image runs - `--log-level info`. -12. **cib must not start before the node's control socket exists.** It restart-loops a few times on - first install, then settles (`restart: unless-stopped`). +- `setup` run twice leaves exactly one system hook. +- `ipcr-hello` was created and mirrored with the crefs rule in its identity. +- A tag created through Gitea's API was canonical on Radicle within seconds, through the webhook. +- A private repository never appeared. +- A public repository made private was frozen. -### Radicle - -13. **The node only reacts to what it receives** from other nodes: pushes come from the developer's - own node. The forge node still needs `rad seed --scope all` per repository (seeding policy - `block`). -14. **Commit and tag signing need an ssh-agent** holding the Radicle key. - -## Verified (wisera, 2026-10-04) - -`ipcr-hello` (`rad:zc9XLrmUrt2xSgTLcEYfKN8eZuxS`), tag `v1.1.2` pushed from a delegate's node: - -| Step | Time | -| --- | --- | -| tag pushed | 12:54:21 | -| build finished (first run on the box: act's image pulled cold) | 13:01:06 | -| imported into IPFS and published under IPNS | ~13:02:30 | -| pulled on wisera, and on holyhorse (26 s) | after | - -Later, with `metadec.eth` pointing at the forge's publisher name (see [naming.md](naming.md)), -`ipcr.localhost:4767/ipns/metadec.eth/ipcr-hello:1.1.2` resolved and pulled on wisera. +The full chain on a PCS (runner → staging → IPCR → pull from a second server) is in +[Open items](#open-items) until it is run on a test box. ## Open items -- **Publish the images.** `ghcr.io/yundera/rad-actions` (workflow `rad-actions.yml`, tag - `rad-actions-v*`) and `ghcr.io/yundera/ipcr:1.2.0` are so far built locally only. -- **Job COBs for tags** (finding 10): an upstream patch to cib. -- **Patches from non-delegates:** a second, registry-less daemon. -- **Warm cache.** The first build pulls `catthehacker/ubuntu:act-latest` (~1.5 GB) and clones the - actions it uses; later builds reuse both (`ci/docker`, `ci/cache`). -- **Path filters** (`on.push.paths`) are not applied: a workflow with only `paths:` runs on every - matching event. -- **Store install.** Tested as a hand install that mimics Maison (compose, `.env`, `.seed/`), not yet - through the store. +- **Run the full chain on a test PCS.** Check that the runner registers, that jobs see the daemon + (`docker_host: ""` mounts its socket into each job), that a `v*` tag on `ipcr-hello` builds and + imports, and that the image pulls from a second server. +- **Publish the images**: `ghcr.io/yundera/ipcr:1.2.0` and `ghcr.io/yundera/ipcr-forge-bridge:0.1.0` + (workflows `image.yml` and `bridge.yml`, tag `bridge-v0.1.0`). +- **Per-repository image names:** the bridge as a registry token server (see Trust). +- **act_runner** is pinned at 0.6.1, proven with Gitea 1.27.3. Upstream has moved on. +- **Radicle → Gitea:** patches opened on Radicle could become pull requests. Not built. +- **Store install:** test through the store, not only as a hand install. diff --git a/forge/Dockerfile b/forge/Dockerfile new file mode 100644 index 0000000..a957fb2 --- /dev/null +++ b/forge/Dockerfile @@ -0,0 +1,37 @@ +# ipcr-forge-bridge: the Gitea → Radicle mirror and IPCR Forge's page. Our Go binary, plus git and +# the stock Radicle CLI (`rad`, `git-remote-rad`) at the version the forge's node runs. +# Build context: this folder (bridge/). +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/ipcr-forge-bridge . + +# Radicle's official static (musl) release, checked against its published digest. +FROM alpine:3.22 AS radicle +ARG RADICLE_VERSION=1.10.1 +ARG RADICLE_SHA256=9354f7f6241a88ddca1d41a080aa6337ddfc5dc682b8de78ab6d418b5782225a +# Retried: release downloads intermittently fail. +RUN apk add --no-cache xz \ + && url="https://files.radicle.dev/releases/${RADICLE_VERSION}/radicle-${RADICLE_VERSION}-x86_64-unknown-linux-musl.tar.xz" \ + && for i in 1 2 3 4 5; do wget -qO /tmp/radicle.tar.xz "$url" && break; sleep $((i * 5)); done \ + && echo "${RADICLE_SHA256} /tmp/radicle.tar.xz" | sha256sum -c - \ + && mkdir /out && tar -xJf /tmp/radicle.tar.xz -C /out --strip-components=2 \ + "radicle-${RADICLE_VERSION}-x86_64-unknown-linux-musl/bin/rad" \ + "radicle-${RADICLE_VERSION}-x86_64-unknown-linux-musl/bin/git-remote-rad" + +FROM alpine:3.22 +ARG VERSION=dev +LABEL org.opencontainers.image.title="ipcr-forge-bridge" \ + org.opencontainers.image.description="IPCR Forge: mirrors public Gitea repositories to Radicle, serves the forge's page" \ + org.opencontainers.image.source="https://github.com/Yundera/ipcr" \ + org.opencontainers.image.version="$VERSION" +RUN apk add --no-cache ca-certificates git \ + # Mirrors and the node's storage belong to the service's uid, but a bind mount can make git see + # another owner ("dubious ownership"); the storage is the node's own, so trust it. + && git config --system safe.directory '*' +COPY --from=radicle /out/ /usr/local/bin/ +COPY --from=build /out/ipcr-forge-bridge /usr/local/bin/ipcr-forge-bridge +ENV RAD_HOME=/radicle-home \ + HOME=/tmp +# No entrypoint: the install step runs `sh -c '… ipcr-forge-bridge setup'` in this image. +CMD ["ipcr-forge-bridge", "serve"] diff --git a/forge/bridge_test.go b/forge/bridge_test.go new file mode 100644 index 0000000..7e21c02 --- /dev/null +++ b/forge/bridge_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func sign(secret, body string) string { + m := hmac.New(sha256.New, []byte(secret)) + m.Write([]byte(body)) + return hex.EncodeToString(m.Sum(nil)) +} + +func TestValidSignature(t *testing.T) { + body := `{"repository":{"id":7}}` + if !validSignature([]byte("s3cret"), []byte(body), sign("s3cret", body)) { + t.Error("good signature rejected") + } + for _, sig := range []string{"", "zz", sign("other", body), sign("s3cret", body+" ")} { + if validSignature([]byte("s3cret"), []byte(body), sig) { + t.Errorf("bad signature %q accepted", sig) + } + } +} + +func TestHook(t *testing.T) { + dir := t.TempDir() + secretFile := filepath.Join(dir, "hook-secret") + var got []int64 + h := &hookHandler{secretFile: secretFile, enqueue: func(id int64) { got = append(got, id) }} + post := func(body, sig string) int { + r := httptest.NewRequest("POST", "/hooks/gitea", strings.NewReader(body)) + r.Header.Set("X-Gitea-Signature", sig) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w.Code + } + body := `{"ref":"refs/tags/v1.0.0","repository":{"id":42,"full_name":"a/b"}}` + if c := post(body, sign("k", body)); c != http.StatusServiceUnavailable { + t.Errorf("no secret yet: %d", c) + } + os.WriteFile(secretFile, []byte("k\n"), 0o600) + if c := post(body, sign("x", body)); c != http.StatusUnauthorized { + t.Errorf("bad signature: %d", c) + } + if c := post(body, sign("k", body)); c != http.StatusAccepted { + t.Errorf("good delivery: %d", c) + } + if c := post(`{"sender":{}}`, sign("k", `{"sender":{}}`)); c != http.StatusNoContent { + t.Errorf("no repository: %d", c) + } + if len(got) != 1 || got[0] != 42 { + t.Errorf("enqueued %v, want [42]", got) + } +} + +func TestQueueCoalesces(t *testing.T) { + q := newQueue() + for _, id := range []int64{3, 1, 3, 3, 2, 1} { + q.add(id) + } + var order []int64 + for { + id, ok := q.next() + if !ok { + break + } + order = append(order, id) + } + if len(order) != 3 || order[0] != 3 || order[1] != 1 || order[2] != 2 { + t.Errorf("order %v, want [3 1 2]", order) + } + q.add(3) // popped, so it may be queued again + if id, ok := q.next(); !ok || id != 3 { + t.Error("re-adding a popped repository") + } +} + +func TestPublic(t *testing.T) { + r := repo{} + if !r.public() { + t.Error("plain repository") + } + for _, f := range []func(*repo){ + func(r *repo) { r.Private = true }, + func(r *repo) { r.Internal = true }, + func(r *repo) { r.Owner.Visibility = "limited" }, + func(r *repo) { r.Owner.Visibility = "private" }, + } { + r := repo{} + f(&r) + if r.public() { + t.Errorf("%+v counted as public", r) + } + } +} + +func TestRIDFromURL(t *testing.T) { + for in, want := range map[string]string{ + "rad://z2pnYhouQzbiTEmXW9bydgXFqjP1o\n": "rad:z2pnYhouQzbiTEmXW9bydgXFqjP1o", + "rad://z2pnYhouQzbiTEmXW9bydgXFqjP1o/z6MkppQoVh2QKyQGkPzGgEaqZcXmbeBVhZseQ2cgxuMzA5gR": "rad:z2pnYhouQzbiTEmXW9bydgXFqjP1o", + "https://example.com/x.git": "", + "": "", + } { + if got := ridFromURL(in); got != want { + t.Errorf("ridFromURL(%q) = %q, want %q", in, got, want) + } + } + if got := ridRe.FindString("Your Repository ID (RID) is rad:z2pnYhouQzbiTEmXW9bydgXFqjP1o\nYou can"); got != "rad:z2pnYhouQzbiTEmXW9bydgXFqjP1o" { + t.Errorf("ridRe: %q", got) + } +} + +func TestPushedRefs(t *testing.T) { + out := "To rad://z2pn/z6Mk\n" + + "=\trefs/heads/main:refs/heads/main\t[up to date]\n" + + "*\trefs/tags/v1.0.0:refs/tags/v1.0.0\t[new tag]\n" + + "-\t:refs/heads/old\t[deleted]\n" + + "+\trefs/heads/dev:refs/heads/dev\t1111111...2222222 (forced update)\n" + + "Done\n" + got := strings.Join(pushedRefs(out), "; ") + want := "tags/v1.0.0 [new tag]; old [deleted]; dev 1111111...2222222 (forced update)" + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } +} + +func TestStateRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "repos.json") + s, err := loadState(path) + if err != nil { + t.Fatal(err) + } + if s.known(5) || s.get(5).ID != 5 { + t.Fatal("empty state") + } + s.put(repoState{ID: 5, FullName: "b/two", RID: "rad:z2", State: stSynced, Refs: "abc", Crefs: true}) + s.put(repoState{ID: 9, FullName: "a/one", State: stWaiting}) + s2, err := loadState(path) + if err != nil { + t.Fatal(err) + } + if r := s2.get(5); r.RID != "rad:z2" || r.Refs != "abc" || !r.Crefs || r.State != stSynced { + t.Errorf("reloaded %+v", r) + } + l := s2.list() + if len(l) != 2 || l[0].FullName != "a/one" || l[1].Refs != "" { + t.Errorf("list %+v", l) + } + if ids := s2.ids(); len(ids) != 2 || ids[0] != 5 || ids[1] != 9 { + t.Errorf("ids %v", ids) + } +} + +func TestExampleEmbedded(t *testing.T) { + for _, p := range []string{"example/Dockerfile", "example/.gitea/workflows/build.yml", "example/README.md"} { + if _, err := exampleFiles.ReadFile(p); err != nil { + t.Errorf("%s: %v", p, err) + } + } + b, _ := exampleFiles.ReadFile("example/.gitea/workflows/build.yml") + if !strings.Contains(string(b), "localhost:5000/${{ github.repository }}") || !strings.Contains(string(b), "network=host") { + t.Error("example workflow lost its staging registry lines") + } +} + +func TestWeb(t *testing.T) { + dir := t.TempDir() + st, _ := loadState(filepath.Join(dir, "repos.json")) + h := webHandler(st, filepath.Join(dir, "published.json")) + get := func(p string) (int, string) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest("GET", p, nil)) + return w.Code, w.Body.String() + } + if c, b := get("/"); c != 200 || !strings.Contains(b, "IPCR Forge") { + t.Errorf("/: %d", c) + } + if c, _ := get("/images.json"); c != 404 { + t.Errorf("/images.json before anything is published: %d", c) + } + os.WriteFile(filepath.Join(dir, "published.json"), []byte(`{"images":{}}`), 0o644) + if c, b := get("/images.json"); c != 200 || b != `{"images":{}}` { + t.Errorf("/images.json: %d %q", c, b) + } + if c, b := get("/repos.json"); c != 200 || strings.TrimSpace(b) != "[]" { + t.Errorf("/repos.json: %d %q", c, b) + } + if c, _ := get("/nothing"); c != 404 { + t.Errorf("/nothing: %d", c) + } +} diff --git a/forge/example/.gitea/workflows/build.yml b/forge/example/.gitea/workflows/build.yml new file mode 100644 index 0000000..0716071 --- /dev/null +++ b/forge/example/.gitea/workflows/build.yml @@ -0,0 +1,32 @@ +# Builds the Dockerfile at the root for every v* tag and pushes it to this forge's staging registry. +# IPCR picks it up from there within a minute and publishes it on IPFS; the forge's page then +# lists the exact `image:` line to use. +# +# Differences from a GitHub-hosted workflow, both needed on this forge: +# images: localhost:5000/… the staging registry (no login, no secret) +# network=host so buildx's builder can reach that registry +name: image +on: + push: + tags: ['v*'] +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/metadata-action@v5 + id: meta + with: + images: localhost:5000/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=raw,value=latest + - uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/forge/example/Dockerfile b/forge/example/Dockerfile new file mode 100644 index 0000000..312c58e --- /dev/null +++ b/forge/example/Dockerfile @@ -0,0 +1,6 @@ +# A one-page web server: busybox httpd on port 8080, as an unprivileged user. +FROM busybox:1.37 +COPY index.html /www/index.html +USER 1000:1000 +EXPOSE 8080 +CMD ["httpd", "-f", "-p", "8080", "-h", "/www"] diff --git a/forge/example/README.md b/forge/example/README.md new file mode 100644 index 0000000..7e27509 --- /dev/null +++ b/forge/example/README.md @@ -0,0 +1,34 @@ +# ipcr-hello + +The IPCR Forge example. A tag on this repository builds its image, IPCR publishes it on IPFS, +and the repository is mirrored to Radicle. + +## Try it + +1. **Tag a release.** In Gitea: *Tags → New tag* (or *Releases → New release*), name it `v1.0.0` + and target `main`. From a clone it works too: + + ```sh + git tag v1.0.0 && git push origin v1.0.0 + ``` + +2. **Watch the build** under *Actions*. The first build on a new server takes a few minutes more, + while it downloads its build images. + +3. **Find the image** on the forge's page (`ipcr-forge-`, *Published images*), about + a minute after the build finishes. Then on this server or on any other running IPCR: + + ```yaml + services: + hello: + image: ipcr.localhost:4767/ipns//gitea_admin/ipcr-hello:1.0.0 + ports: ["8080:8080"] + ``` + +## What makes it work + +- [`.gitea/workflows/build.yml`](.gitea/workflows/build.yml) pushes to `localhost:5000`, the + forge's staging registry. IPCR imports every tag it finds there. +- [`Dockerfile`](Dockerfile) is an ordinary Dockerfile. Anything that builds works. + +Copy the workflow into any public or private repository on this forge to publish it the same way. diff --git a/forge/example/index.html b/forge/example/index.html new file mode 100644 index 0000000..77137dc --- /dev/null +++ b/forge/example/index.html @@ -0,0 +1,9 @@ + + +ipcr-hello + +

Hello from IPFS

+

This page is served by a container whose image was built by IPCR Forge and pulled from IPFS +through IPCR.

+ + diff --git a/forge/gitea.go b/forge/gitea.go new file mode 100644 index 0000000..c13ba61 --- /dev/null +++ b/forge/gitea.go @@ -0,0 +1,143 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// gitea is a minimal Gitea API client. The daemon authenticates with the token in tokenFile (read +// on every call: the setup step may write it after the service starts); the setup step with the +// admin's password instead (Gitea's token API accepts nothing else). +type gitea struct { + base string + owner string + tokenFile string + password string // setup only + http *http.Client +} + +func newGitea(base, owner, tokenFile string) *gitea { + return &gitea{base: strings.TrimRight(base, "/"), owner: owner, tokenFile: tokenFile, + http: &http.Client{Timeout: 30 * time.Second}} +} + +func (g *gitea) token() string { + b, _ := os.ReadFile(g.tokenFile) + return strings.TrimSpace(string(b)) +} + +func (g *gitea) hasToken() bool { return g.token() != "" } + +// gitAuth is the header git sends to Gitea's HTTP endpoint: basic auth with the token as password, +// passed with http.extraHeader so it never appears in a URL, `ps` or a log. +func (g *gitea) gitAuth() string { + return "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(g.owner+":"+g.token())) +} + +// call sends a request and decodes a JSON answer into out (when non-nil and the status is 2xx). +// It returns the status; a transport error is returned as an error with status 0. +func (g *gitea) call(method, path string, body, out any) (int, error) { + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return 0, err + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, g.base+"/api/v1"+path, rd) + if err != nil { + return 0, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + if g.password != "" { + req.SetBasicAuth(g.owner, g.password) + } else { + req.Header.Set("Authorization", "token "+g.token()) + } + resp, err := g.http.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if resp.StatusCode/100 != 2 { + msg := strings.TrimSpace(string(data)) + if len(msg) > 300 { + msg = msg[:300] + } + return resp.StatusCode, fmt.Errorf("%s %s: %s %s", method, path, resp.Status, msg) + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return resp.StatusCode, fmt.Errorf("%s %s: %w", method, path, err) + } + } + return resp.StatusCode, nil +} + +type repo struct { + ID int64 `json:"id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Private bool `json:"private"` + Internal bool `json:"internal"` + Fork bool `json:"fork"` + Mirror bool `json:"mirror"` + Empty bool `json:"empty"` + DefaultBranch string `json:"default_branch"` + HTMLURL string `json:"html_url"` + Owner struct { + Login string `json:"login"` + // public | limited | private. Only a public owner's repositories are public. + Visibility string `json:"visibility"` + } `json:"owner"` +} + +// public reports whether the repository may leave this server. The token is `public-only`, so +// Gitea already hides everything else; this is the second lock on the same door. +func (r *repo) public() bool { + return !r.Private && !r.Internal && (r.Owner.Visibility == "" || r.Owner.Visibility == "public") +} + +// publicRepos lists every repository the token can see. +func (g *gitea) publicRepos() ([]repo, error) { + var all []repo + for page := 1; ; page++ { + var res struct { + OK bool `json:"ok"` + Data []repo `json:"data"` + } + q := url.Values{"limit": {"50"}, "page": {fmt.Sprint(page)}} + if _, err := g.call("GET", "/repos/search?"+q.Encode(), nil, &res); err != nil { + return nil, err + } + all = append(all, res.Data...) + if len(res.Data) < 50 { + return all, nil + } + } +} + +// repoByID returns the repository, or nil when the token cannot see it (private, deleted). +func (g *gitea) repoByID(id int64) (*repo, error) { + var r repo + code, err := g.call("GET", fmt.Sprintf("/repositories/%d", id), nil, &r) + if code == http.StatusNotFound || code == http.StatusForbidden { + return nil, nil + } + if err != nil { + return nil, err + } + return &r, nil +} diff --git a/forge/go.mod b/forge/go.mod new file mode 100644 index 0000000..223d697 --- /dev/null +++ b/forge/go.mod @@ -0,0 +1,3 @@ +module ipcr-forge-bridge + +go 1.24 diff --git a/forge/hook.go b/forge/hook.go new file mode 100644 index 0000000..3fbda53 --- /dev/null +++ b/forge/hook.go @@ -0,0 +1,63 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "log" + "net/http" + "os" + "strings" +) + +// hookHandler receives Gitea's system webhook. It only learns *which* repository changed: the sync +// itself re-reads everything from Gitea, so a payload is never trusted for content, and a burst of +// deliveries for one repository collapses into one sync. +type hookHandler struct { + secretFile string + enqueue func(id int64) +} + +func (h *hookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + secret, _ := os.ReadFile(h.secretFile) + if len(strings.TrimSpace(string(secret))) == 0 { + http.Error(w, "not configured yet", http.StatusServiceUnavailable) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 32<<20)) + if err != nil { + http.Error(w, "read error", http.StatusBadRequest) + return + } + if !validSignature([]byte(strings.TrimSpace(string(secret))), body, r.Header.Get("X-Gitea-Signature")) { + log.Printf("hook: bad signature from %s (event %q)", r.RemoteAddr, r.Header.Get("X-Gitea-Event")) + http.Error(w, "bad signature", http.StatusUnauthorized) + return + } + var p struct { + Repository *struct { + ID int64 `json:"id"` + } `json:"repository"` + } + if err := json.Unmarshal(body, &p); err != nil || p.Repository == nil || p.Repository.ID == 0 { + // Signed, but about no repository (a user or organisation event): nothing to do. + w.WriteHeader(http.StatusNoContent) + return + } + h.enqueue(p.Repository.ID) + // Answered at once: Gitea gives a delivery 5 seconds, a sync can take minutes. + w.WriteHeader(http.StatusAccepted) +} + +// validSignature checks Gitea's X-Gitea-Signature: hex HMAC-SHA256 of the raw body, no prefix. +func validSignature(secret, body []byte, sig string) bool { + got, err := hex.DecodeString(strings.TrimSpace(sig)) + if err != nil || len(got) == 0 { + return false + } + mac := hmac.New(sha256.New, secret) + mac.Write(body) + return hmac.Equal(got, mac.Sum(nil)) +} diff --git a/forge/main.go b/forge/main.go new file mode 100644 index 0000000..9bfb1b9 --- /dev/null +++ b/forge/main.go @@ -0,0 +1,149 @@ +// ipcr-forge-bridge — the glue of IPCR Forge: Gitea is where people work, Radicle is where the code +// is replicated, and this service keeps the second following the first. One way only. +// +// serve (default) webhook receiver + mirror worker + the forge's page +// setup install step: Gitea token, system webhook, example repository (idempotent) +// health exit 0 when `serve` answers, for the container healthcheck +// +// What gets mirrored: every public, non-fork, non-mirror Gitea repository, as seen by a token +// scoped `public-only` — a private repository is invisible to the service, not filtered by it. +// Each one is `rad init`ed by this node on first sight (the node's key is its only delegate) and +// then receives its branches and tags with `git push rad`. A repository that turns private or is +// deleted is frozen: no further sync, and nothing is taken back from the network (it cannot be). +// +// Triggers: Gitea's system webhook (push, create, delete, repository) for latency, and a full +// reconcile every RECONCILE for whatever a lost delivery, a rename or a visibility change missed. +// +// Environment: +// +// GITEA_URL Gitea, internal address (default http://gitea:3000) +// GITEA_OWNER the admin account (default gitea_admin) +// GITEA_PASSWORD setup only: its password, used once to mint the token and the hook +// SECRETS_DIR gitea-token and hook-secret (default /secrets) +// STATE_DIR repos.json and the bare mirrors (default /bridge) +// RAD_HOME the node's Radicle home (default /radicle-home) +// WEB_LISTEN the page (default :8080) +// HOOK_LISTEN the webhook, internal only (default :8081) +// HOOK_URL setup only: how Gitea reaches HOOK_LISTEN (default http://ipcr-forge:8081/hooks/gitea) +// EXAMPLE_REPO setup only: example repository to create under GITEA_OWNER; empty = none +// (default ipcr-hello) +// IPCR_PUBLISHED IPCR's published.json, served as /images.json +// (default /srv/ipcr-state/published.json) +// RECONCILE full pass interval (default 10m) +package main + +import ( + "fmt" + "log" + "net/http" + "os" + "time" +) + +func env(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func main() { + log.SetFlags(log.LstdFlags | log.Lmsgprefix) + cmd := "serve" + if len(os.Args) > 1 { + cmd = os.Args[1] + } + var err error + switch cmd { + case "serve": + err = serve() + case "setup": + err = setup() + case "health": + err = health() + default: + err = fmt.Errorf("usage: %s serve | setup | health", os.Args[0]) + } + if err != nil { + log.Fatal(err) + } +} + +func health() error { + c := &http.Client{Timeout: 5 * time.Second} + r, err := c.Get("http://127.0.0.1" + env("WEB_LISTEN", ":8080") + "/healthz") + if err != nil { + return err + } + r.Body.Close() + if r.StatusCode != http.StatusOK { + return fmt.Errorf("healthz: %s", r.Status) + } + return nil +} + +func serve() error { + every, err := time.ParseDuration(env("RECONCILE", "10m")) + if err != nil { + return fmt.Errorf("RECONCILE: %w", err) + } + stateDir := env("STATE_DIR", "/bridge") + st, err := loadState(stateDir + "/repos.json") + if err != nil { + return err + } + secrets := env("SECRETS_DIR", "/secrets") + m := &mirror{ + gitea: newGitea(env("GITEA_URL", "http://gitea:3000"), env("GITEA_OWNER", "gitea_admin"), secrets+"/gitea-token"), + state: st, + repoDir: stateDir + "/repos", + radHome: env("RAD_HOME", "/radicle-home"), + } + if err := os.MkdirAll(m.repoDir, 0o755); err != nil { + return err + } + q := newQueue() + go q.run(m.sync) + go reconcileLoop(m, q, every) + + hook := &hookHandler{secretFile: secrets + "/hook-secret", enqueue: q.add} + hmux := http.NewServeMux() + hmux.Handle("POST /hooks/gitea", hook) + go func() { + log.Fatal(http.ListenAndServe(env("HOOK_LISTEN", ":8081"), hmux)) + }() + + log.Printf("bridge: %s → Radicle (%s), reconcile every %s", m.gitea.base, m.radHome, every) + return http.ListenAndServe(env("WEB_LISTEN", ":8080"), webHandler(st, env("IPCR_PUBLISHED", "/srv/ipcr-state/published.json"))) +} + +// reconcileLoop queues every repository Gitea shows as public, plus every one already known (so one +// that turned private or was deleted is noticed and frozen). The first pass runs at start-up. +func reconcileLoop(m *mirror, q *queue, every time.Duration) { + for { + if !m.gitea.hasToken() { + // The setup step runs after the stack is up, so on a first install this is normal. + log.Printf("reconcile: waiting for the Gitea token (written by the setup step)") + time.Sleep(15 * time.Second) + continue + } + repos, err := m.gitea.publicRepos() + if err != nil { + // Gitea restarting, typically: try again soon rather than in a full interval. + log.Printf("reconcile: %v", err) + time.Sleep(time.Minute) + continue + } + seen := map[int64]bool{} + for _, r := range repos { + seen[r.ID] = true + q.add(r.ID) + } + for _, id := range m.state.ids() { + if !seen[id] { + q.add(id) + } + } + time.Sleep(every) + } +} diff --git a/forge/mirror.go b/forge/mirror.go new file mode 100644 index 0000000..1724595 --- /dev/null +++ b/forge/mirror.go @@ -0,0 +1,170 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" +) + +// What is copied, both ways of the bare mirror: branches and tags. Not refs/pull/* (Gitea's PR +// heads), not notes or anything else. Branches and tags are forced: Gitea is the truth, and a +// force push or a moved tag there is mirrored as such. --prune makes deletions follow too. +var refspecs = []string{"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"} + +type mirror struct { + gitea *gitea + state *state + repoDir string // one bare repository per Gitea repository: .git + radHome string +} + +// sync brings one Gitea repository's Radicle copy up to date, and records the outcome. +func (m *mirror) sync(id int64) { + prev := m.state.get(id) + r, err := m.gitea.repoByID(id) + if err != nil { + log.Printf("sync %d: %v", id, err) + return + } + if r == nil || !r.public() { + // Private now, or deleted. Only a repository that was mirrored has anything to record. + if m.state.known(id) && prev.State != stFrozen { + prev.State = stFrozen + prev.Error = "no longer public in Gitea: not synced any more (what was published stays on the Radicle network)" + m.save(prev) + log.Printf("sync %s: frozen, no longer public", prev.FullName) + } + return + } + if r.Fork || r.Mirror { + // Did not originate here: a fork's upstream or a mirror's source is the one to publish. + return + } + cur := prev + cur.FullName, cur.HTMLURL = r.FullName, r.HTMLURL + if r.Empty { + if cur.RID == "" { + cur.State, cur.Error = stWaiting, "" + m.save(cur) + } + return + } + if err := m.publish(r, &cur); err != nil { + cur.State, cur.Error = stError, err.Error() + log.Printf("sync %s: %v", r.FullName, err) + } else { + cur.State, cur.Error, cur.LastSync = stSynced, "", time.Now().UTC() + } + m.save(cur) +} + +func (m *mirror) save(r repoState) { + if err := m.state.put(r); err != nil { + log.Printf("state: %v", err) + } +} + +// publish fetches the repository from Gitea into its bare mirror, creates its Radicle repository on +// first sight, and pushes what changed. +func (m *mirror) publish(r *repo, cur *repoState) error { + dir := filepath.Join(m.repoDir, fmt.Sprintf("%d.git", r.ID)) + if _, err := os.Stat(dir); err != nil { + if _, err := m.git("", "init", "--quiet", "--bare", dir); err != nil { + return err + } + } + src := m.gitea.base + "/" + r.FullName + ".git" + args := append([]string{"-c", "http.extraHeader=" + m.gitea.gitAuth(), "fetch", "--quiet", "--prune", "--no-tags", src}, refspecs...) + if _, err := m.git(dir, args...); err != nil { + return fmt.Errorf("fetch from Gitea: %w", err) + } + // rad init and the project's default branch read HEAD. + if _, err := m.git(dir, "symbolic-ref", "HEAD", "refs/heads/"+r.DefaultBranch); err != nil { + return err + } + digest, err := m.refsDigest(dir) + if err != nil { + return err + } + + if cur.RID == "" { + // A previous attempt may have created it and failed afterwards: rad init leaves the remote. + if rid := m.remoteRID(dir); rid != "" { + cur.RID = rid + } else { + rid, err := m.radInit(dir, r) + if err != nil { + return fmt.Errorf("rad init: %w", err) + } + cur.RID = rid + log.Printf("sync %s: created %s", r.FullName, rid) + } + cur.DefaultBranch = r.DefaultBranch + m.save(*cur) // before anything else can fail: never init the same repository twice + } else if m.remoteRID(dir) == "" { + // The mirror was deleted (it is disposable): point a fresh one at the existing repository. + if err := m.addRemote(dir, cur.RID); err != nil { + return err + } + } + if !cur.Crefs { + if err := m.radCanonicalTags(cur.RID); err != nil { + log.Printf("sync %s: canonical tags rule: %v (tags stay under the node's namespace)", r.FullName, err) + } else { + cur.Crefs = true + } + } + + if digest == cur.Refs && cur.State == stSynced && cur.DefaultBranch == r.DefaultBranch { + return nil + } + out, err := m.git(dir, append([]string{"push", "--prune", "--porcelain", "-o", "no-sync", "rad"}, refspecs...)...) + if err != nil { + return fmt.Errorf("push to Radicle: %w", err) + } + if changed := pushedRefs(out); len(changed) > 0 { + log.Printf("sync %s → %s: %s", r.FullName, cur.RID, strings.Join(changed, ", ")) + } + if cur.DefaultBranch != r.DefaultBranch { + if err := m.radDefaultBranch(cur.RID, r.DefaultBranch); err != nil { + return fmt.Errorf("default branch: %w", err) + } + cur.DefaultBranch = r.DefaultBranch + } + cur.Refs = digest + m.announce(cur.RID) + return nil +} + +// refsDigest summarises the mirror's branches and tags; equal digests mean nothing to push. +func (m *mirror) refsDigest(dir string) (string, error) { + out, err := m.git(dir, "for-each-ref", "--format=%(objectname) %(refname)", "refs/heads", "refs/tags") + if err != nil { + return "", err + } + sum := sha256.Sum256([]byte(out)) + return hex.EncodeToString(sum[:]), nil +} + +// pushedRefs reads `git push --porcelain` output and returns the refs that changed, for the log. +// Porcelain lines are "\t:\t"; '=' is up to date. +func pushedRefs(out string) []string { + var changed []string + for _, line := range strings.Split(out, "\n") { + f := strings.Split(line, "\t") + if len(f) < 3 || len(f[0]) != 1 || f[0] == "=" { + continue + } + to := f[1] + if i := strings.LastIndexByte(to, ':'); i >= 0 { + to = to[i+1:] + } + changed = append(changed, strings.TrimPrefix(strings.TrimPrefix(to, "refs/heads/"), "refs/")+" "+f[2]) + } + return changed +} diff --git a/forge/queue.go b/forge/queue.go new file mode 100644 index 0000000..155a5cb --- /dev/null +++ b/forge/queue.go @@ -0,0 +1,54 @@ +package main + +import "sync" + +// queue holds the repositories waiting for a sync, each at most once, served by one worker: a +// burst of webhooks for one repository costs one sync, and two syncs never race on a mirror or on +// the node's storage. +type queue struct { + mu sync.Mutex + order []int64 + pending map[int64]bool + wake chan struct{} +} + +func newQueue() *queue { + return &queue{pending: map[int64]bool{}, wake: make(chan struct{}, 1)} +} + +func (q *queue) add(id int64) { + q.mu.Lock() + if !q.pending[id] { + q.pending[id] = true + q.order = append(q.order, id) + } + q.mu.Unlock() + select { + case q.wake <- struct{}{}: + default: + } +} + +// next pops the oldest repository, or reports false when the queue is empty. +func (q *queue) next() (int64, bool) { + q.mu.Lock() + defer q.mu.Unlock() + if len(q.order) == 0 { + return 0, false + } + id := q.order[0] + q.order = q.order[1:] + delete(q.pending, id) + return id, true +} + +func (q *queue) run(work func(id int64)) { + for { + id, ok := q.next() + if !ok { + <-q.wake + continue + } + work(id) + } +} diff --git a/forge/rad.go b/forge/rad.go new file mode 100644 index 0000000..a58b16e --- /dev/null +++ b/forge/rad.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "os/exec" + "regexp" + "strings" + "time" +) + +// Everything Radicle is done with the stock `rad` CLI and `git-remote-rad`, against the node's own +// home: the repository is created and signed by the node's key (its only delegate), written into +// the node's storage, and announced through the node's control socket. The key has no passphrase +// (the install step creates it that way), so nothing prompts. + +var ridRe = regexp.MustCompile(`rad:z[1-9A-HJ-NP-Za-km-z]+`) + +func (m *mirror) env() []string { + return append(os.Environ(), + "RAD_HOME="+m.radHome, + "GIT_TERMINAL_PROMPT=0", + // rad's output is read by people in the container log, not by a terminal. + "NO_COLOR=1", + // Identity documents are git commits; git wants a name even when rad signs them. + "GIT_AUTHOR_NAME=ipcr-forge", "GIT_AUTHOR_EMAIL=ipcr-forge@localhost", + "GIT_COMMITTER_NAME=ipcr-forge", "GIT_COMMITTER_EMAIL=ipcr-forge@localhost", + ) +} + +// run executes a command in dir (when set) and returns its stdout; on failure the error carries +// the end of stderr, which is where git and rad explain themselves. +func (m *mirror) run(timeout time.Duration, dir, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + cmd.Env = m.env() + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if len(msg) > 500 { + msg = "…" + msg[len(msg)-500:] + } + // Never echo an argument list: a fetch carries the Gitea token in one. + return stdout.String(), fmt.Errorf("%s %s: %v: %s", name, args[0], err, msg) + } + return stdout.String(), nil +} + +func (m *mirror) git(dir string, args ...string) (string, error) { + return m.run(10*time.Minute, dir, "git", args...) +} + +func (m *mirror) rad(dir string, args ...string) (string, error) { + return m.run(2*time.Minute, dir, "rad", args...) +} + +// radInit creates the Radicle repository for r from its bare mirror and returns its RID. It also +// makes this node seed it (the node's policy is `block`: nothing is kept unless asked) and adds +// the `rad` remote that pushes go to. +func (m *mirror) radInit(dir string, r *repo) (string, error) { + desc := r.Description + if desc == "" { + desc = "Mirrored from " + r.HTMLURL + } + out, err := m.rad(dir, "init", dir, "--name", r.Name, "--description", desc, + "--default-branch", r.DefaultBranch, "--public", "--no-confirm") + if err != nil { + return "", err + } + if rid := m.remoteRID(dir); rid != "" { + return rid, nil + } + if rid := ridRe.FindString(out); rid != "" { + return rid, nil + } + return "", fmt.Errorf("no repository ID in rad init's output") +} + +// remoteRID reads the RID from the mirror's `rad` remote (rad://z…), or "" when there is none. +func (m *mirror) remoteRID(dir string) string { + out, err := m.git(dir, "config", "--get", "remote.rad.url") + if err != nil { + return "" + } + return ridFromURL(out) +} + +func ridFromURL(u string) string { + u = strings.TrimSpace(u) + if !strings.HasPrefix(u, "rad://") { + return "" + } + z, _, _ := strings.Cut(strings.TrimPrefix(u, "rad://"), "/") + return "rad:" + z +} + +// addRemote recreates what rad init sets up: fetch from the repository, push into this node's +// namespace of it. +func (m *mirror) addRemote(dir, rid string) error { + nid, err := m.rad("", "self", "--nid") + if err != nil { + return err + } + z := strings.TrimPrefix(rid, "rad:") + if _, err := m.git(dir, "remote", "add", "rad", "rad://"+z); err != nil { + return err + } + _, err = m.git(dir, "config", "remote.rad.pushurl", "rad://"+z+"/"+strings.TrimSpace(nid)) + return err +} + +// radCanonicalTags adds the rule that makes a delegate's tags the repository's refs/tags/*. Without +// it a tag exists only under the node's namespace, where `rad clone` and the explorer do not show +// it. One delegate (this node), so a threshold of 1. +func (m *mirror) radCanonicalTags(rid string) error { + _, err := m.rad("", "id", "update", "--repo", rid, "--no-confirm", "--title", "Canonical tags", + "--description", "Tags pushed by the forge are the repository's tags.", + "--payload", "xyz.radicle.crefs", "rules", `{"refs/tags/*":{"threshold":1,"allow":"delegates"}}`) + return err +} + +func (m *mirror) radDefaultBranch(rid, branch string) error { + _, err := m.rad("", "id", "update", "--repo", rid, "--no-confirm", "--title", "Default branch", + "--description", "Follows the Gitea repository.", + "--payload", "xyz.radicle.project", "defaultBranch", fmt.Sprintf("%q", branch)) + return err +} + +// announce tells the network about the new refs. Best effort: a node without peers is not an error, +// and the node announces again on its own schedule. +func (m *mirror) announce(rid string) { + if _, err := m.rad("", "sync", rid, "--announce", "--timeout", "30s"); err != nil { + log.Printf("announce %s: %v", rid, err) + } +} diff --git a/forge/setup.go b/forge/setup.go new file mode 100644 index 0000000..d00b9da --- /dev/null +++ b/forge/setup.go @@ -0,0 +1,180 @@ +package main + +import ( + "crypto/rand" + "embed" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +//go:embed all:example +var exampleFiles embed.FS + +const tokenName = "ipcr-forge-bridge" + +// setup is the install step, run after the stack is up and on every start (each action is +// idempotent). It signs in once as the admin with the server's default app password, because +// Gitea's token API accepts nothing else, and leaves: +// +// SECRETS_DIR/gitea-token public-only + read:repository: the daemon lists and clones public +// repositories, and cannot see any other. Minted only when missing. +// SECRETS_DIR/hook-secret the webhook's HMAC key. Generated only when missing. +// a system webhook every repository's push/create/delete/repository events → HOOK_URL, +// created or repaired (matched by URL), never duplicated. +// EXAMPLE_REPO a public example repository with a workflow, created when absent. +// +// The admin password is used here only and never stored. If it was changed since install, the +// step leaves the existing token and hook alone instead of failing the start. +func setup() error { + secrets := env("SECRETS_DIR", "/secrets") + g := newGitea(env("GITEA_URL", "http://gitea:3000"), env("GITEA_OWNER", "gitea_admin"), filepath.Join(secrets, "gitea-token")) + g.password = os.Getenv("GITEA_PASSWORD") + if g.password == "" { + return errors.New("GITEA_PASSWORD is required") + } + + // Gitea may still be migrating, and its admin account may not exist yet. Bounded: an install + // step that blocks forever holds the install open forever. + var code int + var err error + for i := 0; i < 80; i++ { + if code, err = g.call("GET", "/user", nil, nil); err == nil { + break + } + if code == http.StatusUnauthorized && g.hasToken() { + log.Printf("setup: the admin password no longer works; keeping the existing token and webhook") + return nil + } + time.Sleep(3 * time.Second) + } + if err != nil { + return fmt.Errorf("Gitea not ready after 240s: %w", err) + } + + if !g.hasToken() { + g.call("DELETE", "/users/"+g.owner+"/tokens/"+tokenName, nil, nil) + var t struct { + SHA1 string `json:"sha1"` + } + if _, err := g.call("POST", "/users/"+g.owner+"/tokens", map[string]any{ + "name": tokenName, "scopes": []string{"public-only", "read:repository"}, + }, &t); err != nil { + return fmt.Errorf("token: %w", err) + } + if err := writeSecret(g.tokenFile, t.SHA1); err != nil { + return err + } + log.Printf("setup: token %s written", tokenName) + } + + secretFile := filepath.Join(secrets, "hook-secret") + secret, _ := os.ReadFile(secretFile) + if strings.TrimSpace(string(secret)) == "" { + b := make([]byte, 32) + rand.Read(b) + secret = []byte(hex.EncodeToString(b)) + if err := writeSecret(secretFile, string(secret)); err != nil { + return err + } + } + if err := ensureHook(g, env("HOOK_URL", "http://ipcr-forge:8081/hooks/gitea"), strings.TrimSpace(string(secret))); err != nil { + return fmt.Errorf("webhook: %w", err) + } + + if name := env("EXAMPLE_REPO", "ipcr-hello"); name != "-" && name != "" { + // Not fatal: the forge works without its example. + if err := ensureExample(g, name); err != nil { + log.Printf("setup: example repository: %v", err) + } + } + return nil +} + +func writeSecret(path, value string) error { + tmp := path + ".new" + if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// ensureHook creates the system webhook, or rewrites the one already pointing at url (Gitea never +// returns a hook's secret, so it is set every time rather than compared). +func ensureHook(g *gitea, url, secret string) error { + config := map[string]string{"url": url, "content_type": "json", "secret": secret, "is_system_webhook": "true"} + events := []string{"push", "create", "delete", "repository"} + var hooks []struct { + ID int64 `json:"id"` + Config map[string]string `json:"config"` + } + if _, err := g.call("GET", "/admin/hooks?type=all&limit=50", nil, &hooks); err != nil { + return err + } + for _, h := range hooks { + if h.Config["url"] == url { + _, err := g.call("PATCH", fmt.Sprintf("/admin/hooks/%d", h.ID), map[string]any{ + "config": config, "events": events, "active": true, "branch_filter": "*", + }, nil) + return err + } + } + _, err := g.call("POST", "/admin/hooks", map[string]any{ + "type": "gitea", "config": config, "events": events, "active": true, "branch_filter": "*", + }, nil) + if err == nil { + log.Printf("setup: system webhook → %s", url) + } + return err +} + +// ensureExample creates /, public, with the files under example/ in one commit. A +// repository that already has content is left as it is: it may be the user's by now. +func ensureExample(g *gitea, name string) error { + var r repo + code, err := g.call("GET", "/repos/"+g.owner+"/"+name, nil, &r) + switch { + case code == http.StatusNotFound: + if _, err := g.call("POST", "/admin/users/"+g.owner+"/repos", map[string]any{ + "name": name, "private": false, "default_branch": "main", + "description": "Example: a tag here builds an image, published on IPFS by IPCR and mirrored to Radicle.", + }, &r); err != nil { + return err + } + log.Printf("setup: created %s/%s", g.owner, name) + case err != nil: + return err + case !r.Empty: + return nil + } + var files []map[string]string + err = fs.WalkDir(exampleFiles, "example", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + b, err := exampleFiles.ReadFile(p) + if err != nil { + return err + } + files = append(files, map[string]string{ + "operation": "create", "path": strings.TrimPrefix(p, "example/"), + "content": base64.StdEncoding.EncodeToString(b), + }) + return nil + }) + if err != nil { + return err + } + _, err = g.call("POST", "/repos/"+g.owner+"/"+name+"/contents", map[string]any{ + "branch": "main", "message": "Example: a Dockerfile and the workflow that publishes it", "files": files, + }, nil) + return err +} diff --git a/forge/state.go b/forge/state.go new file mode 100644 index 0000000..a1e51e2 --- /dev/null +++ b/forge/state.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" +) + +// Repository states, as shown on the page. +const ( + stSynced = "synced" // Radicle has what Gitea has + stError = "error" // the last sync failed; retried on the next event or reconcile + stWaiting = "waiting" // empty repository: nothing to publish yet + stFrozen = "frozen" // no longer public in Gitea (or deleted): no further sync, nothing retracted +) + +// repoState is what the bridge remembers per Gitea repository, keyed by Gitea's numeric ID so a +// rename or a transfer keeps its Radicle ID. +type repoState struct { + ID int64 `json:"id"` + FullName string `json:"full_name"` + HTMLURL string `json:"html_url,omitempty"` + RID string `json:"rid,omitempty"` + DefaultBranch string `json:"default_branch,omitempty"` + State string `json:"state"` + Error string `json:"error,omitempty"` + LastSync time.Time `json:"last_sync,omitzero"` + // A digest of the refs last pushed: a reconcile that finds the same refs pushes nothing. + Refs string `json:"refs,omitempty"` + // The canonical-tags rule is in the repository's identity (see rad.go). + Crefs bool `json:"crefs,omitempty"` +} + +type state struct { + mu sync.Mutex + path string + repos map[int64]*repoState +} + +func loadState(path string) (*state, error) { + s := &state{path: path, repos: map[int64]*repoState{}} + b, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return s, nil + } + if err != nil { + return nil, err + } + var m map[string]*repoState + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + for _, r := range m { + s.repos[r.ID] = r + } + return s, nil +} + +// get returns a copy of the repository's state (a zero one, with the ID set, when unknown). +func (s *state) get(id int64) repoState { + s.mu.Lock() + defer s.mu.Unlock() + if r, ok := s.repos[id]; ok { + return *r + } + return repoState{ID: id} +} + +func (s *state) known(id int64) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.repos[id] + return ok +} + +// put stores r and writes the file (renamed into place, so a reader never sees half of it). +func (s *state) put(r repoState) error { + s.mu.Lock() + defer s.mu.Unlock() + s.repos[r.ID] = &r + m := make(map[string]*repoState, len(s.repos)) + for id, v := range s.repos { + m[strconv.FormatInt(id, 10)] = v + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + tmp := filepath.Join(filepath.Dir(s.path), "."+filepath.Base(s.path)+".new") + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, s.path) +} + +func (s *state) ids() []int64 { + s.mu.Lock() + defer s.mu.Unlock() + ids := make([]int64, 0, len(s.repos)) + for id := range s.repos { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +// list is the page's view: every repository, by name, without internal bookkeeping. +func (s *state) list() []repoState { + s.mu.Lock() + out := make([]repoState, 0, len(s.repos)) + for _, r := range s.repos { + v := *r + v.Refs = "" + out = append(out, v) + } + s.mu.Unlock() + sort.Slice(out, func(i, j int) bool { return out[i].FullName < out[j].FullName }) + return out +} diff --git a/forge/web.go b/forge/web.go new file mode 100644 index 0000000..7b360cd --- /dev/null +++ b/forge/web.go @@ -0,0 +1,49 @@ +package main + +import ( + "embed" + "encoding/json" + "net/http" + "os" +) + +//go:embed web +var webFiles embed.FS + +// webHandler serves the forge's public page. Everything on it is public anyway: links, the +// documentation, IPCR's list of published images (public on IPFS) and the list of mirrored +// repositories (public on Radicle). Nothing here changes anything. +func webHandler(st *state, publishedFile string) http.Handler { + mux := http.NewServeMux() + page, _ := webFiles.ReadFile("web/index.html") + icon, _ := webFiles.ReadFile("web/icon.png") + mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Write(page) + }) + mux.HandleFunc("GET /icon.png", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Cache-Control", "max-age=3600") + w.Write(icon) + }) + mux.HandleFunc("GET /images.json", func(w http.ResponseWriter, r *http.Request) { + b, err := os.ReadFile(publishedFile) + if err != nil { + http.Error(w, "nothing published yet", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Write(b) + }) + mux.HandleFunc("GET /repos.json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + json.NewEncoder(w).Encode(st.list()) + }) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("ok\n")) + }) + return mux +} diff --git a/Apps/IPCR-Forge/seed/site/icon.png b/forge/web/icon.png similarity index 100% rename from Apps/IPCR-Forge/seed/site/icon.png rename to forge/web/icon.png diff --git a/Apps/IPCR-Forge/seed/site/index.html b/forge/web/index.html similarity index 52% rename from Apps/IPCR-Forge/seed/site/index.html rename to forge/web/index.html index 89d480c..b7a1396 100644 --- a/Apps/IPCR-Forge/seed/site/index.html +++ b/forge/web/index.html @@ -6,21 +6,21 @@ IPCR Forge @@ -78,67 +80,62 @@
-

From git push to docker pull on any server, without a central -service. Code lives on Radicle, this node builds it with GitHub-style workflows, and IPCR serves -the image from IPFS.

+

From git push to docker pull on any server. Work in Gitea as +usual: its Actions build your images, IPCR serves them from IPFS, and every public repository is +mirrored to the Radicle peer-to-peer network.

- RadicleBrowse repositories, issues and patches. Public. - CI runsEvery build this node ran, with its log and result. Public. - Published imagesWhat this server serves from IPFS, with the line to put in a compose file. + GiteaYour repositories, pull requests, issues and builds. Sign in as gitea_admin. + Published imagesWhat this forge serves from IPFS, with the line to put in a compose file. + On RadicleThe public repositories mirrored to the Radicle network. Public.

How it works

-
1 · Pushgit push rad a v* tag to the Radicle network
+
1 · Pushgit push a v* tag to Gitea, or create it in Gitea's web UI
→
-
2 · Triggerthis node receives the tag and the Radicle CI broker picks it up
+
2 · BuildGitea Actions runs .gitea/workflows/ and pushes the image to localhost:5000
→
-
3 · Buildact runs the repo's .github/workflows/ and pushes the image
+
3 · IPFSIPCR imports each tag and publishes it under this forge's IPNS name
→
-
4 · IPFSIPCR imports each tag and publishes it under an IPNS name
-
→
-
5 · Pullany server with IPCR: ipcr.localhost:4767/ipns/…
+
4 · Pullany server with IPCR: ipcr.localhost:4767/ipns/…
-

Steps 2 to 4 are automatic and usually take a few minutes. The very first build on this server -takes longer (about 5 to 10 minutes) while it downloads its build images. A repository needs no -configuration beyond committing a workflow. There is no second copy of it anywhere: the build -reads the commit straight from this node. Builds of the default branch and of patches also -write their result back into the repository, signed by the node, as a Radicle job.

-

Only changes signed by one of the repository's delegates are built: release tags, -the default branch and their patches. A build can publish an image, so it gets the trust of a -release.

+

Steps 2 and 3 are automatic and usually take a few minutes. The very first build on this server +takes longer (about 5 to 10 minutes) while it downloads its build images.

+

Alongside, every public repository is copied to Radicle +within seconds of each push: branches and tags, signed by this server's Radicle node. Gitea stays +the one place to work. The Radicle copy is for distribution: anyone can rad clone it, +and it stays available even when this server is not. Private repositories are never copied.

+ +

Try it with the example

+

The install created gitea_admin/ipcr-hello, a tiny web +page with a Dockerfile and the workflow that publishes it.

+
    +
  1. Open it in Gitea, then Tags → New tag (or Releases → New release): name + v1.0.0, target main.
    2. +
  2. Follow the build under the repository's Actions tab.
    3. +
  3. About a minute after it finishes, the image appears below under + Published images, and the tag on Radicle.
    4. +

Published images

Loading…

-

Publish a project

+

Publish your own project

    -
  1. Install Radicle on your machine and create an identity: see - radicle.xyz, then rad auth. - Start your own node, which carries your pushes: rad node start.
    2. -
  2. Connect to this server's node so your pushes reach it: - 
    rad node connect <node-id>@<this server>:8776
    3. -
  3. Publish the repository. It has to be a Git repository with at least one commit: -
    git init -b main
-git add -A
-git commit -m "First commit"
-rad init
    - rad init prints the repository ID (rad:z…).
    4. -
  4. Let this node seed it. The node only keeps repositories it is told to (seeding policy - block). In this server's Terminal app or over SSH: - 
    docker exec radicle-node rad seed rad:z… --scope all
    - It ends with ✗ Target not met: could not fetch from […]. That is about the public - Radicle seeds, not this server, and can be ignored. The repository is in place once it shows - in the Radicle explorer.
    5. -
  5. Add a workflow at .github/workflows/build.yml, as on GitHub. This one - builds the Dockerfile at the root and publishes it for every v* tag. - vars.IMAGE is set for every repository (this node's staging registry plus the - repository's name); there is no secret to configure. Two lines differ from a GitHub-hosted - workflow: context: . (builds the checkout, not a GitHub URL) and buildx's - network=host (so its builder can reach the staging registry). Commit it - (git add .github && git commit -m "Build the image"). -
    on:
+  
  6. Create a repository in Gitea (+ → New repository)
+    and push your code to it:
+
    git remote add origin https://gitea-<domain>/<owner>/<repo>.git
+git push -u origin main
    
+    Git asks for your Gitea user name and password (or an access token, from Settings →
+    Applications).
    7. 
+  
  7. Add a workflow at .gitea/workflows/build.yml. This one builds the
+    Dockerfile at the root for every v* tag. It is a GitHub-style
+    workflow with two differences: the image goes to localhost:5000, this forge's
+    staging registry (no login, no secret), and buildx's builder runs with network=host
+    so it can reach that registry.
+
    name: image
+on:
   push:
     tags: ['v*']
 jobs:
@@ -149,7 +146,7 @@ jobs:
       - uses: docker/metadata-action@v5
         id: meta
         with:
-          images: ${{ vars.IMAGE }}
+          images: localhost:5000/${{ github.repository }}
           tags: |
             type=semver,pattern={{version}}
             type=raw,value=latest
@@ -163,33 +160,45 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
    8. 
   
  8. Release:
-
    git tag -a v1.0.0 -m v1.0.0
-git push rad main v1.0.0
    
-    Follow the build under CI runs. A minute after it finishes, the tag appears
-    under Published images.
    9. 
+
    git add .gitea && git commit -m "Build the image"
+git tag v1.0.0
+git push origin main v1.0.0
    
+    The image is published as <owner>/<repo>:1.0.0 under this forge's name.
+

Who can publish: anyone who can push to a repository on this forge, since a build +can push any image name to the staging registry. Images are public once on IPFS, even when they come from +a private repository. Gitea holds a pull request from a fork until someone with write access approves +its workflow run: read the workflow before you approve.

Use an image

Copy the line from Published images into a compose file. It works on this server and on any server running IPCR:

services:
   myapp:
-    image: ipcr.localhost:4767/ipns/k51qzi5uqu5d…:1.0.0   # a tag: follows new builds of 1.0.0
-    # image: ipcr.localhost:4767/ipfs/bafkrei…           # a CID: this exact image, forever
-

/ipns/<name>:<tag> is a name the publisher can move to a new build. -/ipfs/<cid> is the image's content address and can never change.

+ image: ipcr.localhost:4767/ipns/k51…/<owner>/<repo>:1.0.0 # a tag: follows new builds of 1.0.0 + # image: ipcr.localhost:4767/ipfs/bafkrei… # a CID: this exact image, forever +

Or pull it by hand: docker pull ipcr.localhost:4767/ipns/k51…/<owner>/<repo>:1.0.0. +/ipns/… is a name this forge can move to a new build; /ipfs/<cid> is the +image's content address and can never change.

For a readable name, point a name you own at this forge's publisher name (k51…), which holds every image it builds:

  • ENS: set the content hash of example.eth to ipns://k51… (one transaction, once). Images are then - ipcr.localhost:4767/ipns/example.eth/<repo>:1.0.0.
    • + ipcr.localhost:4767/ipns/example.eth/<owner>/<repo>:1.0.0.
  • DNS: add a TXT record _dnslink.example.com = dnslink=/ipns/k51…. Images are then - ipcr.localhost:4767/ipns/example.com/<repo>:1.0.0.
    • + ipcr.localhost:4767/ipns/example.com/<owner>/<repo>:1.0.0.

New releases need no change to either record: they update the publisher name.

+

On Radicle

+

Public repositories of this forge, as mirrored to the Radicle network by this server's node +(…). Clone one with rad clone <id> from any Radicle node.

+

Loading…

+

A repository that turns private or is deleted in Gitea is no longer synced +(frozen). What was already published stays on the Radicle network: it cannot be taken back.

+

Pull from another server

Any server with IPCR resolves the same lines and fetches the image from the IPFS network, from this server and from every other server that already pulled it.

@@ -210,14 +219,15 @@ for IPFS peers. Details: Addresses on this server
+ + -
AddressWhat
Gitea: repositories, Actions, users (gitea_admin)
This page (public)
Radicle explorer (public)
Radicle API and read-only git over HTTPS (public)
ipcr-forge-…/ci/CI runs and their logs (public)
ipcr.localhost:4767IPCR registry, for this server's Docker only
TCP 8776 · TCP/UDP 4768Radicle peers · IPFS peers
- +
IPCR Forge · Gitea · act_runner · Radicle · Kubo · nerdctl · IPCR