From 2ed05654860b3c12041982cabc2d009975de5c7d Mon Sep 17 00:00:00 2001 From: poprhythm Date: Sun, 16 Aug 2026 14:33:56 +0000 Subject: [PATCH] add deploy-stack skill; falco: fix hook-tamper condition, add three more post-incident rules git-hook-tamper.yaml's condition used proc.exepath, which resolves to the script interpreter's path (e.g. /bin/busybox) for shebang scripts, not the script's own path - switched to proc.cmdline, which retains the originally invoked path. Confirmed via live-testing both ways. ssh-persistence/cloud-metadata-probe/db-spawned-process round out the post-incident hardening pass with a few more incubating-ruleset adaptations. --- .claude/skills/deploy-stack/SKILL.md | 79 +++++++++++++++++++++++++++ falco/docker-compose.yaml | 3 + falco/rules/cloud-metadata-probe.yaml | 25 +++++++++ falco/rules/db-spawned-process.yaml | 30 ++++++++++ falco/rules/git-hook-tamper.yaml | 2 +- falco/rules/ssh-persistence.yaml | 34 ++++++++++++ 6 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 .claude/skills/deploy-stack/SKILL.md create mode 100644 falco/rules/cloud-metadata-probe.yaml create mode 100644 falco/rules/db-spawned-process.yaml create mode 100644 falco/rules/ssh-persistence.yaml diff --git a/.claude/skills/deploy-stack/SKILL.md b/.claude/skills/deploy-stack/SKILL.md new file mode 100644 index 0000000..2f2716a --- /dev/null +++ b/.claude/skills/deploy-stack/SKILL.md @@ -0,0 +1,79 @@ +--- +name: deploy-stack +description: Deploy a docker-infrastructure change - push to Gitea, then redeploy the stack via Portainer. Use whenever a service's docker-compose.yaml (or other tracked file) changed and needs to go live. +--- + +# Deploy a stack (Gitea → Portainer) + +This repo's services go live via GitOps: commit → push to Gitea (self-hosted, SSH +remote) → tell Portainer to pull + redeploy. Always use `./portainer.sh` for the +Portainer side; never raw `curl` unless `portainer.sh` can't do it. + +## Steps + +1. **Stage and commit** only the files that changed for this task (never `git add -A`). + Imperative mood, first line < 72 chars, explain *why* not *what*. + +2. **Sync with remote before pushing.** `git push origin main` fails non-fast-forward + if anyone (or the Gitea web UI) committed since your last pull. Prefer: + ``` + git pull --no-edit origin main # fetch + merge in one step + git push origin main + ``` + The remote is SSH (`git@gitea.kolpacksoftware.com:...`). `GITEA_TOKEN` in + `.credentials` is NOT valid for HTTPS push (403) - don't try HTTPS for push. + +3. **If push/fetch fails with a low-level git protocol error** (`bad pack header`, + `unable to fork git-pack-objects`, `cannot exec '.../hooks/...'`) - this is NOT + a normal merge conflict, it means Gitea's git service itself is broken + (corrupted repo, poisoned global gitconfig, etc). Do not just retry blindly. + Stop and investigate server-side (`docker logs gitea`, check + `/data/gitea/home/.gitconfig` inside the container) before continuing - this + exact failure mode was an active RCE backdoor once (see + `gitea-log-poisoning-attack-2026-08.md` memory). HTTPS fetch with `GITEA_TOKEN` + can help bisect whether it's SSH-specific or instance-wide: + ``` + source .credentials + git fetch "https://${GITEA_USER}:${GITEA_TOKEN}@gitea.kolpacksoftware.com/homelab/docker-infrastructure.git" main + ``` + +4. **Find the stack name/ID** if you don't already know it: + ``` + source .credentials && ./portainer.sh list + ``` + Stack names in Portainer usually match the service directory name, but not + always (verify with `list`, don't assume). + +5. **Redeploy** (pulls latest git commit + recreates containers): + ``` + source .credentials && ./portainer.sh redeploy + ``` + This only works for stacks that are git-linked in Portainer. If it's not + git-linked, changes to `.env` values must go through + `./portainer.sh set-env KEY=VALUE` instead - a `.env` file at + the repo path is gitignored and NOT read by git-linked Portainer deploys. + +6. **Verify**: `docker ps --filter name=` for status, `docker logs + --tail 50` to confirm it actually came up clean, not just "Up". + +## Gotchas + +- **`.env` files are gitignored and invisible to git-linked stacks.** Portainer + reads env vars it has stored for the stack (set via UI or `portainer.sh + set-env`), not the local `.env` file. Check current values first with + `./portainer.sh get-env ` before adding new ones, and set any new + var explicitly - editing the local `.env` alone does nothing for a deployed + git-linked stack. +- **Not every service is a Portainer stack.** Some (e.g. `falco`) are run via + plain `docker run`/manual `docker compose`, bind-mounting config directly from + this repo checkout. For those, `portainer.sh redeploy` will fail with "stack + not found" - check `docker inspect --format '{{json .Mounts}}'` + to see if it's bind-mounted from this repo (if so, a `git push` alone is + enough to update the *source* files; the container itself needs a manual + `docker restart` or recreate to pick up new volume mounts). +- **No `docker compose` CLI on this host.** `docker compose up -d` fails + outright - always go through Portainer, or `docker run`/`docker restart` + directly for manually-managed services. +- Portainer's `GET /api/stacks/` can return Unauthorized for some stacks - + `portainer.sh` already works around this by listing + filtering; don't call + the raw API directly. diff --git a/falco/docker-compose.yaml b/falco/docker-compose.yaml index 18db208..1d321cc 100644 --- a/falco/docker-compose.yaml +++ b/falco/docker-compose.yaml @@ -34,3 +34,6 @@ services: - ./rules/tune-noise.yaml:/etc/falco/rules.d/tune-noise.yaml:ro - ./rules/git-hook-tamper.yaml:/etc/falco/rules.d/git-hook-tamper.yaml:ro - ./rules/unexpected-child-of-git.yaml:/etc/falco/rules.d/unexpected-child-of-git.yaml:ro + - ./rules/ssh-persistence.yaml:/etc/falco/rules.d/ssh-persistence.yaml:ro + - ./rules/cloud-metadata-probe.yaml:/etc/falco/rules.d/cloud-metadata-probe.yaml:ro + - ./rules/db-spawned-process.yaml:/etc/falco/rules.d/db-spawned-process.yaml:ro diff --git a/falco/rules/cloud-metadata-probe.yaml b/falco/rules/cloud-metadata-probe.yaml new file mode 100644 index 0000000..a6eea2d --- /dev/null +++ b/falco/rules/cloud-metadata-probe.yaml @@ -0,0 +1,25 @@ +# Custom rule, adapted from falcosecurity/rules' Incubating ruleset (not +# shipped in this Falco image). This host has no cloud provider, so +# 169.254.169.254 (the AWS/GCP/Azure instance-metadata IP) has zero +# legitimate traffic ever - any outbound connection to it is a strong +# signal, either SSRF probing or a container image/script that assumes a +# cloud environment. Near-zero false-positive risk, so this stays a plain +# CRITICAL with no scoped exception - if something legitimate ever needs +# it, add a proc/container-scoped exception here rather than disabling. + +- rule: Contact cloud metadata service from container + desc: > + Detects attempts to communicate with a cloud instance metadata service + (169.254.169.254) from a container. This host has no cloud provider, + so this endpoint should never see legitimate traffic - treat any hit + as SSRF probing or malware/scripts written for a cloud environment. + condition: > + outbound + and container + and fd.sip="169.254.169.254" + output: > + Outbound connection to cloud instance metadata service + (user=%user.name command=%proc.cmdline connection=%fd.name + container=%container.name image=%container.image.repository pid=%proc.pid) + priority: CRITICAL + tags: [network, credential_access, mitre_credential_access] diff --git a/falco/rules/db-spawned-process.yaml b/falco/rules/db-spawned-process.yaml new file mode 100644 index 0000000..9777be3 --- /dev/null +++ b/falco/rules/db-spawned-process.yaml @@ -0,0 +1,30 @@ +# Custom rule, adapted from falcosecurity/rules' Incubating ruleset (not +# shipped in this Falco image). Generalizes the lesson from the gitea +# log-poisoning RCE (a service's own process forking something unexpected) +# to every other DB-backed service on this host - immich, firefly-iii, +# romm, inbox-zero-db, etc. A database server forking a child process other +# than itself is not normal and often follows a SQL injection attack. + +- list: db_server_binaries + items: [mysqld, postgres, sqlplus] + +- macro: user_known_db_spawned_processes + condition: (never_true) + +- rule: DB program spawned process + desc: > + A program related to a database server created an unexpected child + process (other than itself). This is not supposed to happen and often + follows SQL injection attacks - could indicate unauthorized data + extraction or tampering. + condition: > + spawned_process + and proc.pname in (db_server_binaries) + and not proc.name in (db_server_binaries) + and not user_known_db_spawned_processes + output: > + Database-related program spawned unexpected process + (user=%user.name command=%proc.cmdline parent=%proc.pname + container=%container.name image=%container.image.repository pid=%proc.pid) + priority: WARNING + tags: [database, process, mitre_execution] diff --git a/falco/rules/git-hook-tamper.yaml b/falco/rules/git-hook-tamper.yaml index 96e349b..7579e85 100644 --- a/falco/rules/git-hook-tamper.yaml +++ b/falco/rules/git-hook-tamper.yaml @@ -19,7 +19,7 @@ condition: > spawned_process and container.name = "gitea" - and proc.exepath contains "/hooks/" + and proc.cmdline contains "/hooks/" and not proc.name in (gitea_managed_hook_names) output: > Unexpected git hook executed diff --git a/falco/rules/ssh-persistence.yaml b/falco/rules/ssh-persistence.yaml new file mode 100644 index 0000000..3d85075 --- /dev/null +++ b/falco/rules/ssh-persistence.yaml @@ -0,0 +1,34 @@ +# Custom rule, adapted from falcosecurity/rules' Incubating ruleset (not +# shipped in this Falco image - only the Stable ruleset ships by default). +# Added after research into what else might be useful post-incident: neither +# the gitea/xmrig nor gitea log-poisoning incidents involved SSH-key +# persistence, but it's the textbook next move after any RCE, and gitea +# itself has SSH access (port 222) making this directly relevant here. + +- list: ssh_binaries + items: [ + sshd, sftp-server, ssh-agent, + ssh, scp, sftp, + ssh-keygen, ssh-keysign, ssh-keyscan, ssh-add + ] + +- macro: user_ssh_directory + condition: (fd.name contains '/.ssh/' and fd.name glob '/home/*/.ssh/*') + +- rule: Adding ssh keys to authorized_keys + desc: > + After gaining access, attackers can modify the authorized_keys file to + maintain persistence on a victim host. Detects any write to an + authorized_keys file under a user's .ssh directory or /root/.ssh, + by a process that isn't one of ssh's own binaries. + condition: > + open_write + and (user_ssh_directory or fd.name startswith /root/.ssh) + and fd.name endswith authorized_keys + and not proc.name in (ssh_binaries) + output: > + Adding ssh keys to authorized_keys + (user=%user.name file=%fd.name command=%proc.cmdline + container=%container.name image=%container.image.repository pid=%proc.pid) + priority: WARNING + tags: [ssh, persistence, mitre_persistence]