From 017e3a61a811ec936d94f98b4842fbdb0bb9ca21 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 13:15:19 +0000 Subject: [PATCH 1/5] Harden staging and bring GitHub/Gitea CI to parity Address issue #20 and the CI divergence between Gitea and GitHub. Issue #20 (staging seeded from a prod snapshot): - Read SECRET_KEY from the environment with the insecure dev key as fallback, so each deployment can have its own key. - Add a `scrub_staging` management command that clears django_session and the django-q schedule/queue/results, removing copied prod sessions and the inherited convert_prices() schedule. - Run the scrub from entrypoint.sh when STAGING=true, and wire STAGING plus a per-branch SECRET_KEY into the Gitea staging deploy. CI parity (both systems kept, independent): - Add the Node/pnpm/TypeScript build steps to the Gitea build workflow to match the GitHub test job. - Add a GitHub staging workflow that deploys per-branch ephemeral instances to Fly.io (*.fly.dev) with a fresh database seeded from sample fixtures and its own SECRET_KEY, never production data. Tears the app down on branch delete and comments the URL on the open PR via github-script. - Add fly.staging.toml and a LOAD_SAMPLE_DATA entrypoint hook for the fresh-database public staging. https://claude.ai/code/session_01KYjUcNjLfZ8Hq1GAC8J4oZ --- .gitea/workflows/build.yml | 11 +++ .gitea/workflows/staging.yml | 5 ++ .github/workflows/staging.yml | 96 ++++++++++++++++++++++ entrypoint.sh | 19 +++++ fly.staging.toml | 29 +++++++ games/management/commands/scrub_staging.py | 28 +++++++ tests/test_scrub_staging.py | 36 ++++++++ timetracker/settings.py | 7 +- 8 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/staging.yml create mode 100644 fly.staging.toml create mode 100644 games/management/commands/scrub_staging.py create mode 100644 tests/test_scrub_staging.py diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 4683dc7..4ec654c 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -19,6 +19,17 @@ jobs: - name: Install dependencies run: uv sync --frozen + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install pnpm and JS dependencies + run: npm install -g pnpm && pnpm install --frozen-lockfile --ignore-scripts + + - name: Build TypeScript + run: make ts + - name: Install Playwright browsers run: uv run playwright install --with-deps chromium diff --git a/.gitea/workflows/staging.yml b/.gitea/workflows/staging.yml index a54ce17..2dfbfb3 100644 --- a/.gitea/workflows/staging.yml +++ b/.gitea/workflows/staging.yml @@ -19,6 +19,9 @@ jobs: SLUG=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-+/-/g; s/^-//; s/-$//' | cut -c1-40) echo "SLUG=${SLUG}" >> "$GITHUB_ENV" echo "HOST=tracker-${SLUG}.home.arpa" >> "$GITHUB_ENV" + # Per-staging secret so each instance has its own key, decoupling it + # from prod even though the database is seeded from a prod snapshot. + echo "STAGING_SECRET_KEY=staging-${SLUG}-$(head -c16 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')" >> "$GITHUB_ENV" - name: Build image run: docker build -t "timetracker:staging-${SLUG}" . @@ -32,6 +35,8 @@ jobs: -e PUID=1000 \ -e PGID=100 \ -e DATA_DIR=/home/timetracker/app/data \ + -e STAGING=true \ + -e "SECRET_KEY=${STAGING_SECRET_KEY}" \ -e "CSRF_TRUSTED_ORIGINS=https://${HOST}" \ -v "timetracker-staging-${SLUG}:/home/timetracker/app/data" \ -l "caddy=${HOST}" \ diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 0000000..144528f --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1,96 @@ +name: Staging deployment + +on: + push: + branches-ignore: [main] + delete: + +concurrency: + group: staging-${{ github.event.ref }} + cancel-in-progress: true + +jobs: + deploy: + if: github.event_name == 'push' + runs-on: ubuntu-latest + env: + BRANCH: ${{ github.ref_name }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Compute staging name + run: | + SLUG=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-+/-/g; s/^-//; s/-$//' | cut -c1-30) + APP="timetracker-staging-${SLUG}" + echo "SLUG=${SLUG}" >> "$GITHUB_ENV" + echo "APP=${APP}" >> "$GITHUB_ENV" + echo "HOST=${APP}.fly.dev" >> "$GITHUB_ENV" + + - name: Set up flyctl + uses: superfly/flyctl-actions/setup-flyctl@master + + - name: Create app if missing + run: | + if ! flyctl status --app "$APP" >/dev/null 2>&1; then + flyctl apps create "$APP" --org personal + fi + + - name: Set staging secrets + run: | + # Per-app SECRET_KEY so each staging instance is independent and no + # session cookie is shared across instances or with production. + SECRET_KEY="staging-${SLUG}-$(head -c16 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')" + flyctl secrets set --app "$APP" --stage \ + "SECRET_KEY=${SECRET_KEY}" \ + "CSRF_TRUSTED_ORIGINS=https://${HOST}" + + - name: Deploy + run: flyctl deploy --app "$APP" --config fly.staging.toml --remote-only --yes + + - name: Summary + run: echo "Deployed to https://${HOST}" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment staging URL on PR + uses: actions/github-script@v7 + with: + script: | + const host = process.env.HOST; + const branch = process.env.BRANCH; + const body = `Staging deployment: https://${host}`; + const { owner, repo } = context.repo; + const pulls = await github.rest.pulls.list({ + owner, repo, state: "open", head: `${owner}:${branch}`, + }); + const pr = pulls.data[0]; + if (!pr) { + core.info(`No open PR for branch '${branch}', skipping comment`); + return; + } + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: pr.number, + }); + if (comments.some((comment) => comment.body === body)) { + core.info(`Staging URL already commented on PR #${pr.number}`); + return; + } + await github.rest.issues.createComment({ + owner, repo, issue_number: pr.number, body, + }); + core.info(`Commented staging URL on PR #${pr.number}`); + + teardown: + if: github.event_name == 'delete' && github.event.ref_type == 'branch' + runs-on: ubuntu-latest + env: + BRANCH: ${{ github.event.ref }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + steps: + - name: Set up flyctl + uses: superfly/flyctl-actions/setup-flyctl@master + + - name: Destroy staging app + run: | + SLUG=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-+/-/g; s/^-//; s/-$//' | cut -c1-30) + APP="timetracker-staging-${SLUG}" + flyctl apps destroy "$APP" --yes 2>/dev/null || true diff --git a/entrypoint.sh b/entrypoint.sh index 4ef7de8..0dd3c08 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -20,6 +20,25 @@ chown "$PUID:$PGID" /var/log/supervisor python manage.py migrate python manage.py collectstatic --clear --no-input +# Staging seeded from a production snapshot: remove copied sessions and the +# inherited django-q schedule/queue so staging neither shares prod's session +# cookies nor independently runs scheduled tasks (see issue #20). +if [ "${STAGING:-false}" = "true" ]; then + python manage.py scrub_staging +fi + +# Public staging with a fresh database (e.g. Fly.io): load demo data instead +# of any production snapshot. Runs once while the games table is empty. +if [ "${LOAD_SAMPLE_DATA:-false}" = "true" ]; then + python manage.py shell -c " +from games.models import Game +from django.core.management import call_command +if not Game.objects.exists(): + call_command('loaddata', 'sample.yaml') + print('Loaded sample data.') +" +fi + if [ "${CREATE_DEFAULT_SUPERUSER:-false}" = "true" ]; then python manage.py shell -c " from django.contrib.auth import get_user_model diff --git a/fly.staging.toml b/fly.staging.toml new file mode 100644 index 0000000..82e2668 --- /dev/null +++ b/fly.staging.toml @@ -0,0 +1,29 @@ +# Shared Fly.io configuration for ephemeral, per-branch GitHub staging deploys. +# +# The app name is NOT set here on purpose; each branch supplies its own via +# `flyctl deploy --app timetracker-staging-`. These instances run with a +# fresh database seeded from sample fixtures (never production data) and their +# own SECRET_KEY, so they are safe to expose on a public *.fly.dev hostname. + +primary_region = "ams" + +[build] + dockerfile = "Dockerfile" + +[env] + PROD = "1" + TZ = "Europe/Prague" + DATA_DIR = "/home/timetracker/app/data" + LOAD_SAMPLE_DATA = "true" + CREATE_DEFAULT_SUPERUSER = "true" + +[http_service] + internal_port = 8000 + force_https = true + auto_stop_machines = "stop" + auto_start_machines = true + min_machines_running = 0 + +[[vm]] + size = "shared-cpu-1x" + memory = "512mb" diff --git a/games/management/commands/scrub_staging.py b/games/management/commands/scrub_staging.py new file mode 100644 index 0000000..7cb7b63 --- /dev/null +++ b/games/management/commands/scrub_staging.py @@ -0,0 +1,28 @@ +from django.contrib.sessions.models import Session +from django.core.management.base import BaseCommand +from django_q.models import OrmQ, Schedule, Task + + +class Command(BaseCommand): + help = ( + "Remove copied production artifacts from a staging database seeded " + "from a production snapshot: clears authenticated sessions and the " + "django-q schedule/queue/results so staging does not share prod's " + "session cookies or independently run scheduled tasks." + ) + + def handle(self, *args, **kwargs): + sessions_deleted, _ = Session.objects.all().delete() + schedules_deleted, _ = Schedule.objects.all().delete() + tasks_deleted, _ = Task.objects.all().delete() + queued_deleted, _ = OrmQ.objects.all().delete() + + self.stdout.write( + self.style.SUCCESS( + "Scrubbed staging database: " + f"{sessions_deleted} session(s), " + f"{schedules_deleted} schedule(s), " + f"{tasks_deleted} task result(s), " + f"{queued_deleted} queued task(s) removed." + ) + ) diff --git a/tests/test_scrub_staging.py b/tests/test_scrub_staging.py new file mode 100644 index 0000000..58bf2d7 --- /dev/null +++ b/tests/test_scrub_staging.py @@ -0,0 +1,36 @@ +from datetime import timedelta + +from django.contrib.sessions.models import Session as DjangoSession +from django.core.management import call_command +from django.test import TestCase +from django.utils.timezone import now +from django_q.models import Schedule + + +class ScrubStagingTest(TestCase): + def test_scrub_removes_sessions_and_schedules(self): + DjangoSession.objects.create( + session_key="copied-from-prod", + session_data="", + expire_date=now() + timedelta(days=1), + ) + Schedule.objects.create( + func="games.tasks.convert_prices", + name="Update converted prices", + schedule_type=Schedule.MINUTES, + ) + + self.assertEqual(DjangoSession.objects.count(), 1) + self.assertEqual(Schedule.objects.count(), 1) + + call_command("scrub_staging") + + self.assertEqual(DjangoSession.objects.count(), 0) + self.assertEqual(Schedule.objects.count(), 0) + + def test_scrub_is_safe_on_empty_database(self): + # Should not raise when there is nothing to remove (fresh staging DB). + call_command("scrub_staging") + + self.assertEqual(DjangoSession.objects.count(), 0) + self.assertEqual(Schedule.objects.count(), 0) diff --git a/timetracker/settings.py b/timetracker/settings.py index aaec6d8..014f4a0 100644 --- a/timetracker/settings.py +++ b/timetracker/settings.py @@ -21,7 +21,12 @@ BASE_DIR = Path(__file__).resolve().parent.parent # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "django-insecure-x0_t$gei=_o_p(%%!-db$jezka@y+d67$a8tvw13nl^8$l*t@=" +# Read from the environment so each deployment (prod, staging) can supply its +# own key; falls back to an insecure default for local development and tests. +SECRET_KEY = os.environ.get( + "SECRET_KEY", + "django-insecure-x0_t$gei=_o_p(%%!-db$jezka@y+d67$a8tvw13nl^8$l*t@=", +) # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False if os.environ.get("PROD") else True From 227b1f674d60f5eed093a76be6da391738745473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kucharczyk?= Date: Sun, 14 Jun 2026 16:21:46 +0200 Subject: [PATCH 2/5] Fix scrub_staging test isolation: use TransactionTestCase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestCase wraps each test in a savepoint — when scrub_staging deletes all django_session rows inside that savepoint, the rollback restores any sessions committed by earlier tests (e.g. force_login in test_paths_return_200). Those restored rows then leaked into the e2e live-server tests, causing intermittent Session.MultipleObjectsReturned errors. TransactionTestCase flushes the DB before each test instead of using savepoints, giving scrub_staging a clean slate and removing the leakage. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_scrub_staging.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_scrub_staging.py b/tests/test_scrub_staging.py index 58bf2d7..f4622dc 100644 --- a/tests/test_scrub_staging.py +++ b/tests/test_scrub_staging.py @@ -2,12 +2,18 @@ from datetime import timedelta from django.contrib.sessions.models import Session as DjangoSession from django.core.management import call_command -from django.test import TestCase +from django.test import TransactionTestCase from django.utils.timezone import now from django_q.models import Schedule -class ScrubStagingTest(TestCase): +class ScrubStagingTest(TransactionTestCase): + # TransactionTestCase flushes the DB before each test instead of wrapping + # in a savepoint. Required here because scrub_staging deletes all sessions + # — a TestCase savepoint rollback would restore any sessions committed by + # earlier tests (e.g. force_login in test_paths_return_200) and leak state + # into the e2e live-server tests that follow. + def test_scrub_removes_sessions_and_schedules(self): DjangoSession.objects.create( session_key="copied-from-prod", @@ -29,7 +35,6 @@ class ScrubStagingTest(TestCase): self.assertEqual(Schedule.objects.count(), 0) def test_scrub_is_safe_on_empty_database(self): - # Should not raise when there is nothing to remove (fresh staging DB). call_command("scrub_staging") self.assertEqual(DjangoSession.objects.count(), 0) From d2bf6efdb43f1a0a40dc2399a6d7623f922eaa59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kucharczyk?= Date: Sun, 14 Jun 2026 16:25:44 +0200 Subject: [PATCH 3/5] Fix Docker assets stage missing pnpm-workspace.yaml The tar override lives in pnpm-workspace.yaml, which pnpm-lock.yaml records. Copying only package.json + pnpm-lock.yaml left pnpm without the overrides config, causing ERR_PNPM_LOCKFILE_CONFIG_MISMATCH on frozen install. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 29890c9..5412c8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN uv run python manage.py gen_element_types FROM node:22-bookworm-slim AS assets WORKDIR /app -COPY package.json pnpm-lock.yaml ./ +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ # Corepack ships with Node and activates the pnpm version pinned in # package.json's "packageManager" field — no npm bootstrap needed. RUN corepack enable && pnpm install --frozen-lockfile --ignore-scripts From 9d02121c5b1552cd4bfdf9e3fe7c889cdc91777f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kucharczyk?= Date: Sun, 14 Jun 2026 16:27:58 +0200 Subject: [PATCH 4/5] Grant pull-requests: write to staging deploy job The github-script PR comment step needs this permission; without it the GITHUB_TOKEN gets Resource not accessible by integration. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/staging.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 144528f..3eb47cd 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -13,6 +13,8 @@ jobs: deploy: if: github.event_name == 'push' runs-on: ubuntu-latest + permissions: + pull-requests: write env: BRANCH: ${{ github.ref_name }} FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} From 2b450c6d4748677a8c848d42e0d66237f8157b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kucharczyk?= Date: Sun, 14 Jun 2026 16:36:39 +0200 Subject: [PATCH 5/5] Fix sample.yaml fixture for current schema Three issues from when the fixture was created before schema evolved: - Game and Platform lacked created_at (auto_now_add bypassed by loaddata) - Purchase lacked created_at/updated_at - Purchase used 'game' FK that no longer exists; field is now the M2M 'games', serialized as a list Co-Authored-By: Claude Sonnet 4.6 --- games/fixtures/sample.yaml | 91 +++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/games/fixtures/sample.yaml b/games/fixtures/sample.yaml index 5cbef3f..13bbaf6 100644 --- a/games/fixtures/sample.yaml +++ b/games/fixtures/sample.yaml @@ -1,71 +1,90 @@ -- model: games.game - pk: 1 - fields: - name: Nioh 2 - wikidata: Q67482292 -- model: games.game - pk: 2 - fields: - name: Elden Ring - wikidata: Q64826862 -- model: games.game - pk: 3 - fields: - name: Cyberpunk 2077 - wikidata: Q3182559 -- model: games.purchase - pk: 1 - fields: - game: 1 - platform: 1 - date_purchased: 2021-02-13 - date_refunded: null -- model: games.purchase - pk: 2 - fields: - game: 2 - platform: 1 - date_purchased: 2022-02-24 - date_refunded: null -- model: games.purchase - pk: 3 - fields: - game: 3 - platform: 1 - date_purchased: 2020-12-07 - date_refunded: null - model: games.platform pk: 1 fields: name: Steam group: PC + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 3 fields: name: Xbox Gamepass group: PC + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 4 fields: name: Epic Games Store group: PC + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 5 fields: name: Playstation 5 group: Playstation + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 6 fields: name: Playstation 4 group: Playstation + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 7 fields: name: Nintendo Switch group: Nintendo + created_at: "2020-01-01T00:00:00Z" - model: games.platform pk: 8 fields: name: Nintendo 3DS group: Nintendo + created_at: "2020-01-01T00:00:00Z" +- model: games.game + pk: 1 + fields: + name: Nioh 2 + wikidata: Q67482292 + created_at: "2021-02-13T00:00:00Z" + updated_at: "2021-02-13T00:00:00Z" +- model: games.game + pk: 2 + fields: + name: Elden Ring + wikidata: Q64826862 + created_at: "2022-02-24T00:00:00Z" + updated_at: "2022-02-24T00:00:00Z" +- model: games.game + pk: 3 + fields: + name: Cyberpunk 2077 + wikidata: Q3182559 + created_at: "2020-12-07T00:00:00Z" + updated_at: "2020-12-07T00:00:00Z" +- model: games.purchase + pk: 1 + fields: + games: [1] + platform: 1 + date_purchased: 2021-02-13 + date_refunded: null + created_at: "2021-02-13T00:00:00Z" + updated_at: "2021-02-13T00:00:00Z" +- model: games.purchase + pk: 2 + fields: + games: [2] + platform: 1 + date_purchased: 2022-02-24 + date_refunded: null + created_at: "2022-02-24T00:00:00Z" + updated_at: "2022-02-24T00:00:00Z" +- model: games.purchase + pk: 3 + fields: + games: [3] + platform: 1 + date_purchased: 2020-12-07 + date_refunded: null + created_at: "2020-12-07T00:00:00Z" + updated_at: "2020-12-07T00:00:00Z"