feat: Split off queue workers into a separate worker process

This commit is contained in:
Jesse Wierzbinski 2024-11-25 21:54:31 +01:00
parent 0b3e74107e
commit 1b98381242
No known key found for this signature in database
34 changed files with 987 additions and 676 deletions

View file

@ -1,31 +1,31 @@
name: Check Types
name: Check Types
on:
on:
push:
branches: ["*"]
branches: ["*"]
pull_request:
# The branches below must be a subset of the branches above
branches: ["main"]
# The branches below must be a subset of the branches above
branches: ["main"]
jobs:
jobs:
tests:
runs-on: ubuntu-latest
permissions:
contents: read
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install NPM packages
run: |
bun install
- name: Install NPM packages
run: |
bun install
- name: Run typechecks
run: |
bun run check
- name: Run typechecks
run: |
bun run check

View file

@ -1,74 +0,0 @@
name: Docker Build
on:
push:
branches: [ "main" ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ "main" ]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 # v3.0.0
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@v3 # v3.0.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5 # v5.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Get the commit hash
run: echo "GIT_COMMIT=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_ENV
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5 # v5.0.0
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_COMMIT=${{ env.GIT_COMMIT }}
provenance: mode=max
sbom: true
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max

72
.github/workflows/docker-server.yml vendored Normal file
View file

@ -0,0 +1,72 @@
name: Build Server Docker Image
on:
push:
branches: ["main"]
# Publish semver tags as releases.
tags: ["v*.*.*"]
pull_request:
branches: ["main"]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 # v3.0.0
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@v3 # v3.0.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5 # v5.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Get the commit hash
run: echo "GIT_COMMIT=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_ENV
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5 # v5.0.0
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_COMMIT=${{ env.GIT_COMMIT }}
provenance: mode=max
sbom: true
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max

73
.github/workflows/docker-worker.yml vendored Normal file
View file

@ -0,0 +1,73 @@
name: Build Worker Docker Image
on:
push:
branches: ["main"]
# Publish semver tags as releases.
tags: ["v*.*.*"]
pull_request:
branches: ["main"]
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 # v3.0.0
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@v3 # v3.0.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5 # v5.0.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Get the commit hash
run: echo "GIT_COMMIT=$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_ENV
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5 # v5.0.0
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_COMMIT=${{ env.GIT_COMMIT }}
file: ./Worker.Dockerfile
provenance: mode=max
sbom: true
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max

View file

@ -1,56 +1,56 @@
name: Deploy Docs to GitHub Pages
on:
push:
branches: [main]
push:
branches: [main]
workflow_dispatch:
workflow_dispatch:
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
contents: read
pages: write
id-token: write
contents: read
pages: write
id-token: write
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
group: pages
cancel-in-progress: false
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: oven-sh/setup-bun@v2
- name: Setup Pages
- uses: oven-sh/setup-bun@v2
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Install dependencies
run: bun install
uses: actions/configure-pages@v4
- name: Install dependencies
run: bun install
- name: Build with VitePress
run: bun run docs:build
- name: Build with VitePress
run: bun run docs:build
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: docs/.vitepress/dist
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: docs/.vitepress/dist
# Deployment job
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
needs: build
runs-on: ubuntu-latest
name: Deploy
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
# Deployment job
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
needs: build
runs-on: ubuntu-latest
name: Deploy
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

View file

@ -1,31 +1,31 @@
name: Lint & Format
on:
push:
branches: ["*"]
pull_request:
# The branches below must be a subset of the branches above
branches: ["main"]
push:
branches: ["*"]
pull_request:
# The branches below must be a subset of the branches above
branches: ["main"]
jobs:
tests:
runs-on: ubuntu-latest
permissions:
contents: read
tests:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install NPM packages
run: |
bun install
- name: Install NPM packages
run: |
bun install
- name: Run linting
run: |
bunx @biomejs/biome ci .
- name: Run linting
run: |
bunx @biomejs/biome ci .

View file

@ -2,7 +2,7 @@ name: Mirror to Codeberg
on: [push]
jobs:
mirror:
name: Mirror
uses: versia-pub/.github/.github/workflows/mirror.yml@main
secrets: inherit
mirror:
name: Mirror
uses: versia-pub/.github/.github/workflows/mirror.yml@main
secrets: inherit

View file

@ -1,25 +1,25 @@
name: Nix Build
name: Nix Build
on:
pull_request:
push:
branches: ["*"]
workflow_dispatch:
pull_request:
push:
branches: ["*"]
workflow_dispatch:
jobs:
check:
runs-on: ubuntu-latest
permissions:
id-token: "write"
contents: "read"
steps:
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@main
with:
extra-conf: accept-flake-config = true
- uses: DeterminateSystems/magic-nix-cache-action@main
- uses: DeterminateSystems/flake-checker-action@main
- name: Build default package
run: nix build .
- name: Check flakes
run: nix flake check --allow-import-from-derivation
check:
runs-on: ubuntu-latest
permissions:
id-token: "write"
contents: "read"
steps:
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@main
with:
extra-conf: accept-flake-config = true
- uses: DeterminateSystems/magic-nix-cache-action@main
- uses: DeterminateSystems/flake-checker-action@main
- name: Build default package
run: nix build .
- name: Check flakes
run: nix flake check --allow-import-from-derivation

View file

@ -1,50 +1,50 @@
name: Staging build bundle
on:
push:
branches: ["staging"]
push:
branches: ["staging"]
jobs:
tests:
runs-on: ubuntu-latest
permissions:
contents: read
tests:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install NPM packages
run: |
bun install
- name: Install NPM packages
run: |
bun install
- name: Build dist
run: |
bun run build
- name: Build dist
run: |
bun run build
- name: Bundle
run: |
mkdir bundle
cp -r dist bundle/
cp -r config bundle/
cp -r docs bundle/
cp -r CODE_OF_CONDUCT.md bundle/
cp -r CONTRIBUTING.md bundle/
cp -r README.md bundle/
cp -r flake.nix bundle/
cp -r shell.nix bundle/
cp -r flake.lock bundle/
cp -r LICENSE bundle/
cp -r SECURITY.md bundle/
tar cfJ archive.tar.xz bundle/
- name: Bundle
run: |
mkdir bundle
cp -r dist bundle/
cp -r config bundle/
cp -r docs bundle/
cp -r CODE_OF_CONDUCT.md bundle/
cp -r CONTRIBUTING.md bundle/
cp -r README.md bundle/
cp -r flake.nix bundle/
cp -r shell.nix bundle/
cp -r flake.lock bundle/
cp -r LICENSE bundle/
cp -r SECURITY.md bundle/
tar cfJ archive.tar.xz bundle/
- name: Upload
uses: actions/upload-artifact@v4
with:
name: staging-dist
path: archive.tar.xz
- name: Upload
uses: actions/upload-artifact@v4
with:
name: staging-dist
path: archive.tar.xz

View file

@ -1,59 +1,59 @@
name: Tests
on:
push:
branches: ["*"]
pull_request:
# The branches below must be a subset of the branches above
branches: ["main"]
push:
branches: ["*"]
pull_request:
# The branches below must be a subset of the branches above
branches: ["main"]
jobs:
tests:
runs-on: ubuntu-latest
services:
postgres:
image: ghcr.io/versia-pub/postgres:main
ports:
- 5432:5432
env:
POSTGRES_DB: versia
POSTGRES_USER: versia
POSTGRES_PASSWORD: versia
volumes:
- versia-data:/var/lib/postgresql/data
options: --health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
redis:
image: redis:latest
ports:
- 6379:6379
options: --health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
permissions:
contents: read
security-events: write
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
tests:
runs-on: ubuntu-latest
services:
postgres:
image: ghcr.io/versia-pub/postgres:main
ports:
- 5432:5432
env:
POSTGRES_DB: versia
POSTGRES_USER: versia
POSTGRES_PASSWORD: versia
volumes:
- versia-data:/var/lib/postgresql/data
options: --health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
redis:
image: redis:latest
ports:
- 6379:6379
options: --health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
permissions:
contents: read
security-events: write
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install NPM packages
run: |
bun install
- name: Install NPM packages
run: |
bun install
- name: Move workflow config to config folder
run: |
mv .github/config.workflow.toml config/config.toml
- name: Move workflow config to config folder
run: |
mv .github/config.workflow.toml config/config.toml
- name: Run tests
run: |
bun run test
- name: Run tests
run: |
bun run test