Skip to content

Merge pull request #18602 from Budibase/feat/move-minio-to-s3 #37644

Merge pull request #18602 from Budibase/feat/move-minio-to-s3

Merge pull request #18602 from Budibase/feat/move-minio-to-s3 #37644

Workflow file for this run

name: Budibase CI
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
branches:
- master
pull_request:
workflow_dispatch:
workflow_call:
inputs:
run_as_oss:
type: boolean
required: false
description: Force running checks as if it was an OSS contributor
default: false
permissions:
contents: read
env:
BRANCH: ${{ github.event.pull_request.head.ref }}
BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
NX_BASE_BRANCH: origin/${{ github.base_ref }}
ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }}
HUSKY: 0
jobs:
install-deps:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-install-deps-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
id: hoisted-modules-cache-restore
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
- name: Install dependencies
if: steps.hoisted-modules-cache-restore.outputs.cache-hit != 'true'
run: yarn --frozen-lockfile
- name: Save hoisted node_modules cache
if: steps.hoisted-modules-cache-restore.outputs.cache-hit != 'true'
uses: actions/cache/save@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
lint:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-lint-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- run: yarn lint
litellm-version-sync:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-litellm-version-sync-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Verify LiteLLM versions are synchronized
run: yarn check:litellm-version
build:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-build-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs:
- install-deps
- compute-affected-jobs
if: needs.compute-affected-jobs.outputs.run_build == 'true'
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: Fetch base branch for affected tasks
if: env.ONLY_AFFECTED_TASKS == 'true'
run: git fetch --no-tags --depth=1 origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
# Run build all the projects
- name: Build
run: yarn build
# Check the types of the projects built via esbuild
- name: Check types
run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn check:types --since=${{ env.NX_BASE_BRANCH }}
else
yarn check:types
fi
missing-deps:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-missing-deps-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Check missing deps
run: yarn check:missing-deps
helm-lint:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-helm-lint-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: azure/setup-helm@v3
- run: cd charts/budibase && helm lint .
test-libraries:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-test-libraries-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs:
- install-deps
- compute-affected-jobs
if: needs.compute-affected-jobs.outputs.run_test_libraries == 'true'
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: Fetch base branch for affected tasks
if: env.ONLY_AFFECTED_TASKS == 'true'
run: git fetch --no-tags --depth=1 origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
docker pull redis &
wait $(jobs -p)
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Test
run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --ignore=@budibase/upgrade-tests --ignore=@budibase/client --ignore=@budibase/frontend-core --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --scope=@budibase/client --scope=@budibase/frontend-core --since=${{ env.NX_BASE_BRANCH }}
else
yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --ignore=@budibase/upgrade-tests --ignore=@budibase/client --ignore=@budibase/frontend-core --no-prefix -- --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --scope=@budibase/client --scope=@budibase/frontend-core --no-prefix
fi
compute-affected-jobs:
name: Compute affected jobs
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-compute-affected-jobs-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
outputs:
run_build: ${{ steps.affected.outputs.run_build }}
run_test_libraries: ${{ steps.affected.outputs.run_test_libraries }}
run_worker_tests: ${{ steps.affected.outputs.run_worker_tests }}
run_server_tests: ${{ steps.affected.outputs.run_server_tests }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: Fetch base branch for affected tasks
if: env.ONLY_AFFECTED_TASKS == 'true'
run: git fetch --no-tags --depth=1 origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Check affected tests
id: affected
run: |
# Defaults for non-PR workflows.
RUN_BUILD="true"
RUN_TEST_LIBRARIES="true"
# This gates whole package test jobs (worker/server) based on Nx affected projects.
# Example: if a PR changes only FE projects, server/worker test jobs are skipped.
RUN_WORKER_TESTS="true"
RUN_SERVER_TESTS="true"
if ${{ env.ONLY_AFFECTED_TASKS }}; then
AFFECTED_BUILD=$(yarn --silent nx show projects --affected -t build --base=${{ env.NX_BASE_BRANCH }})
if [ -z "$AFFECTED_BUILD" ]; then
RUN_BUILD="false"
fi
AFFECTED_LIBRARY_TESTS=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} --exclude='@budibase/server,@budibase/worker,@budibase/upgrade-tests')
if [ -z "$AFFECTED_LIBRARY_TESTS" ]; then
RUN_TEST_LIBRARIES="false"
fi
AFFECTED_WORKER=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
if [ -z "$AFFECTED_WORKER" ]; then
RUN_WORKER_TESTS="false"
fi
AFFECTED_SERVER=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
if [ -z "$AFFECTED_SERVER" ]; then
RUN_SERVER_TESTS="false"
fi
fi
echo "run_build=$RUN_BUILD" >> $GITHUB_OUTPUT
echo "run_test_libraries=$RUN_TEST_LIBRARIES" >> $GITHUB_OUTPUT
echo "run_worker_tests=$RUN_WORKER_TESTS" >> $GITHUB_OUTPUT
echo "run_server_tests=$RUN_SERVER_TESTS" >> $GITHUB_OUTPUT
echo "run_build=$RUN_BUILD"
echo "run_test_libraries=$RUN_TEST_LIBRARIES"
echo "run_worker_tests=$RUN_WORKER_TESTS"
echo "run_server_tests=$RUN_SERVER_TESTS"
test-worker:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-test-worker-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs:
- install-deps
- compute-affected-jobs
steps:
- name: Skip worker tests
if: needs.compute-affected-jobs.outputs.run_worker_tests != 'true'
run: echo "Skipping worker tests because @budibase/worker is not affected."
- name: Checkout repo
if: needs.compute-affected-jobs.outputs.run_worker_tests == 'true'
uses: actions/checkout@v6
- name: Use Node.js 22.x
if: needs.compute-affected-jobs.outputs.run_worker_tests == 'true'
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
if: needs.compute-affected-jobs.outputs.run_worker_tests == 'true'
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Test worker
if: needs.compute-affected-jobs.outputs.run_worker_tests == 'true'
run: |
cd packages/worker
yarn test --reporters=default --reporters=github-actions
test-server:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-test-server-${{ matrix.datasource }}-${{ matrix.shard_index }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: false
needs:
- install-deps
- compute-affected-jobs
strategy:
fail-fast: false
matrix:
include:
# Use uneven sharding to balance runtime across datasources.
# Oracle and non-datasource ("none") suites are slower, so they use more shards.
- datasource: mssql
shard_index: 1
shard_total: 1
- datasource: mysql
shard_index: 1
shard_total: 1
- datasource: postgres
shard_index: 1
shard_total: 1
- datasource: postgres_legacy
shard_index: 1
shard_total: 1
- datasource: mongodb
shard_index: 1
shard_total: 1
- datasource: mariadb
shard_index: 1
shard_total: 1
- datasource: elasticsearch
shard_index: 1
shard_total: 1
- datasource: dynamodb
shard_index: 1
shard_total: 1
- datasource: sqs
shard_index: 1
shard_total: 2
- datasource: sqs
shard_index: 2
shard_total: 2
- datasource: none
shard_index: 1
shard_total: 3
- datasource: none
shard_index: 2
shard_total: 3
- datasource: none
shard_index: 3
shard_total: 3
- datasource: oracle
shard_index: 1
shard_total: 4
- datasource: oracle
shard_index: 2
shard_total: 4
- datasource: oracle
shard_index: 3
shard_total: 4
- datasource: oracle
shard_index: 4
shard_total: 4
steps:
- name: Skip server tests
if: needs.compute-affected-jobs.outputs.run_server_tests != 'true'
run: echo "Skipping server tests because @budibase/server is not affected."
- name: Checkout repo
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
uses: actions/checkout@v6
- name: Use Node.js 22.x
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Load dotenv
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
id: dotenv
uses: falti/dotenv-action@v1.2.0
with:
path: ./packages/server/images-sha.env
- name: Pull testcontainers images
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
run: |
if [ "${{ matrix.datasource }}" == "mssql" ]; then
docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
elif [ "${{ matrix.datasource }}" == "mysql" ]; then
docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
elif [ "${{ matrix.datasource }}" == "postgres" ]; then
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
elif [ "${{ matrix.datasource }}" == "oracle" ]; then
docker pull budibase/oracle-database:23.2-slim-faststart
elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
docker pull postgres:9.5.25
elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
elif [ "${{ matrix.datasource }}" == "dynamodb" ]; then
docker pull amazon/dynamodb-local@${{ steps.dotenv.outputs.DYNAMODB_SHA }}
elif [ "${{ matrix.datasource }}" == "none" ]; then
docker pull ${{ steps.dotenv.outputs.KEYCLOAK_IMAGE }} &
fi
docker pull minio/minio &
docker pull redis &
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
wait $(jobs -p)
- name: Build client library - necessary for component tests
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true' && matrix.datasource == 'none'
run: yarn build:client
- name: Test server
if: needs.compute-affected-jobs.outputs.run_server_tests == 'true'
env:
DATASOURCE: ${{ matrix.datasource }}
JEST_SHARD_INDEX: ${{ matrix.shard_index }}
JEST_SHARD_TOTAL: ${{ matrix.shard_total }}
JEST_MAX_WORKERS: 4
# DEBUG: testcontainers*
run: |
FILTER="./src/tests/filters/datasource-tests.js"
if [ "${{ matrix.datasource }}" == "none" ]; then
FILTER="./src/tests/filters/non-datasource-tests.js"
fi
SHARD_ARG=""
if [ "$JEST_SHARD_TOTAL" -gt "1" ]; then
SHARD_ARG="--shard=$JEST_SHARD_INDEX/$JEST_SHARD_TOTAL"
echo "Running shard $JEST_SHARD_INDEX/$JEST_SHARD_TOTAL for datasource '$DATASOURCE'"
fi
cd packages/server
yarn test --filter $FILTER --reporters=default --reporters=github-actions $SHARD_ARG
check-lockfile:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-check-lockfile-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- run: yarn install
- name: Check for yarn.lock changes
run: |
if [[ $(git status --porcelain) == *"yarn.lock"* ]]; then
echo "yarn.lock file needs to be modified. Please update it locally and commit the changes."
exit 1
else
echo "yarn.lock file is unchanged."
fi
check-openapi-specs:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-check-openapi-specs-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Generate OpenAPI specs
run: yarn build:specs
- name: Check for uncommitted changes
run: |
if [[ -n $(git status --porcelain packages/server) ]]; then
echo "OpenAPI specs are not up to date. Please run 'yarn build:specs' and commit the changes."
git status --porcelain packages/server
git diff packages/server
exit 1
else
echo "OpenAPI specs are up to date."
fi
build-database-image:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-build-database-image-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build budibase/database image
uses: docker/build-push-action@v5
with:
context: hosting/couchdb
push: false
load: true
tags: budibase/database:ci-test
cache-from: type=gha
cache-to: type=gha,mode=max
check-client-build-size:
runs-on: ubuntu-24.04
concurrency:
group: ${{ github.workflow }}-check-client-build-size-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
needs: install-deps
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Use Node.js 22.x
uses: actions/setup-node@v6
with:
node-version: 22.x
- name: Restore hoisted node_modules cache
uses: actions/cache/restore@v5
with:
path: |
node_modules
packages/*/node_modules
key: ${{ runner.os }}-node22-hoisted-node-modules-${{ hashFiles('yarn.lock') }}
fail-on-cache-miss: true
- name: Build client library
run: yarn build:client
- name: Check client build size
run: |
CLIENT_FILE="packages/client/dist/budibase-client.js"
if [ ! -f "$CLIENT_FILE" ]; then
echo "Error: Client build file not found at $CLIENT_FILE"
exit 1
fi
SIZE_BYTES=$(stat -c%s "$CLIENT_FILE")
SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE_BYTES / 1024 / 1024}")
MAX_SIZE_BYTES=$((8 * 1024 * 1024))
echo "Client build size: ${SIZE_MB}MB"
echo "Maximum allowed size: 8MB"
if [ $SIZE_BYTES -gt $MAX_SIZE_BYTES ]; then
echo "Client build size (${SIZE_MB}MB) exceeds maximum allowed size (8MB)"
exit 1
else
echo "Client build size (${SIZE_MB}MB) is within acceptable limits"
fi