Skip to content

Commit 074766e

Browse files
wayneeseguinwayneeseguin
committed
fix: yaml.v3 map types in merge and NATS operators
 - Add support for map[string]interface{} in merge logic to handle yaml.v3 parsing - Fix NATS KV/object store YAML type conversion for yaml.v2 compatibility - Preserve string values with special characters in NATS KV store - Parse .yaml/.yml files from object store when no content-type is set - Handle empty KV values explicitly to return empty string instead of nil Fixes test failures in TestExamples, TestVaultNatsEdgeCases, and TestNatsOperator
1 parent d4b602e commit 074766e

15 files changed

Lines changed: 1973 additions & 27 deletions

‎Makefile‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,32 @@
1+
.PHONY: all vet test colortest clitests build integration help
2+
3+
# Default target shows help
4+
.DEFAULT_GOAL := help
5+
16
all: vet test build clitests
27

3-
vet:
8+
help: ## Show this help message
9+
@echo "Available targets:"
10+
@echo ""
11+
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}'
12+
@echo ""
13+
@echo "Use 'make <target>' to run a specific target."
14+
15+
vet: ## Run go vet on all packages
416
go list ./... | grep -v vendor | xargs go vet
517

6-
test:
18+
test: ## Run all unit tests
719
go list ./... | grep -v vendor | xargs go test
820

9-
colortest: build
21+
colortest: build ## Test color output functionality
1022
./assets/color_tester
1123

12-
clitests: build
24+
clitests: build ## Run CLI integration tests
1325
./assets/cli_tests
1426

15-
build:
27+
build: ## Build the graft binary
1628
go build ./cmd/graft
29+
30+
integration: build ## Run integration tests with external services (Vault, NATS)
31+
@echo "Running integration tests..."
32+
@scripts/integration

‎assets/vault-nats/base-config.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Base configuration with references to vault and nats
2+
app:
3+
name: "myapp"
4+
environment: (( grab meta.environment ))
5+
6+
database:
7+
host: (( grab meta.db_host ))
8+
port: (( grab meta.db_port ))
9+
username: (( vault "secret/database:username" ))
10+
password: (( vault "secret/database:password" ))
11+
12+
cache:
13+
type: "redis"
14+
host: (( nats "kv:config/cache/host" ))
15+
port: (( nats "kv:config/cache/port" ))
16+
auth:
17+
enabled: true
18+
token: (( vault "secret/redis:auth_token" ))
19+
20+
meta:
21+
environment: "dev"
22+
db_host: "localhost"
23+
db_port: 5432

‎assets/vault-nats/conditional-sources.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Testing conditional use of vault vs nats based on environment
2+
meta:
3+
use_vault: (( grab $USE_VAULT || false ))
4+
use_nats: (( grab $USE_NATS || true ))
5+
env: (( grab $ENVIRONMENT || "dev" ))
6+
7+
secrets:
8+
database:
9+
# Use vault if enabled, otherwise use nats
10+
username: (( ternary meta.use_vault (vault "secret/db:username") (nats "kv:secrets/db/username") ))
11+
password: (( ternary meta.use_vault (vault "secret/db:password") (nats "kv:secrets/db/password") ))
12+
13+
api_keys:
14+
# Try vault first, fallback to nats
15+
primary: (( vault "secret/api:primary_key" || nats "kv:api/keys/primary" ))
16+
secondary: (( vault "secret/api:secondary_key" || nats "kv:api/keys/secondary" ))
17+
18+
config:
19+
# Always use NATS for configuration
20+
app_settings: (( nats "obj:configs/app-settings.yaml" ))
21+
22+
# Use vault for sensitive feature flags
23+
feature_flags:
24+
premium_features: (( vault (concat "secret/features/" meta.env ":premium") ))
25+
beta_features: (( nats (concat "kv:features/" meta.env "/beta") ))

‎assets/vault-nats/dynamic-paths.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Testing dynamic path construction with vault and nats
2+
environments:
3+
- name: "dev"
4+
config:
5+
database:
6+
host: (( nats (concat "kv:env/" environments.0.name "/database/host") ))
7+
credentials: (( vault (concat "secret/" environments.0.name "/database:credentials") ))
8+
features:
9+
debug: true
10+
11+
- name: "staging"
12+
config:
13+
database:
14+
host: (( nats (concat "kv:env/" environments.1.name "/database/host") ))
15+
credentials: (( vault (concat "secret/" environments.1.name "/database:credentials") ))
16+
features:
17+
debug: false
18+
19+
- name: "production"
20+
config:
21+
database:
22+
host: (( nats (concat "kv:env/" environments.2.name "/database/host") ))
23+
credentials: (( vault (concat "secret/" environments.2.name "/database:credentials") ))
24+
features:
25+
debug: false
26+
27+
# Reference to dynamic environment
28+
current_env:
29+
name: (( grab meta.target_env || "dev" ))
30+
database:
31+
connection_string: (( concat "postgresql://" (vault (concat "secret/" current_env.name "/db:username")) ":" (vault (concat "secret/" current_env.name "/db:password")) "@" (nats (concat "kv:env/" current_env.name "/database/host")) "/" (nats (concat "kv:env/" current_env.name "/database/name")) ))

‎assets/vault-nats/multi-source.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Testing multiple vault/nats sources with fallbacks
2+
api:
3+
endpoints:
4+
- name: "auth"
5+
url: (( concat "https://" (nats "kv:services/auth/host") "/v1/auth" ))
6+
api_key: (( vault "secret/api/auth:key; secret/api/fallback:key" ))
7+
- name: "users"
8+
url: (( concat "https://" (nats "kv:services/users/host") "/v1/users" ))
9+
api_key: (( vault "secret/api/users:key" ))
10+
11+
monitoring:
12+
datadog:
13+
enabled: (( nats "kv:features/monitoring/datadog" ))
14+
api_key: (( vault "secret/datadog:api_key" ))
15+
app_key: (( vault "secret/datadog:app_key" ))
16+
17+
prometheus:
18+
enabled: (( nats "kv:features/monitoring/prometheus" ))
19+
endpoint: (( concat "http://" (nats "kv:services/prometheus/host") ":" (nats "kv:services/prometheus/port") ))

‎assets/vault-nats/targets-integration.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Testing vault and nats with target-specific configurations
2+
defaults:
3+
database:
4+
port: 5432
5+
ssl: true
6+
7+
# Target-specific configurations
8+
production:
9+
database:
10+
host: (( nats@production "kv:database/prod/host" ))
11+
username: (( vault@production "secret/prod/db:username" ))
12+
password: (( vault@production "secret/prod/db:password" ))
13+
ssl_cert: (( vault@production:nocache "secret/prod/db:ssl_cert" ))
14+
15+
staging:
16+
database:
17+
host: (( nats@staging "kv:database/staging/host" ))
18+
username: (( vault@staging "secret/staging/db:username" ))
19+
password: (( vault@staging "secret/staging/db:password" ))
20+
ssl: false
21+
22+
development:
23+
database:
24+
host: "localhost"
25+
username: "dev_user"
26+
password: "dev_pass"
27+
ssl: false
28+
29+
# Merge based on target
30+
database:
31+
host: (( grab (concat meta.target ".database.host") || defaults.database.host ))
32+
port: (( grab (concat meta.target ".database.port") || defaults.database.port ))
33+
username: (( grab (concat meta.target ".database.username") ))
34+
password: (( grab (concat meta.target ".database.password") ))
35+
ssl: (( grab (concat meta.target ".database.ssl") || defaults.database.ssl ))
36+
ssl_cert: (( grab (concat meta.target ".database.ssl_cert") || null ))
37+
38+
meta:
39+
target: (( grab $TARGET || "development" ))

Dockerfile renamed to docker/Dockerfile

File renamed without changes.

‎docs/development/integration-testing.md‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Integration Testing
2+
3+
This document describes how to run integration tests for graft's external operators (Vault and NATS).
4+
5+
## Overview
6+
7+
The integration tests verify that graft correctly interacts with external services like HashiCorp Vault and NATS. These tests use Docker containers to spin up real instances of the services, populate them with test data, and then run graft commands to verify the operators work correctly.
8+
9+
## Prerequisites
10+
11+
- Docker must be installed and running
12+
- The `graft` binary must be built (`make build`)
13+
- Ports 8200 (Vault) and 4222 (NATS) must be available
14+
- For NATS tests: `nats` CLI tool (will be auto-installed if missing)
15+
16+
## Running Tests
17+
18+
### Run All Integration Tests
19+
20+
```bash
21+
make integration
22+
```
23+
24+
### Run Specific Tests
25+
26+
```bash
27+
# Run only Vault tests
28+
scripts/integration --no-nats
29+
30+
# Run only NATS tests
31+
scripts/integration --no-vault
32+
33+
# Run with verbose output
34+
scripts/integration -v
35+
36+
# Keep containers running after tests (for debugging)
37+
scripts/integration --keep
38+
```
39+
40+
## Test Coverage
41+
42+
### Vault Operator Tests
43+
44+
1. **Basic secret retrieval** - Fetches username/password from Vault
45+
2. **Secret with specific key** - Uses `:key` syntax to get specific fields
46+
3. **Multiple vault references** - Tests multiple operators in one document
47+
4. **Vault with defaults** - Tests `||` operator for fallback values
48+
5. **Invalid path handling** - Verifies error handling for missing secrets
49+
6. **Target-specific vault** - Tests `@target` syntax for multi-instance support
50+
7. **Vault in complex merge** - Tests vault operators across merged documents
51+
52+
### NATS Operator Tests
53+
54+
1. **Basic KV retrieval** - Fetches values from JetStream KV store
55+
2. **Object store YAML retrieval** - Fetches and parses YAML from object store
56+
3. **Multiple NATS references** - Tests multiple operators in one document
57+
4. **Invalid key handling** - Verifies error handling for missing keys
58+
5. **Target-specific NATS** - Tests `@target` syntax for multi-instance support
59+
6. **NATS with timeout** - Tests custom timeout configuration
60+
61+
## Test Data
62+
63+
### Vault Test Data
64+
65+
The integration tests create the following secrets in Vault:
66+
67+
- `secret/test/credentials` - Contains username, password, api_key
68+
- `secret/test/database` - Contains host, port, name
69+
- `secret/test/features` - Contains enabled (bool), max_users (int), tier (string)
70+
71+
### NATS Test Data
72+
73+
The integration tests create:
74+
75+
**KV Store (`test-bucket`)**:
76+
- `config.host` = "api.example.com"
77+
- `config.port` = "8080"
78+
- `config.timeout` = "30"
79+
- `features.auth` = "enabled"
80+
- `features.cache` = "true"
81+
82+
**Object Store (`test-objects`)**:
83+
- `config.yml` - A YAML file with version, app, and env fields
84+
85+
## Environment Variables
86+
87+
The integration tests use these environment variables:
88+
89+
### Vault
90+
- `VAULT_ADDR` - Vault server address (default: http://localhost:8200)
91+
- `VAULT_TOKEN` - Vault authentication token
92+
- `VAULT_<TARGET>_ADDR` - Target-specific Vault address
93+
- `VAULT_<TARGET>_TOKEN` - Target-specific Vault token
94+
95+
### NATS
96+
- `NATS_URL` - NATS server URL (default: nats://localhost:4222)
97+
- `NATS_TIMEOUT` - Connection timeout
98+
- `NATS_<TARGET>_URL` - Target-specific NATS URL
99+
100+
## Output Format
101+
102+
The tests use TAP (Test Anything Protocol) format for output:
103+
104+
```
105+
1..7 # Vault operator tests
106+
ok 1 - Vault basic secret retrieval
107+
ok 2 - Retrieved correct username
108+
ok 3 - Retrieved correct password
109+
...
110+
```
111+
112+
## Troubleshooting
113+
114+
### Port Already in Use
115+
116+
If you see "Port 8200 is already in use" or similar:
117+
- Check for running Vault/NATS instances: `docker ps`
118+
- Stop conflicting services or use different ports
119+
120+
### Docker Not Found
121+
122+
Ensure Docker is installed and the Docker daemon is running:
123+
```bash
124+
docker --version
125+
docker ps
126+
```
127+
128+
### Tests Fail to Connect
129+
130+
If tests fail with connection errors:
131+
1. Check Docker is running
132+
2. Verify no firewall is blocking localhost connections
133+
3. Run with `-v` flag for verbose output
134+
4. Use `--keep` to inspect running containers
135+
136+
### Debugging Failed Tests
137+
138+
To debug failing tests:
139+
1. Run with verbose mode: `scripts/integration -v`
140+
2. Keep containers running: `scripts/integration --keep`
141+
3. Inspect container logs: `docker logs graft-test-vault-<PID>`
142+
4. Manually test with graft: `VAULT_ADDR=http://localhost:8200 VAULT_TOKEN=root-token-for-testing ./graft merge test.yml`

‎pkg/graft/interfaces.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,11 @@ func (op *Opcall) Where() *tree.Cursor {
140140
return op.where
141141
}
142142

143+
// SetWhere sets the cursor location for this operator call
144+
func (op *Opcall) SetWhere(cursor *tree.Cursor) {
145+
op.where = cursor
146+
}
147+
143148
// Src returns the source string for this operator call
144149
func (op *Opcall) Src() string {
145150
return op.src

0 commit comments

Comments
 (0)