Merge pull request #435 from databasus/develop

FEATURE (agent): Add postgres verification and e2e tests for agent
This commit is contained in:
Rostislav Dugin
2026-03-14 12:43:51 +03:00
committed by GitHub
24 changed files with 965 additions and 47 deletions

View File

@@ -164,6 +164,25 @@ jobs:
cd agent
go test -count=1 -failfast ./internal/...
e2e-agent:
runs-on: self-hosted
needs: [lint-agent]
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Run e2e tests
run: |
cd agent
make e2e
- name: Cleanup
if: always()
run: |
cd agent/e2e
docker compose down -v --rmi local || true
rm -rf artifacts || true
test-backend:
runs-on: self-hosted
needs: [lint-backend]
@@ -497,7 +516,7 @@ jobs:
runs-on: self-hosted
container:
image: node:20
needs: [test-backend, test-frontend, test-agent]
needs: [test-backend, test-frontend, test-agent, e2e-agent]
if: ${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, '[skip-release]') }}
outputs:
should_release: ${{ steps.version_bump.outputs.should_release }}
@@ -590,7 +609,7 @@ jobs:
build-only:
runs-on: self-hosted
needs: [test-backend, test-frontend, test-agent]
needs: [test-backend, test-frontend, test-agent, e2e-agent]
if: ${{ github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[skip-release]') }}
steps:
- name: Clean workspace

View File

@@ -1,3 +1,5 @@
.PHONY: run build test lint e2e e2e-clean
# Usage: make run ARGS="start --pg-host localhost"
run:
go run cmd/main.go $(ARGS)
@@ -10,3 +12,15 @@ test:
lint:
golangci-lint fmt ./cmd/... ./internal/... && golangci-lint run ./cmd/... ./internal/...
e2e:
cd e2e && docker compose build
cd e2e && docker compose run --rm e2e-agent-builder
cd e2e && docker compose up -d e2e-postgres e2e-mock-server
cd e2e && docker compose run --rm e2e-agent-runner
cd e2e && docker compose run --rm e2e-agent-docker
cd e2e && docker compose down -v
e2e-clean:
cd e2e && docker compose down -v --rmi local
rm -rf e2e/artifacts

View File

@@ -3,6 +3,7 @@ package main
import (
"flag"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
@@ -116,12 +117,7 @@ func printUsage() {
fmt.Fprintln(os.Stderr, " version Print agent version")
}
func runUpdateCheck(host string, isSkipUpdate, isDev bool, log interface {
Info(string, ...any)
Warn(string, ...any)
Error(string, ...any)
},
) {
func runUpdateCheck(host string, isSkipUpdate, isDev bool, log *slog.Logger) {
if isSkipUpdate {
return
}

1
agent/e2e/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
artifacts/

View File

@@ -0,0 +1,13 @@
# Builds agent binaries with different versions so
# we can test upgrade behavior (v1 -> v2)
FROM golang:1.26.1-alpine AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -ldflags "-X main.Version=v1.0.0" -o /out/agent-v1 ./cmd/main.go
RUN CGO_ENABLED=0 go build -ldflags "-X main.Version=v2.0.0" -o /out/agent-v2 ./cmd/main.go
FROM alpine:3.21
COPY --from=build /out/ /out/
CMD ["cp", "-v", "/out/agent-v1", "/out/agent-v2", "/artifacts/"]

View File

@@ -0,0 +1,8 @@
# Runs pg_basebackup-via-docker-exec test (test 5) which tests
# that the agent can connect to Postgres inside Docker container
FROM docker:27-cli
RUN apk add --no-cache bash curl
WORKDIR /tmp
ENTRYPOINT []

View File

@@ -0,0 +1,14 @@
# Runs upgrade and host-mode pg_basebackup tests (tests 1-4). Needs
# Postgres client tools to be installed inside the system
FROM debian:bookworm-slim
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates curl gnupg2 postgresql-common && \
/usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y && \
apt-get install -y --no-install-recommends \
postgresql-client-17 && \
rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
ENTRYPOINT []

View File

@@ -0,0 +1,10 @@
# Mock databasus API server for version checks and binary downloads. Just
# serves static responses and files from the `artifacts` directory.
FROM golang:1.26.1-alpine AS build
WORKDIR /app
COPY mock-server/main.go .
RUN CGO_ENABLED=0 go build -o mock-server main.go
FROM alpine:3.21
COPY --from=build /app/mock-server /usr/local/bin/mock-server
ENTRYPOINT ["mock-server"]

View File

@@ -0,0 +1,84 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
)
type server struct {
mu sync.RWMutex
version string
binaryPath string
}
func main() {
version := "v2.0.0"
binaryPath := "/artifacts/agent-v2"
port := "4050"
s := &server{version: version, binaryPath: binaryPath}
http.HandleFunc("/api/v1/system/version", s.handleVersion)
http.HandleFunc("/api/v1/system/agent", s.handleAgentDownload)
http.HandleFunc("/mock/set-version", s.handleSetVersion)
http.HandleFunc("/health", s.handleHealth)
addr := ":" + port
log.Printf("Mock server starting on %s (version=%s, binary=%s)", addr, version, binaryPath)
if err := http.ListenAndServe(addr, nil); err != nil {
log.Fatalf("Server failed: %v", err)
}
}
func (s *server) handleVersion(w http.ResponseWriter, r *http.Request) {
s.mu.RLock()
v := s.version
s.mu.RUnlock()
log.Printf("GET /api/v1/system/version -> %s", v)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"version": v})
}
func (s *server) handleAgentDownload(w http.ResponseWriter, r *http.Request) {
s.mu.RLock()
path := s.binaryPath
s.mu.RUnlock()
log.Printf("GET /api/v1/system/agent (arch=%s) -> serving %s", r.URL.Query().Get("arch"), path)
http.ServeFile(w, r, path)
}
func (s *server) handleSetVersion(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "POST only", http.StatusMethodNotAllowed)
return
}
var body struct {
Version string `json:"version"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
s.mu.Lock()
s.version = body.Version
s.mu.Unlock()
log.Printf("POST /mock/set-version -> %s", body.Version)
fmt.Fprintf(w, "version set to %s", body.Version)
}
func (s *server) handleHealth(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}

View File

@@ -0,0 +1,48 @@
#!/bin/bash
set -euo pipefail
MODE="${1:-host}"
SCRIPT_DIR="$(dirname "$0")"
PASSED=0
FAILED=0
run_test() {
local name="$1"
local script="$2"
echo ""
echo "========================================"
echo " $name"
echo "========================================"
if bash "$script"; then
echo " PASSED: $name"
PASSED=$((PASSED + 1))
else
echo " FAILED: $name"
FAILED=$((FAILED + 1))
fi
}
if [ "$MODE" = "host" ]; then
run_test "Test 1: Upgrade success (v1 -> v2)" "$SCRIPT_DIR/test-upgrade-success.sh"
run_test "Test 2: Upgrade skip (version matches)" "$SCRIPT_DIR/test-upgrade-skip.sh"
run_test "Test 3: pg_basebackup in PATH" "$SCRIPT_DIR/test-pg-host-path.sh"
run_test "Test 4: pg_basebackup via bindir" "$SCRIPT_DIR/test-pg-host-bindir.sh"
elif [ "$MODE" = "docker" ]; then
run_test "Test 5: pg_basebackup via docker exec" "$SCRIPT_DIR/test-pg-docker-exec.sh"
else
echo "Unknown mode: $MODE (expected 'host' or 'docker')"
exit 1
fi
echo ""
echo "========================================"
echo " Results: $PASSED passed, $FAILED failed"
echo "========================================"
if [ "$FAILED" -gt 0 ]; then
exit 1
fi

View File

@@ -0,0 +1,51 @@
#!/bin/bash
set -euo pipefail
ARTIFACTS="/opt/agent/artifacts"
AGENT="/tmp/test-agent"
PG_CONTAINER="e2e-agent-postgres"
# Copy agent binary
cp "$ARTIFACTS/agent-v1" "$AGENT"
chmod +x "$AGENT"
# Verify docker CLI works and PG container is accessible
if ! docker exec "$PG_CONTAINER" pg_basebackup --version > /dev/null 2>&1; then
echo "FAIL: Cannot reach pg_basebackup inside container $PG_CONTAINER (test setup issue)"
exit 1
fi
# Run start with --skip-update and pg-type=docker
echo "Running agent start (pg_basebackup via docker exec)..."
OUTPUT=$("$AGENT" start \
--skip-update \
--databasus-host http://e2e-mock-server:4050 \
--db-id test-db-id \
--token test-token \
--pg-host e2e-postgres \
--pg-port 5432 \
--pg-user testuser \
--pg-password testpassword \
--wal-dir /tmp/wal \
--pg-type docker \
--pg-docker-container-name "$PG_CONTAINER" 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ "$EXIT_CODE" -ne 0 ]; then
echo "FAIL: Agent exited with code $EXIT_CODE"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "pg_basebackup verified (docker)"; then
echo "FAIL: Expected output to contain 'pg_basebackup verified (docker)'"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "PostgreSQL connection verified"; then
echo "FAIL: Expected output to contain 'PostgreSQL connection verified'"
exit 1
fi
echo "pg_basebackup found via docker exec and DB connection verified"

View File

@@ -0,0 +1,57 @@
#!/bin/bash
set -euo pipefail
ARTIFACTS="/opt/agent/artifacts"
AGENT="/tmp/test-agent"
CUSTOM_BIN_DIR="/opt/pg/bin"
# Copy agent binary
cp "$ARTIFACTS/agent-v1" "$AGENT"
chmod +x "$AGENT"
# Move pg_basebackup out of PATH into custom directory
mkdir -p "$CUSTOM_BIN_DIR"
cp "$(which pg_basebackup)" "$CUSTOM_BIN_DIR/pg_basebackup"
# Hide the system one by prepending an empty dir to PATH
export PATH="/opt/empty-path:$PATH"
mkdir -p /opt/empty-path
# Verify pg_basebackup is NOT directly callable from default location
# (we copied it, but the original is still there in debian — so we test
# that the agent uses the custom dir, not PATH, by checking the output)
# Run start with --skip-update and custom bin dir
echo "Running agent start (pg_basebackup via --pg-host-bin-dir)..."
OUTPUT=$("$AGENT" start \
--skip-update \
--databasus-host http://e2e-mock-server:4050 \
--db-id test-db-id \
--token test-token \
--pg-host e2e-postgres \
--pg-port 5432 \
--pg-user testuser \
--pg-password testpassword \
--wal-dir /tmp/wal \
--pg-type host \
--pg-host-bin-dir "$CUSTOM_BIN_DIR" 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ "$EXIT_CODE" -ne 0 ]; then
echo "FAIL: Agent exited with code $EXIT_CODE"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "pg_basebackup verified"; then
echo "FAIL: Expected output to contain 'pg_basebackup verified'"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "PostgreSQL connection verified"; then
echo "FAIL: Expected output to contain 'PostgreSQL connection verified'"
exit 1
fi
echo "pg_basebackup found via custom bin dir and DB connection verified"

View File

@@ -0,0 +1,49 @@
#!/bin/bash
set -euo pipefail
ARTIFACTS="/opt/agent/artifacts"
AGENT="/tmp/test-agent"
# Copy agent binary
cp "$ARTIFACTS/agent-v1" "$AGENT"
chmod +x "$AGENT"
# Verify pg_basebackup is in PATH
if ! which pg_basebackup > /dev/null 2>&1; then
echo "FAIL: pg_basebackup not found in PATH (test setup issue)"
exit 1
fi
# Run start with --skip-update and pg-type=host
echo "Running agent start (pg_basebackup in PATH)..."
OUTPUT=$("$AGENT" start \
--skip-update \
--databasus-host http://e2e-mock-server:4050 \
--db-id test-db-id \
--token test-token \
--pg-host e2e-postgres \
--pg-port 5432 \
--pg-user testuser \
--pg-password testpassword \
--wal-dir /tmp/wal \
--pg-type host 2>&1)
EXIT_CODE=$?
echo "$OUTPUT"
if [ "$EXIT_CODE" -ne 0 ]; then
echo "FAIL: Agent exited with code $EXIT_CODE"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "pg_basebackup verified"; then
echo "FAIL: Expected output to contain 'pg_basebackup verified'"
exit 1
fi
if ! echo "$OUTPUT" | grep -q "PostgreSQL connection verified"; then
echo "FAIL: Expected output to contain 'PostgreSQL connection verified'"
exit 1
fi
echo "pg_basebackup found in PATH and DB connection verified"

View File

@@ -0,0 +1,51 @@
#!/bin/bash
set -euo pipefail
ARTIFACTS="/opt/agent/artifacts"
AGENT="/tmp/test-agent"
# Set mock server to return v1.0.0 (same as agent)
curl -sf -X POST http://e2e-mock-server:4050/mock/set-version \
-H "Content-Type: application/json" \
-d '{"version":"v1.0.0"}'
# Copy v1 binary to writable location
cp "$ARTIFACTS/agent-v1" "$AGENT"
chmod +x "$AGENT"
# Verify initial version
VERSION=$("$AGENT" version)
if [ "$VERSION" != "v1.0.0" ]; then
echo "FAIL: Expected initial version v1.0.0, got $VERSION"
exit 1
fi
# Run start — agent should see version matches and skip upgrade
echo "Running agent start (expecting upgrade skip)..."
OUTPUT=$("$AGENT" start \
--databasus-host http://e2e-mock-server:4050 \
--db-id test-db-id \
--token test-token \
--pg-host e2e-postgres \
--pg-port 5432 \
--pg-user testuser \
--pg-password testpassword \
--wal-dir /tmp/wal \
--pg-type host 2>&1) || true
echo "$OUTPUT"
# Verify output contains "up to date"
if ! echo "$OUTPUT" | grep -qi "up to date"; then
echo "FAIL: Expected output to contain 'up to date'"
exit 1
fi
# Verify binary is still v1
VERSION=$("$AGENT" version)
if [ "$VERSION" != "v1.0.0" ]; then
echo "FAIL: Expected version v1.0.0 (unchanged), got $VERSION"
exit 1
fi
echo "Upgrade correctly skipped, version still $VERSION"

View File

@@ -0,0 +1,52 @@
#!/bin/bash
set -euo pipefail
ARTIFACTS="/opt/agent/artifacts"
AGENT="/tmp/test-agent"
# Ensure mock server returns v2.0.0
curl -sf -X POST http://e2e-mock-server:4050/mock/set-version \
-H "Content-Type: application/json" \
-d '{"version":"v2.0.0"}'
# Copy v1 binary to writable location
cp "$ARTIFACTS/agent-v1" "$AGENT"
chmod +x "$AGENT"
# Verify initial version
VERSION=$("$AGENT" version)
if [ "$VERSION" != "v1.0.0" ]; then
echo "FAIL: Expected initial version v1.0.0, got $VERSION"
exit 1
fi
echo "Initial version: $VERSION"
# Run start — agent will:
# 1. Fetch version from mock (v2.0.0 != v1.0.0)
# 2. Download v2 binary from mock
# 3. Replace itself on disk
# 4. Re-exec with same args
# 5. Re-exec'd v2 fetches version (v2.0.0 == v2.0.0) → skips update
# 6. Proceeds to start → verifies pg_basebackup + DB → exits 0 (stub)
echo "Running agent start (expecting upgrade v1 -> v2)..."
OUTPUT=$("$AGENT" start \
--databasus-host http://e2e-mock-server:4050 \
--db-id test-db-id \
--token test-token \
--pg-host e2e-postgres \
--pg-port 5432 \
--pg-user testuser \
--pg-password testpassword \
--wal-dir /tmp/wal \
--pg-type host 2>&1) || true
echo "$OUTPUT"
# Verify binary on disk is now v2
VERSION=$("$AGENT" version)
if [ "$VERSION" != "v2.0.0" ]; then
echo "FAIL: Expected upgraded version v2.0.0, got $VERSION"
exit 1
fi
echo "Binary upgraded successfully to $VERSION"

View File

@@ -2,10 +2,18 @@ module databasus-agent
go 1.26.1
require github.com/stretchr/testify v1.11.1
require (
github.com/jackc/pgx/v5 v5.8.0
github.com/stretchr/testify v1.11.1
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
golang.org/x/text v0.29.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -1,10 +1,35 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -3,6 +3,7 @@ package config
import (
"encoding/json"
"flag"
"fmt"
"os"
"databasus-agent/internal/logger"
@@ -13,9 +14,18 @@ var log = logger.GetLogger()
const configFileName = "databasus.json"
type Config struct {
DatabasusHost string `json:"databasusHost"`
DbID string `json:"dbId"`
Token string `json:"token"`
DatabasusHost string `json:"databasusHost"`
DbID string `json:"dbId"`
Token string `json:"token"`
PgHost string `json:"pgHost"`
PgPort int `json:"pgPort"`
PgUser string `json:"pgUser"`
PgPassword string `json:"pgPassword"`
PgType string `json:"pgType"`
PgHostBinDir string `json:"pgHostBinDir"`
PgDockerContainerName string `json:"pgDockerContainerName"`
WalDir string `json:"walDir"`
IsDeleteWalAfterUpload *bool `json:"deleteWalAfterUpload"`
flags parsedFlags
}
@@ -24,15 +34,24 @@ type Config struct {
// and overrides JSON values with any explicitly provided CLI flags.
func (c *Config) LoadFromJSONAndArgs(fs *flag.FlagSet, args []string) {
c.loadFromJSON()
c.applyDefaults()
c.initSources()
c.flags.host = fs.String(
c.flags.databasusHost = fs.String(
"databasus-host",
"",
"Databasus server URL (e.g. http://your-server:4005)",
)
c.flags.dbID = fs.String("db-id", "", "Database ID")
c.flags.token = fs.String("token", "", "Agent token")
c.flags.pgHost = fs.String("pg-host", "", "PostgreSQL host")
c.flags.pgPort = fs.Int("pg-port", 0, "PostgreSQL port")
c.flags.pgUser = fs.String("pg-user", "", "PostgreSQL user")
c.flags.pgPassword = fs.String("pg-password", "", "PostgreSQL password")
c.flags.pgType = fs.String("pg-type", "", "PostgreSQL type: host or docker")
c.flags.pgHostBinDir = fs.String("pg-host-bin-dir", "", "Path to PG bin directory (host mode)")
c.flags.pgDockerContainerName = fs.String("pg-docker-container-name", "", "Docker container name (docker mode)")
c.flags.walDir = fs.String("wal-dir", "", "Path to WAL queue directory")
if err := fs.Parse(args); err != nil {
os.Exit(1)
@@ -76,11 +95,35 @@ func (c *Config) loadFromJSON() {
log.Info("Configuration loaded from " + configFileName)
}
func (c *Config) applyDefaults() {
if c.PgPort == 0 {
c.PgPort = 5432
}
if c.PgType == "" {
c.PgType = "host"
}
if c.IsDeleteWalAfterUpload == nil {
v := true
c.IsDeleteWalAfterUpload = &v
}
}
func (c *Config) initSources() {
c.flags.sources = map[string]string{
"databasus-host": "not configured",
"db-id": "not configured",
"token": "not configured",
"databasus-host": "not configured",
"db-id": "not configured",
"token": "not configured",
"pg-host": "not configured",
"pg-port": "not configured",
"pg-user": "not configured",
"pg-password": "not configured",
"pg-type": "not configured",
"pg-host-bin-dir": "not configured",
"pg-docker-container-name": "not configured",
"wal-dir": "not configured",
"delete-wal-after-upload": "not configured",
}
if c.DatabasusHost != "" {
@@ -94,11 +137,44 @@ func (c *Config) initSources() {
if c.Token != "" {
c.flags.sources["token"] = configFileName
}
if c.PgHost != "" {
c.flags.sources["pg-host"] = configFileName
}
// PgPort always has a value after applyDefaults
c.flags.sources["pg-port"] = configFileName
if c.PgUser != "" {
c.flags.sources["pg-user"] = configFileName
}
if c.PgPassword != "" {
c.flags.sources["pg-password"] = configFileName
}
// PgType always has a value after applyDefaults
c.flags.sources["pg-type"] = configFileName
if c.PgHostBinDir != "" {
c.flags.sources["pg-host-bin-dir"] = configFileName
}
if c.PgDockerContainerName != "" {
c.flags.sources["pg-docker-container-name"] = configFileName
}
if c.WalDir != "" {
c.flags.sources["wal-dir"] = configFileName
}
// IsDeleteWalAfterUpload always has a value after applyDefaults
c.flags.sources["delete-wal-after-upload"] = configFileName
}
func (c *Config) applyFlags() {
if c.flags.host != nil && *c.flags.host != "" {
c.DatabasusHost = *c.flags.host
if c.flags.databasusHost != nil && *c.flags.databasusHost != "" {
c.DatabasusHost = *c.flags.databasusHost
c.flags.sources["databasus-host"] = "command line args"
}
@@ -111,18 +187,73 @@ func (c *Config) applyFlags() {
c.Token = *c.flags.token
c.flags.sources["token"] = "command line args"
}
if c.flags.pgHost != nil && *c.flags.pgHost != "" {
c.PgHost = *c.flags.pgHost
c.flags.sources["pg-host"] = "command line args"
}
if c.flags.pgPort != nil && *c.flags.pgPort != 0 {
c.PgPort = *c.flags.pgPort
c.flags.sources["pg-port"] = "command line args"
}
if c.flags.pgUser != nil && *c.flags.pgUser != "" {
c.PgUser = *c.flags.pgUser
c.flags.sources["pg-user"] = "command line args"
}
if c.flags.pgPassword != nil && *c.flags.pgPassword != "" {
c.PgPassword = *c.flags.pgPassword
c.flags.sources["pg-password"] = "command line args"
}
if c.flags.pgType != nil && *c.flags.pgType != "" {
c.PgType = *c.flags.pgType
c.flags.sources["pg-type"] = "command line args"
}
if c.flags.pgHostBinDir != nil && *c.flags.pgHostBinDir != "" {
c.PgHostBinDir = *c.flags.pgHostBinDir
c.flags.sources["pg-host-bin-dir"] = "command line args"
}
if c.flags.pgDockerContainerName != nil && *c.flags.pgDockerContainerName != "" {
c.PgDockerContainerName = *c.flags.pgDockerContainerName
c.flags.sources["pg-docker-container-name"] = "command line args"
}
if c.flags.walDir != nil && *c.flags.walDir != "" {
c.WalDir = *c.flags.walDir
c.flags.sources["wal-dir"] = "command line args"
}
}
func (c *Config) logConfigSources() {
log.Info(
"databasus-host",
"value",
c.DatabasusHost,
"source",
c.flags.sources["databasus-host"],
)
log.Info("databasus-host", "value", c.DatabasusHost, "source", c.flags.sources["databasus-host"])
log.Info("db-id", "value", c.DbID, "source", c.flags.sources["db-id"])
log.Info("token", "value", maskSensitive(c.Token), "source", c.flags.sources["token"])
log.Info("pg-host", "value", c.PgHost, "source", c.flags.sources["pg-host"])
log.Info("pg-port", "value", c.PgPort, "source", c.flags.sources["pg-port"])
log.Info("pg-user", "value", c.PgUser, "source", c.flags.sources["pg-user"])
log.Info("pg-password", "value", maskSensitive(c.PgPassword), "source", c.flags.sources["pg-password"])
log.Info("pg-type", "value", c.PgType, "source", c.flags.sources["pg-type"])
log.Info("pg-host-bin-dir", "value", c.PgHostBinDir, "source", c.flags.sources["pg-host-bin-dir"])
log.Info(
"pg-docker-container-name",
"value",
c.PgDockerContainerName,
"source",
c.flags.sources["pg-docker-container-name"],
)
log.Info("wal-dir", "value", c.WalDir, "source", c.flags.sources["wal-dir"])
log.Info(
"delete-wal-after-upload",
"value",
fmt.Sprintf("%v", *c.IsDeleteWalAfterUpload),
"source",
c.flags.sources["delete-wal-after-upload"],
)
}
func maskSensitive(value string) string {

View File

@@ -86,10 +86,12 @@ func Test_LoadFromJSONAndArgs_PartialArgsOverrideJSON(t *testing.T) {
func Test_SaveToJSON_ConfigSavedCorrectly(t *testing.T) {
setupTempDir(t)
deleteWal := true
cfg := &Config{
DatabasusHost: "http://save-host:4005",
DbID: "save-db-id",
Token: "save-token",
DatabasusHost: "http://save-host:4005",
DbID: "save-db-id",
Token: "save-token",
IsDeleteWalAfterUpload: &deleteWal,
}
err := cfg.SaveToJSON()
@@ -126,6 +128,143 @@ func Test_SaveToJSON_AfterArgsOverrideJSON_SavedFileContainsMergedValues(t *test
assert.Equal(t, "json-token", saved.Token)
}
func Test_LoadFromJSONAndArgs_PgFieldsLoadedFromJSON(t *testing.T) {
dir := setupTempDir(t)
deleteWal := false
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
PgHost: "pg-json-host",
PgPort: 5433,
PgUser: "pg-json-user",
PgPassword: "pg-json-pass",
PgType: "docker",
PgHostBinDir: "/usr/bin",
PgDockerContainerName: "pg-container",
WalDir: "/opt/wal",
IsDeleteWalAfterUpload: &deleteWal,
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{})
assert.Equal(t, "pg-json-host", cfg.PgHost)
assert.Equal(t, 5433, cfg.PgPort)
assert.Equal(t, "pg-json-user", cfg.PgUser)
assert.Equal(t, "pg-json-pass", cfg.PgPassword)
assert.Equal(t, "docker", cfg.PgType)
assert.Equal(t, "/usr/bin", cfg.PgHostBinDir)
assert.Equal(t, "pg-container", cfg.PgDockerContainerName)
assert.Equal(t, "/opt/wal", cfg.WalDir)
assert.Equal(t, false, *cfg.IsDeleteWalAfterUpload)
}
func Test_LoadFromJSONAndArgs_PgFieldsLoadedFromArgs(t *testing.T) {
setupTempDir(t)
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--pg-host", "arg-pg-host",
"--pg-port", "5433",
"--pg-user", "arg-pg-user",
"--pg-password", "arg-pg-pass",
"--pg-type", "docker",
"--pg-host-bin-dir", "/custom/bin",
"--pg-docker-container-name", "my-pg",
"--wal-dir", "/var/wal",
})
assert.Equal(t, "arg-pg-host", cfg.PgHost)
assert.Equal(t, 5433, cfg.PgPort)
assert.Equal(t, "arg-pg-user", cfg.PgUser)
assert.Equal(t, "arg-pg-pass", cfg.PgPassword)
assert.Equal(t, "docker", cfg.PgType)
assert.Equal(t, "/custom/bin", cfg.PgHostBinDir)
assert.Equal(t, "my-pg", cfg.PgDockerContainerName)
assert.Equal(t, "/var/wal", cfg.WalDir)
}
func Test_LoadFromJSONAndArgs_PgArgsOverrideJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
PgHost: "json-host",
PgPort: 5432,
PgUser: "json-user",
PgType: "host",
WalDir: "/json/wal",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--pg-host", "arg-host",
"--pg-port", "5433",
"--pg-user", "arg-user",
"--pg-type", "docker",
"--pg-docker-container-name", "my-container",
"--wal-dir", "/arg/wal",
})
assert.Equal(t, "arg-host", cfg.PgHost)
assert.Equal(t, 5433, cfg.PgPort)
assert.Equal(t, "arg-user", cfg.PgUser)
assert.Equal(t, "docker", cfg.PgType)
assert.Equal(t, "my-container", cfg.PgDockerContainerName)
assert.Equal(t, "/arg/wal", cfg.WalDir)
}
func Test_LoadFromJSONAndArgs_DefaultsApplied_WhenNoJSONAndNoArgs(t *testing.T) {
setupTempDir(t)
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{})
assert.Equal(t, 5432, cfg.PgPort)
assert.Equal(t, "host", cfg.PgType)
require.NotNil(t, cfg.IsDeleteWalAfterUpload)
assert.Equal(t, true, *cfg.IsDeleteWalAfterUpload)
}
func Test_SaveToJSON_PgFieldsSavedCorrectly(t *testing.T) {
setupTempDir(t)
deleteWal := false
cfg := &Config{
DatabasusHost: "http://host:4005",
DbID: "db-id",
Token: "token",
PgHost: "pg-host",
PgPort: 5433,
PgUser: "pg-user",
PgPassword: "pg-pass",
PgType: "docker",
PgHostBinDir: "/usr/bin",
PgDockerContainerName: "pg-container",
WalDir: "/opt/wal",
IsDeleteWalAfterUpload: &deleteWal,
}
err := cfg.SaveToJSON()
require.NoError(t, err)
saved := readConfigJSON(t)
assert.Equal(t, "pg-host", saved.PgHost)
assert.Equal(t, 5433, saved.PgPort)
assert.Equal(t, "pg-user", saved.PgUser)
assert.Equal(t, "pg-pass", saved.PgPassword)
assert.Equal(t, "docker", saved.PgType)
assert.Equal(t, "/usr/bin", saved.PgHostBinDir)
assert.Equal(t, "pg-container", saved.PgDockerContainerName)
assert.Equal(t, "/opt/wal", saved.WalDir)
require.NotNil(t, saved.IsDeleteWalAfterUpload)
assert.Equal(t, false, *saved.IsDeleteWalAfterUpload)
}
func setupTempDir(t *testing.T) string {
t.Helper()

View File

@@ -1,9 +1,17 @@
package config
type parsedFlags struct {
host *string
dbID *string
token *string
databasusHost *string
dbID *string
token *string
pgHost *string
pgPort *int
pgUser *string
pgPassword *string
pgType *string
pgHostBinDir *string
pgDockerContainerName *string
walDir *string
sources map[string]string
}

View File

@@ -1,17 +1,38 @@
package start
import (
"context"
"errors"
"fmt"
"log/slog"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/jackc/pgx/v5"
"databasus-agent/internal/config"
)
const (
pgBasebackupVerifyTimeout = 10 * time.Second
dbVerifyTimeout = 10 * time.Second
)
func Run(cfg *config.Config, log *slog.Logger) error {
if err := validateConfig(cfg); err != nil {
return err
}
if err := verifyPgBasebackup(cfg, log); err != nil {
return err
}
if err := verifyDatabase(cfg, log); err != nil {
return err
}
log.Info("start: stub — not yet implemented",
"dbId", cfg.DbID,
"hasToken", cfg.Token != "",
@@ -33,5 +54,126 @@ func validateConfig(cfg *config.Config) error {
return errors.New("argument token is required")
}
if cfg.PgHost == "" {
return errors.New("argument pg-host is required")
}
if cfg.PgPort <= 0 {
return errors.New("argument pg-port must be a positive number")
}
if cfg.PgUser == "" {
return errors.New("argument pg-user is required")
}
if cfg.PgType != "host" && cfg.PgType != "docker" {
return fmt.Errorf("argument pg-type must be 'host' or 'docker', got '%s'", cfg.PgType)
}
if cfg.WalDir == "" {
return errors.New("argument wal-dir is required")
}
if cfg.PgType == "docker" && cfg.PgDockerContainerName == "" {
return errors.New("argument pg-docker-container-name is required when pg-type is 'docker'")
}
return nil
}
func verifyPgBasebackup(cfg *config.Config, log *slog.Logger) error {
switch cfg.PgType {
case "host":
return verifyPgBasebackupHost(cfg, log)
case "docker":
return verifyPgBasebackupDocker(cfg, log)
default:
return fmt.Errorf("unexpected pg-type: %s", cfg.PgType)
}
}
func verifyPgBasebackupHost(cfg *config.Config, log *slog.Logger) error {
binary := "pg_basebackup"
if cfg.PgHostBinDir != "" {
binary = filepath.Join(cfg.PgHostBinDir, "pg_basebackup")
}
ctx, cancel := context.WithTimeout(context.Background(), pgBasebackupVerifyTimeout)
defer cancel()
output, err := exec.CommandContext(ctx, binary, "--version").CombinedOutput()
if err != nil {
if cfg.PgHostBinDir != "" {
return fmt.Errorf(
"pg_basebackup not found at '%s': %w. Verify pg-host-bin-dir is correct",
binary, err,
)
}
return fmt.Errorf(
"pg_basebackup not found in PATH: %w. Install PostgreSQL client tools or set pg-host-bin-dir",
err,
)
}
log.Info("pg_basebackup verified", "version", strings.TrimSpace(string(output)))
return nil
}
func verifyPgBasebackupDocker(cfg *config.Config, log *slog.Logger) error {
ctx, cancel := context.WithTimeout(context.Background(), pgBasebackupVerifyTimeout)
defer cancel()
output, err := exec.CommandContext(ctx,
"docker", "exec", cfg.PgDockerContainerName,
"pg_basebackup", "--version",
).CombinedOutput()
if err != nil {
return fmt.Errorf(
"pg_basebackup not available in container '%s': %w. "+
"Check that the container is running and pg_basebackup is installed inside it",
cfg.PgDockerContainerName, err,
)
}
log.Info("pg_basebackup verified (docker)",
"container", cfg.PgDockerContainerName,
"version", strings.TrimSpace(string(output)),
)
return nil
}
func verifyDatabase(cfg *config.Config, log *slog.Logger) error {
connStr := fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=postgres sslmode=disable",
cfg.PgHost, cfg.PgPort, cfg.PgUser, cfg.PgPassword,
)
ctx, cancel := context.WithTimeout(context.Background(), dbVerifyTimeout)
defer cancel()
conn, err := pgx.Connect(ctx, connStr)
if err != nil {
return fmt.Errorf(
"failed to connect to PostgreSQL at %s:%d as user '%s': %w",
cfg.PgHost, cfg.PgPort, cfg.PgUser, err,
)
}
defer func() { _ = conn.Close(ctx) }()
if err := conn.Ping(ctx); err != nil {
return fmt.Errorf("PostgreSQL ping failed at %s:%d: %w",
cfg.PgHost, cfg.PgPort, err,
)
}
log.Info("PostgreSQL connection verified",
"host", cfg.PgHost,
"port", cfg.PgPort,
"user", cfg.PgUser,
)
return nil
}

View File

@@ -0,0 +1,5 @@
package upgrade
type versionResponse struct {
Version string `json:"version"`
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"os/exec"
@@ -14,17 +15,7 @@ import (
"time"
)
type Logger interface {
Info(msg string, args ...any)
Warn(msg string, args ...any)
Error(msg string, args ...any)
}
type versionResponse struct {
Version string `json:"version"`
}
func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log Logger) error {
func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log *slog.Logger) error {
if isDev {
log.Info("Skipping update check (development mode)")
return nil
@@ -32,7 +23,11 @@ func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log Logger
serverVersion, err := fetchServerVersion(databasusHost, log)
if err != nil {
return nil
return fmt.Errorf(
"unable to check version, please verify Databasus server is available at %s: %w",
databasusHost,
err,
)
}
if serverVersion == currentVersion {
@@ -74,7 +69,7 @@ func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log Logger
return syscall.Exec(selfPath, os.Args, os.Environ())
}
func fetchServerVersion(host string, log Logger) (string, error) {
func fetchServerVersion(host string, log *slog.Logger) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()

View File

@@ -32,8 +32,6 @@ func Init(isDebug bool) {
return a
},
}))
loggerInstance.Info("Text structured logger initialized")
})
}