515 lines
15 KiB
Bash
515 lines
15 KiB
Bash
#!/bin/bash
|
||
|
||
###############################################################################
|
||
# E2E Test Collection - Service Health & Connectivity Checks
|
||
# Purpose: Verify all services are running and accessible
|
||
# Author: QA/Testing Agent
|
||
# Date: 2026-03-16
|
||
###############################################################################
|
||
|
||
set -e # Exit on error
|
||
|
||
# Color output
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# Counters
|
||
PASSED=0
|
||
FAILED=0
|
||
SKIPPED=0
|
||
|
||
# Configuration
|
||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||
|
||
# Load environment variables if .env exists
|
||
if [ -f "$PROJECT_ROOT/.env" ]; then
|
||
source "$PROJECT_ROOT/.env"
|
||
fi
|
||
|
||
# Default values (can be overridden by env vars)
|
||
FREESCOUT_HOST="${FREESCOUT_HOST:-https://ekshelpdesk.fft-it.de}"
|
||
LITELLM_HOST="${LITELLM_HOST:-http://llm.eks-ai.apps.asgard.eks-lnx.fft-it.de}"
|
||
MILVUS_HOST="${MILVUS_HOST:-127.0.0.1}"
|
||
MILVUS_PORT="${MILVUS_PORT:-19530}"
|
||
POSTGRES_HOST="${POSTGRES_HOST:-127.0.0.1}"
|
||
POSTGRES_PORT="${POSTGRES_PORT:-5432}"
|
||
POSTGRES_USER="${POSTGRES_USER:-kb_user}"
|
||
POSTGRES_DB="${POSTGRES_DB:-n8n_kb}"
|
||
N8N_HOST="${N8N_HOST:-http://localhost}"
|
||
N8N_PORT="${N8N_PORT:-5678}"
|
||
|
||
###############################################################################
|
||
# Helper Functions
|
||
###############################################################################
|
||
|
||
log_header() {
|
||
echo -e "\n${BLUE}=== $1 ===${NC}\n"
|
||
}
|
||
|
||
log_pass() {
|
||
echo -e "${GREEN}✓ PASS${NC}: $1"
|
||
((PASSED++))
|
||
}
|
||
|
||
log_fail() {
|
||
echo -e "${RED}✗ FAIL${NC}: $1"
|
||
((FAILED++))
|
||
}
|
||
|
||
log_skip() {
|
||
echo -e "${YELLOW}⊘ SKIP${NC}: $1"
|
||
((SKIPPED++))
|
||
}
|
||
|
||
log_info() {
|
||
echo -e "${BLUE}ℹ INFO${NC}: $1"
|
||
}
|
||
|
||
test_endpoint() {
|
||
local name="$1"
|
||
local method="${2:-GET}"
|
||
local url="$3"
|
||
local headers="$4"
|
||
local data="$5"
|
||
local expected_code="${6:-200}"
|
||
|
||
echo -n "Testing $name... "
|
||
|
||
# Build curl command
|
||
local curl_cmd="curl -s -w '%{http_code}' -X $method"
|
||
|
||
if [ -n "$headers" ]; then
|
||
curl_cmd="$curl_cmd -H '$headers'"
|
||
fi
|
||
|
||
if [ "$method" != "GET" ] && [ -n "$data" ]; then
|
||
curl_cmd="$curl_cmd -d '$data'"
|
||
fi
|
||
|
||
curl_cmd="$curl_cmd '$url' -o /tmp/response.txt"
|
||
|
||
# Execute and capture response code
|
||
local response_code=$(eval "$curl_cmd" 2>/dev/null || echo "000")
|
||
|
||
if [ "$response_code" = "$expected_code" ]; then
|
||
log_pass "$name (HTTP $response_code)"
|
||
return 0
|
||
else
|
||
log_fail "$name (Expected HTTP $expected_code, got $response_code)"
|
||
cat /tmp/response.txt 2>/dev/null | head -c 200
|
||
echo ""
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
###############################################################################
|
||
# Test Suite
|
||
###############################################################################
|
||
|
||
main() {
|
||
echo -e "${BLUE}"
|
||
cat << 'EOF'
|
||
╔════════════════════════════════════════════════════════════════════════════╗
|
||
║ E2E Test Collection - Health Checks ║
|
||
║ ║
|
||
║ This script verifies that all microservices are running and accessible. ║
|
||
║ It performs basic connectivity tests and returns a summary report. ║
|
||
╚════════════════════════════════════════════════════════════════════════════╝
|
||
EOF
|
||
echo -e "${NC}\n"
|
||
|
||
# Test Milvus Vector DB
|
||
log_header "1. Testing Milvus Vector Database"
|
||
test_milvus
|
||
|
||
# Test PostgreSQL
|
||
log_header "2. Testing PostgreSQL Database"
|
||
test_postgres
|
||
|
||
# Test Freescout
|
||
log_header "3. Testing Freescout API"
|
||
test_freescout
|
||
|
||
# Test LiteLLM
|
||
log_header "4. Testing LiteLLM API"
|
||
test_litellm
|
||
|
||
# Test n8n
|
||
log_header "5. Testing n8n Workflow Engine"
|
||
test_n8n
|
||
|
||
# Test Docker Compose Stack
|
||
log_header "6. Testing Docker Compose Services"
|
||
test_docker_compose
|
||
|
||
# Print Summary
|
||
print_summary
|
||
}
|
||
|
||
###############################################################################
|
||
# Individual Test Functions
|
||
###############################################################################
|
||
|
||
test_milvus() {
|
||
local milvus_url="http://$MILVUS_HOST:$MILVUS_PORT"
|
||
|
||
echo "Checking Milvus at $milvus_url"
|
||
|
||
# Test health endpoint
|
||
echo -n " Health check... "
|
||
if curl -s "$milvus_url/healthz" >/dev/null 2>&1; then
|
||
log_pass "Milvus is responding"
|
||
else
|
||
log_fail "Milvus health endpoint not reachable"
|
||
return 1
|
||
fi
|
||
|
||
# Test metrics endpoint
|
||
echo -n " Metrics check... "
|
||
if curl -s "$milvus_url/metrics" >/dev/null 2>&1; then
|
||
log_pass "Milvus metrics available"
|
||
else
|
||
log_skip "Milvus metrics endpoint (non-critical)"
|
||
fi
|
||
|
||
# Test REST API version
|
||
echo -n " API version check... "
|
||
if curl -s "$milvus_url/v1/version" >/dev/null 2>&1; then
|
||
log_pass "Milvus REST API responding"
|
||
else
|
||
log_fail "Milvus REST API not responding"
|
||
fi
|
||
}
|
||
|
||
test_postgres() {
|
||
echo "Checking PostgreSQL at $POSTGRES_HOST:$POSTGRES_PORT"
|
||
|
||
# Test basic connectivity
|
||
echo -n " Connection test... "
|
||
if nc -z "$POSTGRES_HOST" "$POSTGRES_PORT" 2>/dev/null; then
|
||
log_pass "PostgreSQL port is open"
|
||
else
|
||
log_fail "Cannot connect to PostgreSQL port"
|
||
return 1
|
||
fi
|
||
|
||
# Test database query (if inside docker network)
|
||
echo -n " Database query test... "
|
||
if command -v docker &> /dev/null; then
|
||
if docker-compose -f "$PROJECT_ROOT/docker-compose.yml" exec postgres \
|
||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT COUNT(*) FROM knowledge_base_updates;" \
|
||
>/dev/null 2>&1; then
|
||
log_pass "PostgreSQL database accessible and configured"
|
||
else
|
||
log_fail "PostgreSQL query failed - might be authentication issue"
|
||
fi
|
||
else
|
||
log_skip "PostgreSQL query test (Docker not available)"
|
||
fi
|
||
|
||
# Check for required tables
|
||
echo -n " Schema validation... "
|
||
if docker-compose -f "$PROJECT_ROOT/docker-compose.yml" exec postgres \
|
||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
|
||
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name='knowledge_base_updates');" \
|
||
2>/dev/null | grep -q "t"; then
|
||
log_pass "Required tables exist"
|
||
else
|
||
log_skip "Schema validation (might not be initialized)"
|
||
fi
|
||
}
|
||
|
||
test_freescout() {
|
||
echo "Checking Freescout at $FREESCOUT_HOST"
|
||
|
||
# Test basic connectivity
|
||
echo -n " Web server check... "
|
||
if curl -s -I "$FREESCOUT_HOST/login" 2>/dev/null | grep -q "200\|301\|302"; then
|
||
log_pass "Freescout web interface responding"
|
||
else
|
||
log_fail "Freescout web interface not responding"
|
||
return 1
|
||
fi
|
||
|
||
# Test API health (if API key available)
|
||
if [ -n "$FREESCOUT_API_KEY" ]; then
|
||
echo -n " API authentication test... "
|
||
local response=$(curl -s -w '%{http_code}' \
|
||
-H "Authorization: Bearer $FREESCOUT_API_KEY" \
|
||
"$FREESCOUT_HOST/api/v1/mailboxes" -o /tmp/freescout_api_response.txt)
|
||
|
||
if [ "$response" = "200" ]; then
|
||
log_pass "Freescout API authenticated"
|
||
local mailbox_count=$(jq '.mailboxes | length' /tmp/freescout_api_response.txt 2>/dev/null)
|
||
log_info "Found $mailbox_count mailbox(es)"
|
||
else
|
||
log_fail "Freescout API authentication failed (HTTP $response)"
|
||
fi
|
||
else
|
||
log_skip "Freescout API authentication test (FREESCOUT_API_KEY not set)"
|
||
fi
|
||
|
||
# Test custom fields
|
||
if [ -n "$FREESCOUT_API_KEY" ]; then
|
||
echo -n " Custom fields check... "
|
||
if curl -s -H "Authorization: Bearer $FREESCOUT_API_KEY" \
|
||
"$FREESCOUT_HOST/api/v1/custom-fields" 2>/dev/null | \
|
||
grep -q "AI_SUGGESTION"; then
|
||
log_pass "AI custom fields configured"
|
||
else
|
||
log_fail "AI custom fields not found - check Freescout configuration"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
test_litellm() {
|
||
echo "Checking LiteLLM at $LITELLM_HOST"
|
||
|
||
# Test health endpoint
|
||
echo -n " Health check... "
|
||
if curl -s "$LITELLM_HOST/health" >/dev/null 2>&1; then
|
||
log_pass "LiteLLM is responding"
|
||
else
|
||
log_fail "LiteLLM health endpoint not reachable"
|
||
return 1
|
||
fi
|
||
|
||
# Test API endpoint
|
||
echo -n " API endpoint test... "
|
||
local test_payload='{
|
||
"model": "gpt-3.5-turbo",
|
||
"messages": [{"role": "user", "content": "test"}],
|
||
"max_tokens": 10
|
||
}'
|
||
|
||
if curl -s -X POST "$LITELLM_HOST/v1/chat/completions" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$test_payload" >/dev/null 2>&1; then
|
||
log_pass "LiteLLM chat API responding"
|
||
else
|
||
log_fail "LiteLLM chat API not responding"
|
||
fi
|
||
|
||
# Test model availability
|
||
if [ -n "$LITELLM_API_KEY" ]; then
|
||
echo -n " Model availability check... "
|
||
if curl -s -X GET "$LITELLM_HOST/v1/models" \
|
||
-H "Authorization: Bearer $LITELLM_API_KEY" 2>/dev/null | \
|
||
grep -q "gpt"; then
|
||
log_pass "Models available"
|
||
else
|
||
log_skip "Model availability (non-critical)"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
test_n8n() {
|
||
echo "Checking n8n at $N8N_HOST:$N8N_PORT"
|
||
|
||
local n8n_url="$N8N_HOST:$N8N_PORT"
|
||
|
||
# Test REST API
|
||
echo -n " REST API health... "
|
||
if curl -s "$n8n_url/healthz" >/dev/null 2>&1 || \
|
||
curl -s "$n8n_url/health" >/dev/null 2>&1; then
|
||
log_pass "n8n REST API responding"
|
||
else
|
||
log_fail "n8n REST API not responding (might be behind reverse proxy)"
|
||
fi
|
||
|
||
# Test web interface
|
||
echo -n " Web interface check... "
|
||
if curl -s "$n8n_url/" >/dev/null 2>&1; then
|
||
log_pass "n8n web interface accessible"
|
||
else
|
||
log_skip "n8n web interface check (might require authentication)"
|
||
fi
|
||
|
||
# Test workflows endpoint (if authenticated)
|
||
if [ -n "$N8N_AUTH_TOKEN" ]; then
|
||
echo -n " Workflow list test... "
|
||
if curl -s -H "Authorization: Bearer $N8N_AUTH_TOKEN" \
|
||
"$n8n_url/api/v1/workflows" >/dev/null 2>&1; then
|
||
log_pass "n8n workflows accessible"
|
||
else
|
||
log_fail "Cannot access n8n workflows"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
test_docker_compose() {
|
||
if ! command -v docker-compose &> /dev/null; then
|
||
log_skip "Docker Compose not available"
|
||
return
|
||
fi
|
||
|
||
echo "Checking Docker Compose services..."
|
||
|
||
# Get list of running services
|
||
local services=$(docker-compose -f "$PROJECT_ROOT/docker-compose.yml" ps --services 2>/dev/null)
|
||
|
||
if [ -z "$services" ]; then
|
||
log_fail "No Docker Compose services found"
|
||
return 1
|
||
fi
|
||
|
||
# Check each service
|
||
while read -r service; do
|
||
echo -n " Service: $service... "
|
||
if docker-compose -f "$PROJECT_ROOT/docker-compose.yml" ps "$service" 2>/dev/null | \
|
||
grep -q "Up"; then
|
||
log_pass "$service is running"
|
||
else
|
||
log_fail "$service is not running"
|
||
fi
|
||
done <<< "$services"
|
||
|
||
# Check resource usage
|
||
echo ""
|
||
echo -n " Container resource check... "
|
||
if docker-compose -f "$PROJECT_ROOT/docker-compose.yml" stats --no-stream 2>/dev/null | \
|
||
tail -n +2 | wc -l | grep -qv "0"; then
|
||
log_pass "Docker containers have allocated resources"
|
||
else
|
||
log_skip "Could not read container stats"
|
||
fi
|
||
}
|
||
|
||
###############################################################################
|
||
# Summary Report
|
||
###############################################################################
|
||
|
||
print_summary() {
|
||
local total=$((PASSED + FAILED + SKIPPED))
|
||
local status="PASSED"
|
||
|
||
if [ $FAILED -gt 0 ]; then
|
||
status="FAILED"
|
||
fi
|
||
|
||
echo ""
|
||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${BLUE}║ Test Summary Report ║${NC}"
|
||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||
echo ""
|
||
echo -e " Total Tests: $total"
|
||
echo -e " ${GREEN}Passed:${NC} $PASSED"
|
||
if [ $FAILED -gt 0 ]; then
|
||
echo -e " ${RED}Failed:${NC} $FAILED"
|
||
fi
|
||
if [ $SKIPPED -gt 0 ]; then
|
||
echo -e " ${YELLOW}Skipped:${NC} $SKIPPED"
|
||
fi
|
||
echo ""
|
||
|
||
if [ $FAILED -eq 0 ]; then
|
||
echo -e "${GREEN}✓ All critical services are running and accessible!${NC}"
|
||
echo -e " You can proceed with E2E testing."
|
||
echo ""
|
||
return 0
|
||
else
|
||
echo -e "${RED}✗ Some services are not responding!${NC}"
|
||
echo -e " Please check the failures above and ensure all services are running."
|
||
echo ""
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
###############################################################################
|
||
# Usage Information
|
||
###############################################################################
|
||
|
||
show_help() {
|
||
cat << EOF
|
||
Usage: $0 [OPTIONS]
|
||
|
||
E2E Test Collection - Service Health & Connectivity Checks
|
||
|
||
Options:
|
||
-h, --help Show this help message
|
||
-v, --verbose Enable verbose output
|
||
-m, --milvus HOST:PORT Override Milvus address (default: 127.0.0.1:19530)
|
||
-p, --postgres HOST:PORT Override PostgreSQL address (default: 127.0.0.1:5432)
|
||
-f, --freescout URL Override Freescout URL
|
||
-l, --litellm URL Override LiteLLM URL
|
||
-n, --n8n URL Override n8n URL
|
||
|
||
Environment Variables:
|
||
FREESCOUT_API_KEY Freescout API authentication token
|
||
LITELLM_API_KEY LiteLLM API authentication token
|
||
N8N_AUTH_TOKEN n8n authentication token
|
||
POSTGRES_USER PostgreSQL username (default: kb_user)
|
||
POSTGRES_DB PostgreSQL database name (default: n8n_kb)
|
||
|
||
Examples:
|
||
# Run basic health checks with defaults
|
||
$0
|
||
|
||
# Override Milvus host
|
||
$0 --milvus 192.168.1.100:19530
|
||
|
||
# With API keys
|
||
FREESCOUT_API_KEY=token123 $0
|
||
|
||
EOF
|
||
}
|
||
|
||
###############################################################################
|
||
# Main Execution
|
||
###############################################################################
|
||
|
||
# Parse command line arguments
|
||
while [[ $# -gt 0 ]]; do
|
||
case $1 in
|
||
-h|--help)
|
||
show_help
|
||
exit 0
|
||
;;
|
||
-v|--verbose)
|
||
set -x
|
||
shift
|
||
;;
|
||
-m|--milvus)
|
||
MILVUS_HOST="${2%:*}"
|
||
MILVUS_PORT="${2##*:}"
|
||
shift 2
|
||
;;
|
||
-p|--postgres)
|
||
POSTGRES_HOST="${2%:*}"
|
||
POSTGRES_PORT="${2##*:}"
|
||
shift 2
|
||
;;
|
||
-f|--freescout)
|
||
FREESCOUT_HOST="$2"
|
||
shift 2
|
||
;;
|
||
-l|--litellm)
|
||
LITELLM_HOST="$2"
|
||
shift 2
|
||
;;
|
||
-n|--n8n)
|
||
N8N_HOST="$2"
|
||
shift 2
|
||
;;
|
||
*)
|
||
echo "Unknown option: $1"
|
||
show_help
|
||
exit 1
|
||
;;
|
||
esac
|
||
done
|
||
|
||
# Run main test suite
|
||
main
|
||
|
||
# Exit with appropriate code
|
||
if [ $FAILED -gt 0 ]; then
|
||
exit 1
|
||
else
|
||
exit 0
|
||
fi
|