Testing Guide

Comprehensive testing standards for the LLM Ecosystem. This guide consolidates testing patterns from all platform projects.

---

Quick Reference

---

1. Testing Frameworks

JavaScript/TypeScript Projects

Vitest (Recommended)

Vitest is the preferred testing framework for new projects. It offers native TypeScript support, fast execution, and excellent ESM compatibility.

# Install
npm install -D vitest @vitest/coverage-v8

# Run tests
npm test                    # Unit tests
npm run test:integration    # Integration tests
npm run test:e2e           # End-to-end tests
npm run test:coverage      # Coverage report

Jest (Legacy Support)

Jest is still supported for existing projects. New projects should use Vitest.

# Install
npm install -D jest ts-jest @types/jest

# Run tests
npm test
npm run test:coverage

Go Projects

Go's standard testing package with additional tooling:

# Run tests
go test ./...

# With coverage
go test -cover ./...
go test -coverprofile=coverage.out ./...
go tool cover -html=coverage.out

# Verbose output
go test -v ./...

# Run specific package
go test ./internal/ossa/runtime/...

PHP/Drupal Projects

PHPUnit with Drupal testing infrastructure:

# Run tests
./vendor/bin/phpunit

# With coverage
./vendor/bin/phpunit --coverage-html coverage/

# Run specific test
./vendor/bin/phpunit tests/src/Unit/MyTest.php

---

2. Unit Testing Standards

Unit tests verify individual functions and components in isolation.

Vitest Configuration (Unit)

// vitest.config.ts
import { defineConfig } from 'vitest/config';
import path from 'path';

export default defineConfig({
  test: {
    name: 'unit',
    globals: true,
    environment: 'node',
    include: ['tests/unit/**/*.test.ts', 'src/**/*.test.ts'],
    exclude: ['node_modules', 'dist', 'coverage'],
    coverage: {
      provider: 'v8',
      reporter: ['text', 'json', 'html', 'lcov', 'cobertura'],
      reportsDirectory: './coverage',
      include: ['src/**/*.ts'],
      exclude: [
        '**/*.test.ts',
        '**/*.spec.ts',
        '**/node_modules/**',
        '**/dist/**',
        '**/*.config.ts',
        '**/types/**',
        '**/fixtures/**',
        '**/mocks/**'
      ],
      thresholds: {
        lines: 90,
        functions: 90,
        branches: 85,
        statements: 90
      }
    },
    testTimeout: 10000,
    hookTimeout: 10000,
    teardownTimeout: 10000,
    isolate: true,
    threads: true,
    maxConcurrency: 5,
    reporters: ['default', 'junit', 'json'],
    outputFile: {
      junit: './coverage/junit.xml',
      json: './coverage/test-results.json'
    }
  },
  resolve: {
    alias: {
      '@': path.resolve(__dirname, './src'),
      '@tests': path.resolve(__dirname, './tests')
    }
  }
});

Jest Configuration (Legacy)

// jest.config.js
module.exports = {
  preset: 'ts-jest',
  testEnvironment: 'node',
  roots: ['<rootDir>/tests', '<rootDir>/src'],
  setupFilesAfterEnv: ['<rootDir>/tests/setup.ts'],
  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
  testPathIgnorePatterns: [
    '<rootDir>/tests/e2e/',
    '<rootDir>/tests/integration/',
    '<rootDir>/node_modules/',
    '<rootDir>/dist/',
  ],
  testMatch: [
    '<rootDir>/tests/unit/**/*.test.ts',
    '<rootDir>/src/**/*.test.ts',
  ],
  transform: {
    '^.+\\.ts$': ['ts-jest', {
      tsconfig: '<rootDir>/tsconfig.json',
      isolatedModules: true,
    }],
  },
  moduleNameMapper: {
    '^@/(.*)$': '<rootDir>/src/$1',
    '^@tests/(.*)$': '<rootDir>/tests/$1',
  },
  collectCoverageFrom: [
    'src/**/*.ts',
    '!src/**/*.d.ts',
    '!src/**/*.spec.ts',
    '!src/**/*.test.ts',
  ],
  coverageDirectory: 'coverage',
  coverageReporters: ['text', 'lcov', 'html', 'cobertura'],
  coverageThreshold: {
    global: {
      branches: 85,
      functions: 90,
      lines: 90,
      statements: 90
    }
  },
  verbose: true,
};

Unit Test Best Practices

1. Test Structure (AAA Pattern)

describe('MyService', () => {
  describe('processData', () => {
    it('should transform valid input correctly', () => {
      // Arrange
      const input = { name: 'test', value: 42 };
      const service = new MyService();

      // Act
      const result = service.processData(input);

      // Assert
      expect(result.processed).toBe(true);
      expect(result.name).toBe('TEST');
    });

    it('should throw error for invalid input', () => {
      // Arrange
      const input = { name: '', value: -1 };
      const service = new MyService();

      // Act & Assert
      expect(() => service.processData(input)).toThrow('Invalid input');
    });
  });
});

2. Mock External Dependencies

import { vi } from 'vitest';

// Mock modules
vi.mock('@/services/api', () => ({
  fetchData: vi.fn(),
}));

// Mock implementations
import { fetchData } from '@/services/api';
const mockFetchData = vi.mocked(fetchData);

describe('DataProcessor', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('should process API response', async () => {
    // Arrange
    mockFetchData.mockResolvedValue({ items: [1, 2, 3] });
    const processor = new DataProcessor();

    // Act
    const result = await processor.process();

    // Assert
    expect(mockFetchData).toHaveBeenCalledOnce();
    expect(result.count).toBe(3);
  });
});

3. Table-Driven Tests

describe('validateInput', () => {
  const testCases = [
    { input: 'valid', expected: true },
    { input: '', expected: false },
    { input: null, expected: false },
    { input: 'a'.repeat(256), expected: false },
  ];

  it.each(testCases)('should return $expected for input: $input', ({ input, expected }) => {
    expect(validateInput(input)).toBe(expected);
  });
});

Go Unit Testing

package runtime_test

import (
    "context"
    "testing"
    "time"

    "gitlab.com/blueflyio/project/internal/ossa/runtime"
)

func TestLifecycleExecution(t *testing.T) {
    tests := []struct {
        name     string
        input    map[string]interface{}
        wantErr  bool
        validate func(t *testing.T, result *runtime.TurnContext)
    }{
        {
            name:    "valid input executes successfully",
            input:   map[string]interface{}{"action": "test"},
            wantErr: false,
            validate: func(t *testing.T, result *runtime.TurnContext) {
                if result.TurnID == "" {
                    t.Error("Expected non-empty turn ID")
                }
            },
        },
        {
            name:    "empty input returns error",
            input:   nil,
            wantErr: true,
        },
    }

    for _, tt := range tests {
        t.Run(tt.name, func(t *testing.T) {
            ctx := context.Background()
            lifecycle := runtime.NewLifecycle(nil, nil, nil)

            result, err := lifecycle.Execute(ctx, tt.input)

            if (err != nil) != tt.wantErr {
                t.Errorf("Execute() error = %v, wantErr %v", err, tt.wantErr)
                return
            }

            if tt.validate != nil && result != nil {
                tt.validate(t, result)
            }
        })
    }
}

// Benchmark test
func BenchmarkLifecycleExecution(b *testing.B) {
    ctx := context.Background()
    lifecycle := runtime.NewLifecycle(nil, nil, nil)
    input := map[string]interface{}{"action": "benchmark"}

    b.ResetTimer()
    for i := 0; i < b.N; i++ {
        lifecycle.Execute(ctx, input)
    }
}

---

3. Integration Testing

Integration tests verify component interactions and external service integrations.

Vitest Integration Configuration

// vitest.integration.config.ts
import { defineConfig } from 'vitest/config';
import path from 'path';

export default defineConfig({
  test: {
    name: 'integration',
    globals: true,
    environment: 'node',
    include: ['tests/integration/**/*.test.ts'],
    exclude: ['node_modules', 'dist', 'coverage'],
    testTimeout: 30000,
    hookTimeout: 30000,
    teardownTimeout: 10000,
    isolate: true,
    threads: false, // Sequential execution for integration tests
    maxConcurrency: 1,
    reporters: ['default', 'junit'],
    outputFile: {
      junit: './coverage/integration-junit.xml'
    },
    setupFiles: ['./tests/integration/setup.ts']
  },
  resolve: {
    alias: {
      '@': path.resolve(__dirname, './src'),
      '@tests': path.resolve(__dirname, './tests')
    }
  }
});

Integration Test Setup

// tests/integration/setup.ts
import { beforeAll, afterAll, beforeEach } from 'vitest';

let testDb: TestDatabase;
let testServer: TestServer;

beforeAll(async () => {
  // Start test database
  testDb = await TestDatabase.start({
    port: 5433,
    database: 'test_db'
  });

  // Start test server
  testServer = await TestServer.start({
    port: 3001,
    db: testDb.connectionString
  });
});

afterAll(async () => {
  await testServer?.stop();
  await testDb?.stop();
});

beforeEach(async () => {
  // Reset database state
  await testDb?.reset();
});

Integration Test Example

describe('API Integration', () => {
  describe('POST /api/agents', () => {
    it('should create agent and persist to database', async () => {
      // Arrange
      const agentData = {
        name: 'test-agent',
        type: 'worker',
        capabilities: ['k8s:read']
      };

      // Act
      const response = await fetch('http://localhost:3001/api/agents', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(agentData)
      });
      const created = await response.json();

      // Assert - API response
      expect(response.status).toBe(201);
      expect(created.id).toBeDefined();
      expect(created.name).toBe('test-agent');

      // Assert - Database state
      const dbAgent = await testDb.query(
        'SELECT * FROM agents WHERE id = $1',
        [created.id]
      );
      expect(dbAgent.rows[0].name).toBe('test-agent');
    });
  });
});

Go Integration Testing

package integration

import (
    "context"
    "testing"
    "time"

    "gitlab.com/blueflyio/project/internal/ossa/capabilities"
    "gitlab.com/blueflyio/project/internal/ossa/runtime"
)

func TestFullOSSALifecycle(t *testing.T) {
    if testing.Short() {
        t.Skip("Skipping integration test in short mode")
    }

    // Setup
    state := runtime.NewInMemoryStateBackend()
    telemetry := runtime.NewSimpleTelemetryProvider()

    k8sModule := NewMockCapabilityModule([]string{
        string(capabilities.CapKubernetesAPI),
    })

    capModules := map[string]runtime.CapabilityModule{
        string(capabilities.CapKubernetesAPI): k8sModule,
    }

    lifecycle := setupFullLifecycle(state, telemetry, capModules, nil)

    // Execute
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    input := map[string]interface{}{
        "action": "apply",
        "capabilities": []interface{}{
            string(capabilities.CapKubernetesAPI),
        },
    }

    turn, err := lifecycle.Execute(ctx, input)

    // Verify
    if err != nil {
        t.Fatalf("Lifecycle execution failed: %v", err)
    }

    if turn.TurnID == "" {
        t.Error("Expected non-empty turn ID")
    }

    if turn.CurrentPhase != runtime.PhaseEmit {
        t.Errorf("Expected final phase %s, got %s",
            runtime.PhaseEmit, turn.CurrentPhase)
    }

    if k8sModule.GetCallCount() != 1 {
        t.Errorf("Expected 1 capability call, got %d",
            k8sModule.GetCallCount())
    }
}

---

4. End-to-End Testing

E2E tests verify complete user workflows from start to finish.

Vitest E2E Configuration

// vitest.e2e.config.ts
import { defineConfig } from 'vitest/config';
import path from 'path';

export default defineConfig({
  test: {
    name: 'e2e',
    globals: true,
    environment: 'node',
    include: ['tests/e2e/**/*.test.ts'],
    exclude: ['node_modules', 'dist', 'coverage'],
    testTimeout: 60000,
    hookTimeout: 30000,
    teardownTimeout: 15000,
    isolate: true,
    threads: false, // Sequential execution for e2e tests
    maxConcurrency: 1,
    reporters: ['default', 'junit'],
    outputFile: {
      junit: './coverage/e2e-junit.xml'
    },
    setupFiles: ['./tests/e2e/setup.ts']
  },
  resolve: {
    alias: {
      '@': path.resolve(__dirname, './src'),
      '@tests': path.resolve(__dirname, './tests')
    }
  }
});

E2E Test Example

describe('Agent Workflow E2E', () => {
  let browser: Browser;
  let page: Page;

  beforeAll(async () => {
    browser = await chromium.launch();
  });

  afterAll(async () => {
    await browser?.close();
  });

  beforeEach(async () => {
    page = await browser.newPage();
  });

  afterEach(async () => {
    await page?.close();
  });

  it('should complete full agent creation workflow', async () => {
    // 1. Navigate to agent studio
    await page.goto('http://localhost:3000/agents');

    // 2. Click create agent
    await page.click('[data-testid="create-agent-button"]');

    // 3. Fill agent form
    await page.fill('[name="name"]', 'e2e-test-agent');
    await page.selectOption('[name="type"]', 'worker');
    await page.click('[data-testid="capability-k8s-read"]');

    // 4. Submit form
    await page.click('[data-testid="submit-button"]');

    // 5. Verify success
    await expect(page.locator('.toast-success')).toBeVisible();
    await expect(page.locator('[data-testid="agent-list"]'))
      .toContainText('e2e-test-agent');
  }, 60000);
});

---

5. Coverage Requirements

Coverage Thresholds by Project Type

Coverage Configuration

Vitest

coverage: {
  thresholds: {
    lines: 90,
    functions: 90,
    branches: 85,
    statements: 90
  }
}

Jest

coverageThreshold: {
  global: {
    branches: 85,
    functions: 90,
    lines: 90,
    statements: 90
  }
}

Coverage Reporting

# Generate coverage report
npm run test:coverage

# View HTML report
open coverage/index.html

# CI reports (generated automatically)
# - coverage/lcov.info (for SonarQube/GitLab)
# - coverage/cobertura.xml (for GitLab MR widget)
# - coverage/junit.xml (for test results)

---

6. CI Test Automation

GitLab CI Integration

The Golden Pipeline component automatically runs tests for all project types:

# .gitlab-ci.yml
include:
  - component: $CI_SERVER_FQDN/blueflyio/gitlab_components/golden@release/v0.1.x

This provides:

• test:npm - Runs npm tests with coverage extraction
• test:python - Runs pytest with coverage
• lint:phpcs - PHP CodeSniffer for Drupal
• lint:phpstan - PHPStan static analysis

Custom Test Job

test:custom:
  stage: test
  image: node:20-alpine
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      exists: [package.json]
  cache:
    key: "${CI_COMMIT_REF_SLUG}-npm"
    paths: [node_modules/]
  script:
    - npm ci
    - npm run test:coverage
  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
  artifacts:
    when: always
    reports:
      junit: coverage/junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage/cobertura.xml

Parallel Test Execution

test:parallel:
  stage: test
  parallel:
    matrix:
      - TEST_SUITE: [unit, integration, e2e]
  script:
    - npm run test:${TEST_SUITE}

---

7. Test Organization

Directory Structure

project/
  src/
    services/
      my-service.ts
      my-service.test.ts    # Unit test co-located
  tests/
    setup.ts                # Shared setup
    fixtures/               # Test data
      agents.json
      users.json
    mocks/                  # Mock implementations
      api-client.ts
      database.ts
    unit/                   # Additional unit tests
      utils.test.ts
    integration/
      setup.ts
      api.test.ts
      database.test.ts
    e2e/
      setup.ts
      workflow.test.ts

Test File Naming

---

8. Testing OSSA Agents

Agent Test Patterns

describe('MyAgent', () => {
  describe('manifest validation', () => {
    it('should have valid OSSA manifest', async () => {
      const manifest = await loadManifest('.agents/my-agent/manifest.yaml');
      const errors = await validateManifest(manifest, 'v0.3.2');
      expect(errors).toHaveLength(0);
    });
  });

  describe('capability execution', () => {
    it('should execute registered capabilities', async () => {
      const agent = new MyAgent();
      const result = await agent.execute('k8s:read', {
        resource: 'pods',
        namespace: 'default'
      });
      expect(result.success).toBe(true);
    });
  });

  describe('budget awareness', () => {
    it('should respect token budget', async () => {
      const agent = new MyAgent({ budget: 100 });
      await agent.execute('expensive-operation', {});
      expect(agent.remainingBudget).toBeLessThan(100);
    });
  });
});

Compliance Testing

func TestComplianceProfileValidation(t *testing.T) {
    profiles := []*runtime.ComplianceProfile{
        runtime.FedRAMPProfile(),
        runtime.HIPAAProfile(),
        runtime.GDPRProfile(),
        runtime.SOC2Profile(),
    }

    for _, profile := range profiles {
        t.Run(string(profile.Type), func(t *testing.T) {
            validator := runtime.NewSimpleComplianceValidator(profile)

            turn := &runtime.TurnContext{
                TurnID:   "test-turn",
                Metadata: map[string]string{"mfa_verified": "true"},
            }

            violations, err := validator.ValidateTurnStart(
                context.Background(), turn,
            )

            if err != nil {
                t.Fatalf("Validation failed: %v", err)
            }

            if len(violations) > 0 {
                t.Logf("Violations: %v", violations)
            }
        })
    }
}

---

9. Mocking and Fixtures

Mock Factory Pattern

// tests/mocks/agent.ts
import type { Agent, AgentConfig } from '@/types';

export function createMockAgent(overrides: Partial<Agent> = {}): Agent {
  return {
    id: 'mock-agent-1',
    name: 'Mock Agent',
    type: 'worker',
    status: 'active',
    capabilities: ['k8s:read'],
    createdAt: new Date().toISOString(),
    ...overrides
  };
}

export function createMockAgentConfig(
  overrides: Partial<AgentConfig> = {}
): AgentConfig {
  return {
    maxRetries: 3,
    timeout: 30000,
    budgetLimit: 1000,
    ...overrides
  };
}

Fixture Loading

// tests/fixtures/loader.ts
import { readFileSync } from 'fs';
import { join } from 'path';

const fixturesDir = join(__dirname, '../fixtures');

export function loadFixture<T>(name: string): T {
  const path = join(fixturesDir, `${name}.json`);
  return JSON.parse(readFileSync(path, 'utf-8'));
}

// Usage
const agents = loadFixture<Agent[]>('agents');

---

10. Running Tests

NPM Scripts

{
  "scripts": {
    "test": "vitest run",
    "test:watch": "vitest",
    "test:unit": "vitest run --config vitest.config.ts",
    "test:integration": "vitest run --config vitest.integration.config.ts",
    "test:e2e": "vitest run --config vitest.e2e.config.ts",
    "test:coverage": "vitest run --coverage",
    "test:ci": "vitest run --coverage --reporter=junit --outputFile=coverage/junit.xml"
  }
}

Go Commands

# All tests
go test ./...

# Specific package
go test ./internal/ossa/runtime/...

# With verbose output
go test -v ./...

# Skip integration tests
go test -short ./...

# Race detection
go test -race ./...

# Coverage
go test -coverprofile=coverage.out ./...
go tool cover -html=coverage.out -o coverage.html

---

Related Documentation

• Code Review Standards
• CI/CD Pipeline
• OSSA Specification
• BuildKit CLI

---

<- Developer Guides | Next: Drupal Development ->