Validation & Quality Gates

Overview

Quality gates ensure that only valid, well-formed business units are onboarded to Backstage. This prevents catalog pollution, ensures data quality, and maintains system integrity.

Validation Pipeline

Detection → Pre-Validation → Deep Validation → Quality Scoring → Decision

Pre-Validation (Fast Fail)

Purpose

Quickly reject obviously invalid candidates before expensive operations.

Checks

1. Tenant Authorization

// src/onboarding/validation/tenant-validator.ts
export class TenantValidator {
  async validate(tenantId: string): Promise<ValidationResult> {
    // Check tenant exists
    const tenant = await db('tenants').where('id', tenantId).first();

    if (!tenant) {
      return this.fail('Tenant not found');
    }

    // Check tenant is active
    if (tenant.status !== 'active') {
      return this.fail(`Tenant status is ${tenant.status}, expected active`);
    }

    // Check tenant has onboarding permissions
    if (!tenant.permissions.includes('backstage:onboard')) {
      return this.fail('Tenant does not have onboarding permissions');
    }

    // Check rate limits
    const recentOnboardings = await db('onboarding_history')
      .where('tenant_id', tenantId)
      .where('created_at', '>', new Date(Date.now() - 3600000)) // Last hour
      .count();

    if (recentOnboardings > tenant.rate_limits.onboarding_per_hour) {
      return this.fail('Rate limit exceeded');
    }

    return this.pass();
  }
}

2. Duplicate Detection

// src/onboarding/validation/duplicate-detector.ts
export class DuplicateDetector {
  async validate(repoUrl: string, force: boolean = false): Promise<ValidationResult> {
    const existing = await db('onboarding_history')
      .where('repo_url', repoUrl)
      .orderBy('created_at', 'desc')
      .first();

    if (!existing) {
      return this.pass('No existing onboarding found');
    }

    // If force=true, allow re-onboarding
    if (force) {
      return this.pass('Force onboarding requested');
    }

    // Check status of existing onboarding
    if (existing.status === 'completed') {
      return this.fail('Repository already onboarded', {
        existingId: existing.id,
        completedAt: existing.completed_at,
        entities: existing.entities_created,
      });
    }

    if (existing.status === 'in_progress') {
      return this.fail('Onboarding already in progress', {
        onboardingId: existing.id,
        startedAt: existing.created_at,
        currentState: existing.current_state,
      });
    }

    // If previous onboarding failed, allow retry
    if (existing.status === 'failed') {
      const hoursSinceFailure = (Date.now() - existing.failed_at.getTime()) / 3600000;

      if (hoursSinceFailure < 1) {
        return this.fail('Recent onboarding failure, wait before retry');
      }

      return this.pass('Previous onboarding failed, allowing retry');
    }

    return this.pass();
  }
}

3. Repository Accessibility

// src/onboarding/validation/repo-accessibility.ts
export class RepositoryAccessibilityValidator {
  async validate(repoUrl: string, tenantId: string): Promise<ValidationResult> {
    try {
      const { owner, repo } = this.parseRepoUrl(repoUrl);

      // Verify repository exists
      const repoData = await this.githubClient.repos.get({ owner, repo });

      if (!repoData.data) {
        return this.fail('Repository not found');
      }

      // Verify we have access
      if (repoData.data.private && !repoData.data.permissions?.admin) {
        return this.fail('Insufficient permissions on private repository');
      }

      // Verify repository belongs to correct organization (tenant isolation)
      const expectedOrg = await this.getExpectedOrg(tenantId);
      if (owner !== expectedOrg) {
        return this.fail(`Repository owner ${owner} does not match tenant organization ${expectedOrg}`);
      }

      return this.pass('Repository accessible');
    } catch (error) {
      if (error.status === 404) {
        return this.fail('Repository not found');
      }

      return this.fail(`GitHub API error: ${error.message}`);
    }
  }
}

4. Terraform Workspace Availability

// src/onboarding/validation/workspace-validator.ts
export class WorkspaceValidator {
  async validate(workspaceId: string, tenantId: string): Promise<ValidationResult> {
    try {
      const workspace = await this.tfcClient.getWorkspace(workspaceId);

      if (!workspace) {
        return this.fail('Workspace not found');
      }

      // Verify workspace belongs to correct organization
      const expectedOrg = await this.getExpectedTfcOrg(tenantId);
      if (workspace.organization.name !== expectedOrg) {
        return this.fail(`Workspace organization ${workspace.organization.name} does not match tenant ${expectedOrg}`);
      }

      // Verify workspace has at least one successful run
      if (workspace.run_count === 0) {
        return this.fail('Workspace has no runs yet');
      }

      const latestRun = await this.tfcClient.getLatestRun(workspaceId);

      if (latestRun.status !== 'applied') {
        return this.fail(`Latest run status is ${latestRun.status}, expected applied`);
      }

      return this.pass('Workspace valid and has successful runs');
    } catch (error) {
      return this.fail(`Terraform Cloud API error: ${error.message}`);
    }
  }
}

---

Deep Validation

Purpose

Thorough validation of metadata, state, and resources after detection.

Checks

1. Required Metadata

// src/onboarding/validation/metadata-validator.ts
export class MetadataValidator {
  private requiredFields = [
    'businessUnit',
    'owner',
    'system',
  ];

  private optionalFields = [
    'environment',
    'lifecycle',
    'description',
    'tags',
  ];

  validate(metadata: BusinessUnitMetadata): ValidationResult {
    const errors: string[] = [];
    const warnings: string[] = [];

    // Check required fields
    for (const field of this.requiredFields) {
      if (!metadata[field]) {
        errors.push(`Missing required field: ${field}`);
      }
    }

    // Validate business unit name format
    if (metadata.businessUnit && !this.isValidBusinessUnitName(metadata.businessUnit)) {
      errors.push('Business unit name must be lowercase alphanumeric with hyphens');
    }

    // Validate owner format
    if (metadata.owner && !this.isValidOwner(metadata.owner)) {
      warnings.push('Owner should be in format "group:team-name" or "user:email"');
    }

    // Validate system name
    if (metadata.system && !this.isValidSystemName(metadata.system)) {
      errors.push('System name must be lowercase alphanumeric with hyphens');
    }

    // Check optional fields for best practices
    if (!metadata.description) {
      warnings.push('No description provided, consider adding one');
    }

    if (!metadata.tags || metadata.tags.length === 0) {
      warnings.push('No tags provided, consider adding for better discoverability');
    }

    if (errors.length > 0) {
      return this.fail('Metadata validation failed', { errors, warnings });
    }

    if (warnings.length > 0) {
      return this.pass('Metadata valid with warnings', { warnings });
    }

    return this.pass('Metadata valid');
  }

  private isValidBusinessUnitName(name: string): boolean {
    return /^[a-z0-9-]+$/.test(name) && name.length >= 2 && name.length <= 50;
  }

  private isValidOwner(owner: string): boolean {
    return owner.match(/^(group|user):[a-z0-9-@.]+$/) !== null;
  }

  private isValidSystemName(name: string): boolean {
    return /^[a-z0-9-]+$/.test(name) && name.length >= 2 && name.length <= 100;
  }
}

2. Terraform State Validation

// src/onboarding/validation/state-validator.ts
export class TerraformStateValidator {
  private requiredResourceTypes = [
    'google_folder',
    'google_project',
  ];

  validate(state: TerraformState): ValidationResult {
    const errors: string[] = [];
    const warnings: string[] = [];

    // Validate state version
    if (!state.version || state.version < 4) {
      errors.push('Terraform state version must be >= 4');
    }

    // Check for required resources
    const resourceTypes = new Set(state.resources.map(r => r.type));

    for (const required of this.requiredResourceTypes) {
      if (!resourceTypes.has(required)) {
        errors.push(`Missing required resource type: ${required}`);
      }
    }

    // Validate resource instances
    for (const resource of state.resources) {
      if (resource.instances.length === 0) {
        warnings.push(`Resource ${resource.type}.${resource.name} has no instances`);
        continue;
      }

      for (const instance of resource.instances) {
        if (!instance.attributes) {
          errors.push(`Resource ${resource.type}.${resource.name} instance missing attributes`);
        }
      }
    }

    // Check for secrets in state (security check)
    const secretsFound = this.detectSecrets(state);
    if (secretsFound.length > 0) {
      warnings.push(`Potential secrets detected in state: ${secretsFound.join(', ')}`);
    }

    // Validate GCP folder structure
    const folderResources = state.resources.filter(r => r.type === 'google_folder');
    if (folderResources.length > 1) {
      warnings.push('Multiple folders detected, expected single root folder per BU');
    }

    if (errors.length > 0) {
      return this.fail('State validation failed', { errors, warnings });
    }

    return this.pass('State valid', { warnings });
  }

  private detectSecrets(state: TerraformState): string[] {
    const secretPatterns = [
      /api[_-]?key/i,
      /secret/i,
      /password/i,
      /token/i,
      /private[_-]?key/i,
    ];

    const secrets: string[] = [];
    const stateStr = JSON.stringify(state);

    for (const pattern of secretPatterns) {
      const matches = stateStr.match(pattern);
      if (matches) {
        secrets.push(...matches);
      }
    }

    return [...new Set(secrets)];
  }
}

3. GCP Resource Validation

// src/onboarding/validation/gcp-resource-validator.ts
export class GcpResourceValidator {
  async validate(resources: TerraformResource[]): Promise<ValidationResult> {
    const errors: string[] = [];
    const warnings: string[] = [];

    // Extract GCP resources
    const gcpResources = resources.filter(r => r.type.startsWith('google_'));

    if (gcpResources.length === 0) {
      return this.fail('No GCP resources found in state');
    }

    // Validate folder
    const folders = gcpResources.filter(r => r.type === 'google_folder');

    if (folders.length === 0) {
      errors.push('No GCP folder found');
    } else if (folders.length > 1) {
      warnings.push(`Multiple folders found: ${folders.length}`);
    } else {
      const folder = folders[0];
      const folderId = folder.instances[0]?.attributes?.name;

      // Verify folder exists in GCP
      try {
        const folderData = await this.gcpClient.cloudresourcemanager.folders.get({ name: folderId });
        if (!folderData) {
          errors.push(`Folder ${folderId} not found in GCP`);
        }
      } catch (error) {
        errors.push(`Failed to verify folder in GCP: ${error.message}`);
      }
    }

    // Validate projects
    const projects = gcpResources.filter(r => r.type === 'google_project');

    if (projects.length === 0) {
      warnings.push('No GCP projects found');
    } else {
      for (const project of projects) {
        const projectId = project.instances[0]?.attributes?.project_id;

        if (!projectId) {
          errors.push(`Project ${project.name} missing project_id`);
          continue;
        }

        // Verify project naming convention
        if (!this.isValidProjectId(projectId)) {
          warnings.push(`Project ID ${projectId} does not follow naming convention`);
        }

        // Verify project exists in GCP
        try {
          const projectData = await this.gcpClient.cloudresourcemanager.projects.get({ projectId });
          if (!projectData) {
            errors.push(`Project ${projectId} not found in GCP`);
          }
        } catch (error) {
          errors.push(`Failed to verify project ${projectId} in GCP: ${error.message}`);
        }
      }
    }

    if (errors.length > 0) {
      return this.fail('GCP resource validation failed', { errors, warnings });
    }

    return this.pass('GCP resources valid', { warnings });
  }

  private isValidProjectId(projectId: string): boolean {
    // GCP project IDs must be 6-30 characters, lowercase, numbers, hyphens
    return /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(projectId);
  }
}

4. Entity Consistency Validation

// src/onboarding/validation/entity-validator.ts
export class EntityConsistencyValidator {
  validate(entities: BackstageEntity[]): ValidationResult {
    const errors: string[] = [];
    const warnings: string[] = [];

    // Check for duplicate entity names
    const names = entities.map(e => e.metadata.name);
    const duplicates = names.filter((name, index) => names.indexOf(name) !== index);

    if (duplicates.length > 0) {
      errors.push(`Duplicate entity names: ${duplicates.join(', ')}`);
    }

    // Validate entity relationships
    const domain = entities.find(e => e.kind === 'Domain');
    const systems = entities.filter(e => e.kind === 'System');
    const components = entities.filter(e => e.kind === 'Component');

    if (!domain) {
      errors.push('No Domain entity found');
    }

    if (systems.length === 0) {
      warnings.push('No System entities found');
    }

    // Validate system references domain
    for (const system of systems) {
      if (system.spec.domain !== domain?.metadata.name) {
        errors.push(`System ${system.metadata.name} does not reference domain ${domain?.metadata.name}`);
      }
    }

    // Validate components reference system
    for (const component of components) {
      const referencedSystem = component.spec.system;

      if (!systems.find(s => s.metadata.name === referencedSystem)) {
        errors.push(`Component ${component.metadata.name} references non-existent system ${referencedSystem}`);
      }
    }

    // Validate annotations
    for (const entity of entities) {
      if (!entity.metadata.annotations?.['backstage.io/managed-by-location']) {
        warnings.push(`Entity ${entity.metadata.name} missing managed-by-location annotation`);
      }
    }

    if (errors.length > 0) {
      return this.fail('Entity consistency validation failed', { errors, warnings });
    }

    return this.pass('Entities consistent', { warnings });
  }
}

---

Quality Scoring

Purpose

Assign a quality score to each onboarding based on completeness and best practices.

Scoring Criteria

// src/onboarding/validation/quality-scorer.ts
export class QualityScorer {
  calculateScore(metadata: BusinessUnitMetadata, entities: BackstageEntity[]): QualityScore {
    let score = 0;
    const maxScore = 100;
    const breakdown: Record<string, number> = {};

    // Required metadata (30 points)
    if (metadata.businessUnit) score += 10;
    if (metadata.owner) score += 10;
    if (metadata.system) score += 10;
    breakdown.required_metadata = Math.min(30, score);

    // Optional metadata (20 points)
    if (metadata.description) score += 5;
    if (metadata.tags && metadata.tags.length > 0) score += 5;
    if (metadata.lifecycle) score += 5;
    if (metadata.links && metadata.links.length > 0) score += 5;
    breakdown.optional_metadata = Math.min(20, score - breakdown.required_metadata);

    // Entity completeness (20 points)
    const hasDomain = entities.some(e => e.kind === 'Domain');
    const hasSystem = entities.some(e => e.kind === 'System');
    const hasComponents = entities.some(e => e.kind === 'Component');

    if (hasDomain) score += 8;
    if (hasSystem) score += 7;
    if (hasComponents) score += 5;
    breakdown.entity_completeness = Math.min(20, score - breakdown.required_metadata - breakdown.optional_metadata);

    // Annotations (15 points)
    let annotationScore = 0;
    for (const entity of entities) {
      if (entity.metadata.annotations?.['backstage.io/managed-by-location']) annotationScore += 3;
      if (entity.metadata.annotations?.['cloud.google.com/project-id']) annotationScore += 2;
      if (entity.metadata.annotations?.['terraform.io/workspace']) annotationScore += 2;
    }
    score += Math.min(15, annotationScore);
    breakdown.annotations = Math.min(15, annotationScore);

    // Relationships (15 points)
    const domainSystemLinks = entities.filter(e =>
      e.kind === 'System' && e.spec.domain
    ).length;

    const systemComponentLinks = entities.filter(e =>
      e.kind === 'Component' && e.spec.system
    ).length;

    score += Math.min(8, domainSystemLinks * 4);
    score += Math.min(7, systemComponentLinks * 2);
    breakdown.relationships = Math.min(15, domainSystemLinks * 4 + systemComponentLinks * 2);

    // Best practices (bonus, up to 10 extra points)
    let bonusScore = 0;

    if (entities.some(e => e.metadata.labels?.['environment'])) bonusScore += 3;
    if (entities.some(e => e.metadata.annotations?.['backstage.io/techdocs-ref'])) bonusScore += 3;
    if (metadata.documentation) bonusScore += 4;

    score += bonusScore;
    breakdown.bonus = bonusScore;

    // Calculate grade
    const grade = this.calculateGrade(score);

    return {
      score: Math.min(maxScore, score),
      maxScore,
      grade,
      breakdown,
    };
  }

  private calculateGrade(score: number): string {
    if (score >= 90) return 'A';
    if (score >= 80) return 'B';
    if (score >= 70) return 'C';
    if (score >= 60) return 'D';
    return 'F';
  }
}

Quality Gate Decision

export class QualityGateDecision {
  decide(score: QualityScore, validationResult: ValidationResult): GateDecision {
    // Block onboarding if validation failed
    if (!validationResult.passed) {
      return {
        approved: false,
        reason: 'Validation failed',
        errors: validationResult.errors,
        blockers: true,
      };
    }

    // Require minimum score of 60 (grade D)
    if (score.score < 60) {
      return {
        approved: false,
        reason: `Quality score ${score.score} below minimum threshold of 60`,
        warnings: ['Consider adding more metadata and documentation'],
        blockers: false, // Can be overridden
      };
    }

    // Warn on low scores but approve
    if (score.score < 80) {
      return {
        approved: true,
        reason: `Quality score ${score.score} (grade ${score.grade})`,
        warnings: [
          'Consider improving metadata completeness',
          'Add documentation and tech docs',
          'Ensure all entities have proper annotations',
        ],
      };
    }

    // High quality onboarding
    return {
      approved: true,
      reason: `Excellent quality score ${score.score} (grade ${score.grade})`,
    };
  }
}

---

Validation Orchestrator

Combining All Validators

// src/onboarding/validation/orchestrator.ts
export class ValidationOrchestrator {
  constructor(
    private tenantValidator: TenantValidator,
    private duplicateDetector: DuplicateDetector,
    private repoAccessibilityValidator: RepositoryAccessibilityValidator,
    private workspaceValidator: WorkspaceValidator,
    private metadataValidator: MetadataValidator,
    private stateValidator: TerraformStateValidator,
    private gcpResourceValidator: GcpResourceValidator,
    private entityValidator: EntityConsistencyValidator,
    private qualityScorer: QualityScorer
  ) {}

  async validate(context: OnboardingContext): Promise<ValidationReport> {
    const report: ValidationReport = {
      passed: true,
      errors: [],
      warnings: [],
      validations: {},
    };

    // Pre-validation (fast fail)
    console.log('Running pre-validation...');

    const preValidations = await Promise.all([
      this.tenantValidator.validate(context.tenantId),
      this.duplicateDetector.validate(context.repoUrl, context.force),
      this.repoAccessibilityValidator.validate(context.repoUrl, context.tenantId),
      this.workspaceValidator.validate(context.workspaceId, context.tenantId),
    ]);

    for (const result of preValidations) {
      report.validations[result.name] = result;

      if (!result.passed) {
        report.passed = false;
        report.errors.push(...result.errors);
      }

      report.warnings.push(...result.warnings);
    }

    // If pre-validation failed, stop here
    if (!report.passed) {
      return report;
    }

    // Deep validation
    console.log('Running deep validation...');

    const deepValidations = await Promise.all([
      this.metadataValidator.validate(context.metadata),
      this.stateValidator.validate(context.terraformState),
      this.gcpResourceValidator.validate(context.terraformState.resources),
      this.entityValidator.validate(context.entities),
    ]);

    for (const result of deepValidations) {
      report.validations[result.name] = result;

      if (!result.passed) {
        report.passed = false;
        report.errors.push(...result.errors);
      }

      report.warnings.push(...result.warnings);
    }

    // Quality scoring
    console.log('Calculating quality score...');
    report.qualityScore = this.qualityScorer.calculateScore(
      context.metadata,
      context.entities
    );

    // Final decision
    const decision = new QualityGateDecision().decide(report.qualityScore, report);
    report.approved = decision.approved;
    report.decision = decision;

    return report;
  }
}

---

Error Handling

Validation Failure Actions

export class ValidationFailureHandler {
  async handle(context: OnboardingContext, report: ValidationReport): Promise<void> {
    if (!report.approved) {
      // Log failure
      await this.logValidationFailure(context, report);

      // Send notification
      if (report.decision.blockers) {
        await this.notifyBlockingFailure(context, report);
      } else {
        await this.notifyWarnings(context, report);
      }

      // Update onboarding status
      await db('onboarding_history').where('id', context.onboardingId).update({
        status: 'validation_failed',
        validation_report: JSON.stringify(report),
        failed_at: new Date(),
      });

      // Throw error to stop workflow
      throw new ValidationError('Onboarding validation failed', { report });
    }
  }

  private async notifyBlockingFailure(context: OnboardingContext, report: ValidationReport) {
    await slack.postMessage({
      channel: '#backstage-alerts',
      text: `❌ Onboarding validation failed for ${context.repoUrl}`,
      blocks: [
        {
          type: 'section',
          text: {
            type: 'mrkdwn',
            text: `*Onboarding ID:* ${context.onboardingId}\n` +
                  `*Repository:* ${context.repoUrl}\n` +
                  `*Business Unit:* ${context.metadata.businessUnit}\n` +
                  `*Errors:* ${report.errors.length}\n` +
                  `*Quality Score:* ${report.qualityScore?.score || 'N/A'}`,
          },
        },
        {
          type: 'section',
          text: {
            type: 'mrkdwn',
            text: `*Errors:*\n${report.errors.map(e => `• ${e}`).join('\n')}`,
          },
        },
      ],
    });
  }
}

---

Summary

Validation Flow

Input → Pre-Validation → Deep Validation → Quality Scoring → Decision → Action
  ↓           ↓                 ↓                ↓            ↓         ↓
Event    Tenant Check    Metadata Check    Score 0-100   Approve/   Continue/
         Duplicate       State Check       Grade A-F     Reject     Block
         Access          GCP Resources
         Workspace       Entity Consistency

Quality Standards

Grade	Score	Description	Action
A	90-100	Excellent	Auto-approve, silent onboarding
B	80-89	Good	Auto-approve, send notification
C	70-79	Acceptable	Approve with warnings
D	60-69	Minimal	Approve but flag for review
F	0-59	Insufficient	Block unless force=true

Next Steps

See 06-multi-client-isolation.md for tenant isolation strategy.