Ongoing Synchronization Setup

Overview

After successful onboarding, the system must continuously synchronize Terraform state changes with the Backstage catalog. This ensures entities remain accurate as infrastructure evolves.

Synchronization Architecture

Terraform Changes → TFC Webhook → Sync Handler → State Parser → Entity Updater → Backstage Catalog
                        │
                        ├─→ Scheduled Jobs (backup)
                        └─→ GitHub Watchers (code changes)

1. Terraform Cloud Webhooks

Webhook Configuration

// src/sync/setup/tfc-webhook-setup.ts
export class TfcWebhookSetup {
  /**
   * Configure TFC webhook for ongoing sync
   */
  async setup(
    workspaceId: string,
    tenantId: string,
    businessUnit: string
  ): Promise<WebhookConfig> {
    // Generate secure webhook token
    const webhookToken = await this.generateWebhookToken(workspaceId, tenantId);

    // Store token for verification
    await this.storeWebhookToken(workspaceId, webhookToken);

    // Create notification configuration
    const webhook = await this.tfcClient.createNotification({
      workspaceId,
      name: 'backstage-sync',
      enabled: true,
      destinationType: 'generic',
      url: `${process.env.BACKSTAGE_URL}/api/sync/tfc-webhook`,
      token: webhookToken,
      triggers: [
        'run:completed', // Successful applies
        'run:errored', // Failed runs (for alerting)
      ],
    });

    // Store webhook configuration
    await db('sync_webhooks').insert({
      workspace_id: workspaceId,
      tenant_id: tenantId,
      business_unit: businessUnit,
      webhook_id: webhook.id,
      webhook_url: webhook.url,
      enabled: true,
      created_at: new Date(),
    });

    console.log(`TFC webhook configured for workspace ${workspaceId}`);

    return {
      webhookId: webhook.id,
      url: webhook.url,
      token: webhookToken,
    };
  }

  private async generateWebhookToken(
    workspaceId: string,
    tenantId: string
  ): Promise<string> {
    // Generate cryptographically secure token
    const token = crypto.randomBytes(32).toString('hex');

    // Hash for storage
    const hashedToken = crypto
      .createHash('sha256')
      .update(token)
      .digest('hex');

    return token;
  }

  private async storeWebhookToken(
    workspaceId: string,
    token: string
  ): Promise<void> {
    const hashedToken = crypto
      .createHash('sha256')
      .update(token)
      .digest('hex');

    await db('webhook_tokens').insert({
      workspace_id: workspaceId,
      token_hash: hashedToken,
      created_at: new Date(),
    });
  }
}

Webhook Handler

// src/sync/handlers/tfc-webhook-handler.ts
export class TfcWebhookHandler {
  /**
   * Handle incoming TFC webhook
   */
  async handle(req: Request, res: Response): Promise<void> {
    // Verify webhook signature
    const isValid = await this.verifyWebhook(req);

    if (!isValid) {
      return res.status(401).json({ error: 'Invalid webhook signature' });
    }

    const payload = req.body;

    // Extract relevant data
    const { run_id, workspace_id, run_status, run_message } = payload;

    console.log(`TFC webhook received: workspace=${workspace_id}, run=${run_id}, status=${run_status}`);

    // Acknowledge webhook immediately
    res.status(202).json({ message: 'Webhook received, processing async' });

    // Process webhook asynchronously
    this.processWebhookAsync(payload).catch(error => {
      console.error('Webhook processing failed:', error);
      // Don't throw - webhook already acknowledged
    });
  }

  private async processWebhookAsync(payload: TfcWebhookPayload): Promise<void> {
    const { run_id, workspace_id, run_status } = payload;

    // Only process successful applies
    if (run_status !== 'applied') {
      console.log(`Skipping non-applied run: ${run_status}`);
      return;
    }

    // Find workspace configuration
    const syncConfig = await db('sync_webhooks')
      .where('workspace_id', workspace_id)
      .first();

    if (!syncConfig) {
      console.error(`No sync configuration found for workspace ${workspace_id}`);
      return;
    }

    // Trigger synchronization
    await this.syncService.syncWorkspace({
      workspaceId: workspace_id,
      tenantId: syncConfig.tenant_id,
      businessUnit: syncConfig.business_unit,
      runId: run_id,
      trigger: 'webhook',
    });
  }

  private async verifyWebhook(req: Request): Promise<boolean> {
    const signature = req.headers['x-tfc-notification-signature'];
    const workspaceId = req.body.workspace_id;

    if (!signature || !workspaceId) {
      return false;
    }

    // Retrieve stored token
    const tokenRecord = await db('webhook_tokens')
      .where('workspace_id', workspaceId)
      .first();

    if (!tokenRecord) {
      return false;
    }

    // Verify signature
    const payload = JSON.stringify(req.body);
    const expectedSignature = crypto
      .createHmac('sha256', tokenRecord.token_hash)
      .update(payload)
      .digest('hex');

    return crypto.timingSafeEqual(
      Buffer.from(signature),
      Buffer.from(expectedSignature)
    );
  }
}

---

2. Scheduled Synchronization Jobs

Job Scheduler

// src/sync/scheduler/sync-job-scheduler.ts
import { CronJob } from 'cron';

export class SyncJobScheduler {
  private jobs: Map<string, CronJob> = new Map();

  /**
   * Schedule periodic sync for workspace
   */
  async schedule(
    workspaceId: string,
    tenantId: string,
    businessUnit: string,
    options: ScheduleOptions = {}
  ): Promise<void> {
    const {
      schedule = '0 */6 * * *', // Every 6 hours by default
      enabled = true,
    } = options;

    // Create cron job
    const job = new CronJob(schedule, async () => {
      console.log(`Running scheduled sync for workspace ${workspaceId}`);

      try {
        await this.syncService.syncWorkspace({
          workspaceId,
          tenantId,
          businessUnit,
          trigger: 'scheduled',
        });

        console.log(`Scheduled sync completed for workspace ${workspaceId}`);
      } catch (error) {
        console.error(`Scheduled sync failed for workspace ${workspaceId}:`, error);

        // Alert on repeated failures
        await this.checkRepeatedFailures(workspaceId);
      }
    });

    // Store job configuration
    await db('sync_schedules').insert({
      workspace_id: workspaceId,
      tenant_id: tenantId,
      business_unit: businessUnit,
      schedule,
      enabled,
      next_run: this.calculateNextRun(schedule),
      created_at: new Date(),
    });

    // Start job if enabled
    if (enabled) {
      job.start();
      this.jobs.set(workspaceId, job);
      console.log(`Scheduled sync job for workspace ${workspaceId}: ${schedule}`);
    }
  }

  /**
   * Start all scheduled jobs
   */
  async startAll(): Promise<void> {
    const schedules = await db('sync_schedules')
      .where('enabled', true)
      .select('*');

    for (const schedule of schedules) {
      const job = new CronJob(schedule.schedule, async () => {
        await this.syncService.syncWorkspace({
          workspaceId: schedule.workspace_id,
          tenantId: schedule.tenant_id,
          businessUnit: schedule.business_unit,
          trigger: 'scheduled',
        });
      });

      job.start();
      this.jobs.set(schedule.workspace_id, job);
    }

    console.log(`Started ${schedules.length} scheduled sync jobs`);
  }

  /**
   * Stop scheduled job
   */
  async stop(workspaceId: string): Promise<void> {
    const job = this.jobs.get(workspaceId);

    if (job) {
      job.stop();
      this.jobs.delete(workspaceId);

      await db('sync_schedules')
        .where('workspace_id', workspaceId)
        .update({ enabled: false });

      console.log(`Stopped scheduled sync for workspace ${workspaceId}`);
    }
  }

  private calculateNextRun(schedule: string): Date {
    const job = new CronJob(schedule, () => {});
    return job.nextDate().toJSDate();
  }

  private async checkRepeatedFailures(workspaceId: string): Promise<void> {
    const recentFailures = await db('sync_history')
      .where('workspace_id', workspaceId)
      .where('status', 'failed')
      .where('created_at', '>', new Date(Date.now() - 86400000)) // Last 24 hours
      .count();

    if (recentFailures[0].count >= 3) {
      await this.alertRepeatedFailures(workspaceId, recentFailures[0].count);
    }
  }

  private async alertRepeatedFailures(workspaceId: string, failureCount: number): Promise<void> {
    await slack.postMessage({
      channel: '#backstage-alerts',
      text: `⚠️ Repeated sync failures for workspace ${workspaceId}`,
      blocks: [
        {
          type: 'section',
          text: {
            type: 'mrkdwn',
            text: `*Workspace:* ${workspaceId}\n` +
                  `*Failures:* ${failureCount} in last 24 hours\n` +
                  `*Action:* Manual investigation required`,
          },
        },
      ],
    });
  }
}

Sync Schedule Examples

# Default schedule (every 6 hours)
schedule: "0 */6 * * *"

# Hourly sync (high-activity workspaces)
schedule: "0 * * * *"

# Daily sync (stable workspaces)
schedule: "0 3 * * *"  # 3 AM daily

# Weekly sync (archived workspaces)
schedule: "0 2 * * 0"  # 2 AM Sunday

---

3. Synchronization Service

Core Sync Logic

// src/sync/service/sync-service.ts
export class SyncService {
  /**
   * Synchronize workspace with Backstage catalog
   */
  async syncWorkspace(params: SyncParams): Promise<SyncResult> {
    const { workspaceId, tenantId, businessUnit, runId, trigger } = params;

    const syncId = this.generateSyncId();

    console.log(`Starting sync ${syncId} for workspace ${workspaceId}`);

    try {
      // Record sync start
      await this.recordSyncStart(syncId, params);

      // 1. Fetch current Terraform state
      const state = await this.tfcClient.getCurrentState(workspaceId);

      // 2. Fetch existing entities
      const existingEntities = await this.fetchExistingEntities(tenantId, businessUnit);

      // 3. Parse state and generate new entities
      const newEntities = await this.stateParser.parse(state, {
        tenantId,
        businessUnit,
        workspaceId,
      });

      // 4. Calculate diff
      const diff = this.calculateDiff(existingEntities, newEntities);

      // 5. Apply changes
      const changes = await this.applyChanges(diff, tenantId);

      // 6. Record sync completion
      await this.recordSyncComplete(syncId, changes);

      console.log(`Sync ${syncId} completed: +${changes.added} ~${changes.updated} -${changes.deleted}`);

      return {
        syncId,
        status: 'success',
        changes,
      };
    } catch (error) {
      console.error(`Sync ${syncId} failed:`, error);

      // Record sync failure
      await this.recordSyncFailure(syncId, error);

      throw error;
    }
  }

  private async fetchExistingEntities(
    tenantId: string,
    businessUnit: string
  ): Promise<BackstageEntity[]> {
    const entities = await db('entities')
      .where('tenant_id', tenantId)
      .where('metadata->labels->business-unit', businessUnit)
      .select('*');

    return entities.map(this.deserializeEntity);
  }

  private calculateDiff(
    existing: BackstageEntity[],
    updated: BackstageEntity[]
  ): EntityDiff {
    const existingMap = new Map(existing.map(e => [e.metadata.name, e]));
    const updatedMap = new Map(updated.map(e => [e.metadata.name, e]));

    const added: BackstageEntity[] = [];
    const modified: BackstageEntity[] = [];
    const deleted: BackstageEntity[] = [];

    // Find added and modified
    for (const [name, entity] of updatedMap) {
      if (!existingMap.has(name)) {
        added.push(entity);
      } else {
        const existingEntity = existingMap.get(name);
        if (!this.areEntitiesEqual(existingEntity, entity)) {
          modified.push(entity);
        }
      }
    }

    // Find deleted
    for (const [name, entity] of existingMap) {
      if (!updatedMap.has(name)) {
        deleted.push(entity);
      }
    }

    return { added, modified, deleted };
  }

  private async applyChanges(
    diff: EntityDiff,
    tenantId: string
  ): Promise<SyncChanges> {
    const changes: SyncChanges = {
      added: 0,
      updated: 0,
      deleted: 0,
    };

    await db.transaction(async (trx) => {
      // Add new entities
      for (const entity of diff.added) {
        await trx('entities').insert(this.serializeEntity(entity, tenantId));
        changes.added++;
      }

      // Update modified entities
      for (const entity of diff.modified) {
        await trx('entities')
          .where({
            'metadata->name': entity.metadata.name,
            'metadata->namespace': entity.metadata.namespace,
          })
          .update({
            metadata: JSON.stringify(entity.metadata),
            spec: JSON.stringify(entity.spec),
            updated_at: new Date(),
          });
        changes.updated++;
      }

      // Delete removed entities
      for (const entity of diff.deleted) {
        await trx('entities')
          .where({
            'metadata->name': entity.metadata.name,
            'metadata->namespace': entity.metadata.namespace,
          })
          .delete();
        changes.deleted++;
      }
    });

    return changes;
  }

  private areEntitiesEqual(a: BackstageEntity, b: BackstageEntity): boolean {
    // Deep comparison (excluding timestamps)
    const aClean = { ...a, metadata: { ...a.metadata } };
    const bClean = { ...b, metadata: { ...b.metadata } };

    delete aClean.metadata.annotations?.['backstage.io/updated-at'];
    delete bClean.metadata.annotations?.['backstage.io/updated-at'];

    return JSON.stringify(aClean) === JSON.stringify(bClean);
  }
}

---

4. GitHub Repository Watchers

Code Change Detection

// src/sync/watchers/github-watcher.ts
export class GitHubRepositoryWatcher {
  /**
   * Setup GitHub webhook for repository changes
   */
  async setupWebhook(
    owner: string,
    repo: string,
    tenantId: string
  ): Promise<void> {
    const webhookUrl = `${process.env.BACKSTAGE_URL}/api/sync/github-webhook`;
    const webhookSecret = await this.generateWebhookSecret(owner, repo);

    // Store secret
    await this.storeWebhookSecret(owner, repo, webhookSecret);

    // Create webhook
    await this.githubClient.repos.createWebhook({
      owner,
      repo,
      config: {
        url: webhookUrl,
        content_type: 'json',
        secret: webhookSecret,
      },
      events: [
        'push', // Code changes
        'repository', // Repository metadata changes
      ],
    });

    console.log(`GitHub webhook configured for ${owner}/${repo}`);
  }

  /**
   * Handle GitHub webhook
   */
  async handleWebhook(req: Request, res: Response): Promise<void> {
    // Verify signature
    const isValid = await this.verifyGitHubSignature(req);

    if (!isValid) {
      return res.status(401).json({ error: 'Invalid signature' });
    }

    const event = req.headers['x-github-event'];
    const payload = req.body;

    // Acknowledge immediately
    res.status(202).json({ message: 'Webhook received' });

    // Process asynchronously
    if (event === 'push') {
      await this.handlePushEvent(payload);
    } else if (event === 'repository') {
      await this.handleRepositoryEvent(payload);
    }
  }

  private async handlePushEvent(payload: GitHubPushPayload): Promise<void> {
    const { repository, commits } = payload;

    // Check if .backstage/config.yaml was modified
    const configModified = commits.some(commit =>
      commit.modified?.includes('.backstage/config.yaml') ||
      commit.added?.includes('.backstage/config.yaml')
    );

    if (configModified) {
      console.log(`Backstage config modified in ${repository.full_name}`);

      // Trigger re-sync
      await this.syncService.syncRepository({
        repoUrl: repository.html_url,
        trigger: 'config_change',
      });
    }
  }

  private async handleRepositoryEvent(payload: GitHubRepositoryPayload): Promise<void> {
    const { action, repository } = payload;

    if (action === 'edited') {
      console.log(`Repository metadata changed: ${repository.full_name}`);

      // Update entity metadata (description, topics, etc.)
      await this.updateEntityMetadata(repository);
    }
  }
}

---

5. Drift Detection

State Drift Monitor

// src/sync/drift/drift-detector.ts
export class DriftDetector {
  /**
   * Detect drift between Terraform state and Backstage catalog
   */
  async detectDrift(workspaceId: string, tenantId: string): Promise<DriftReport> {
    // Fetch Terraform state
    const tfState = await this.tfcClient.getCurrentState(workspaceId);

    // Fetch Backstage entities
    const backstageEntities = await this.fetchEntities(workspaceId, tenantId);

    // Parse state to entities
    const stateEntities = await this.stateParser.parse(tfState, { tenantId });

    // Compare
    const drift = this.compareSets(stateEntities, backstageEntities);

    if (drift.hasDrift) {
      console.warn(`Drift detected for workspace ${workspaceId}`);

      // Notify
      await this.notifyDrift(workspaceId, drift);

      // Auto-reconcile if configured
      if (await this.shouldAutoReconcile(workspaceId)) {
        await this.reconcile(workspaceId, tenantId, drift);
      }
    }

    return drift;
  }

  private compareSets(
    stateEntities: BackstageEntity[],
    catalogEntities: BackstageEntity[]
  ): DriftReport {
    const stateMap = new Map(stateEntities.map(e => [e.metadata.name, e]));
    const catalogMap = new Map(catalogEntities.map(e => [e.metadata.name, e]));

    const missing: string[] = []; // In state but not in catalog
    const extra: string[] = []; // In catalog but not in state
    const diverged: Array<{ name: string; diff: object }> = [];

    // Check for missing and diverged
    for (const [name, stateEntity] of stateMap) {
      const catalogEntity = catalogMap.get(name);

      if (!catalogEntity) {
        missing.push(name);
      } else if (!this.areEntitiesEqual(stateEntity, catalogEntity)) {
        diverged.push({
          name,
          diff: this.calculateDiff(stateEntity, catalogEntity),
        });
      }
    }

    // Check for extra
    for (const name of catalogMap.keys()) {
      if (!stateMap.has(name)) {
        extra.push(name);
      }
    }

    return {
      hasDrift: missing.length > 0 || extra.length > 0 || diverged.length > 0,
      missing,
      extra,
      diverged,
    };
  }

  private async reconcile(
    workspaceId: string,
    tenantId: string,
    drift: DriftReport
  ): Promise<void> {
    console.log(`Auto-reconciling drift for workspace ${workspaceId}`);

    await this.syncService.syncWorkspace({
      workspaceId,
      tenantId,
      trigger: 'drift_reconciliation',
    });
  }
}

---

6. Sync Monitoring & Observability

Sync Metrics

// src/sync/monitoring/sync-metrics.ts
export class SyncMetrics {
  recordSyncDuration(workspaceId: string, durationMs: number): void {
    metrics.histogram('sync.duration_ms', durationMs, {
      workspace_id: workspaceId,
    });
  }

  recordSyncSuccess(workspaceId: string, changes: SyncChanges): void {
    metrics.counter('sync.success', 1, { workspace_id: workspaceId });
    metrics.gauge('sync.changes.added', changes.added, { workspace_id: workspaceId });
    metrics.gauge('sync.changes.updated', changes.updated, { workspace_id: workspaceId });
    metrics.gauge('sync.changes.deleted', changes.deleted, { workspace_id: workspaceId });
  }

  recordSyncFailure(workspaceId: string, error: Error): void {
    metrics.counter('sync.failure', 1, {
      workspace_id: workspaceId,
      error_type: error.constructor.name,
    });
  }

  recordDriftDetected(workspaceId: string, drift: DriftReport): void {
    metrics.gauge('sync.drift.missing', drift.missing.length, { workspace_id: workspaceId });
    metrics.gauge('sync.drift.extra', drift.extra.length, { workspace_id: workspaceId });
    metrics.gauge('sync.drift.diverged', drift.diverged.length, { workspace_id: workspaceId });
  }
}

Sync Dashboard

# Grafana dashboard
Dashboard: Backstage Synchronization
Panels:
  - Sync Success Rate (24h)
  - Sync Duration (p50, p95, p99)
  - Entity Changes (added, updated, deleted)
  - Drift Detection
  - Webhook Delivery Rate
  - Scheduled Job Runs
  - Error Rate by Workspace

---

7. Configuration Management

Sync Configuration

# config/sync-config.yaml
synchronization:
  webhooks:
    tfc:
      enabled: true
      url: "${BACKSTAGE_URL}/api/sync/tfc-webhook"
      timeout: 5000
    github:
      enabled: true
      url: "${BACKSTAGE_URL}/api/sync/github-webhook"
      events: [push, repository]

  scheduled:
    enabled: true
    default_schedule: "0 */6 * * *"  # Every 6 hours
    max_concurrent: 5

  drift:
    detection_enabled: true
    auto_reconcile: true
    check_interval: "0 */12 * * *"  # Every 12 hours

  retry:
    max_attempts: 3
    backoff: exponential
    base_delay: 2000

  notifications:
    slack:
      enabled: true
      channel: "#backstage-sync"
      notify_on_failure: true
      notify_on_drift: true

---

Summary

Sync Mechanisms

Mechanism	Trigger	Latency	Use Case
TFC Webhook	Terraform apply	< 1s	Real-time state changes
GitHub Webhook	Code push	< 1s	Config file changes
Scheduled Job	Cron schedule	Hours	Backup sync, drift detection
Manual Trigger	API/UI	Immediate	On-demand refresh

Reliability Features

✅ Real-time synchronization (webhooks) ✅ Backup scheduled sync (cron jobs) ✅ Drift detection (automated reconciliation) ✅ Retry with backoff (automatic recovery) ✅ Comprehensive monitoring (metrics & alerts) ✅ Idempotent operations (safe to retry)

Next Steps

See 09-implementation-guide.md for step-by-step implementation instructions.