Skip to content

manage-data-sources.mdx

Latest commit

 

History

History
405 lines (316 loc) · 11.2 KB

manage-data-sources.mdx

File metadata and controls

405 lines (316 loc) · 11.2 KB

import WorkInProgressNotice from '@site/src/components/WorkInProgressNotice';

How to Manage Content Origins

Switch between different ATT&CK content origins efficiently

This guide shows you how to manage multiple ATT&CK content origins, switch between different versions, and work with local files, URLs, and the official repository.

Problem Scenarios

Use this guide when you need to:

  • Switch between different ATT&CK versions for compatibility testing
  • Load ATT&CK data from local files instead of the internet
  • Fetch data from custom URLs or mirrors
  • Manage multiple content origins in a production application
  • Cache and reuse content origins efficiently

Switch Between ATT&CK Versions

Compare Multiple Versions

import { registerContentOrigin, loadDataModel, ContentOriginRegistration } from '@mitre-attack/attack-data-model';

async function compareVersions() {
    const versions = ['15.0', '15.1'];
    const models: { [version: string]: any } = {};

    // Load multiple versions
    for (const version of versions) {
        const contentOrigin = new ContentOriginRegistration({
            source: 'mitre',
            domain: 'enterprise-attack',
            version: version,
            parsingMode: 'relaxed'
        });

        const uuid = await registerContentOrigin(contentOrigin);
        models[version] = loadDataModel(uuid);
    }

    // Compare technique counts
    versions.forEach(version => {
        const count = models[version].techniques.length;
        console.log(`ATT&CK ${version}: ${count} techniques`);
    });
}

Use Latest Version

// Omit version to get the latest available
const latestDataSource = new DataSource({
    source: 'mitre',
    domain: 'enterprise-attack',
    // No version specified = latest
    parsingMode: 'relaxed'
});

const uuid = await registerDataSource(latestDataSource);
const latestModel = loadDataModel(uuid);

Load from Local Files

Prepare Local STIX Bundle

First, save a STIX bundle JSON file locally:

# Download ATT&CK data
curl -o enterprise-attack.json https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/enterprise-attack/enterprise-attack.json

Load from Local File

import { DataSource } from '@mitre-attack/attack-data-model';
import path from 'path';

async function loadLocalData() {
    const filePath = path.resolve('./enterprise-attack.json');

    const localDataSource = new DataSource({
        source: 'file',
        filePath: filePath,
        parsingMode: 'relaxed'
    });

    try {
        const uuid = await registerDataSource(localDataSource);
        const model = loadDataModel(uuid);

        console.log(`Loaded ${model.techniques.length} techniques from local file`);
        return model;

    } catch (error) {
        console.error('Failed to load local file:', error);
        throw error;
    }
}

Load from Custom URLs

Load from URL Source

async function loadFromUrl() {
    const customUrl = 'https://your-custom-server.com/attack-data.json';

    const urlDataSource = new DataSource({
        source: 'url',
        url: customUrl,
        parsingMode: 'relaxed'
    });

    const uuid = await registerDataSource(urlDataSource);
    const model = loadDataModel(uuid);

    return model;
}

Load with Custom Headers

async function loadWithAuth() {
    const contentOrigin = new ContentOriginRegistration({
        source: 'url',
        url: 'https://private-server.com/attack-data.json',
        requestOptions: {
            headers: {
                'Authorization': 'Bearer YOUR_TOKEN',
                'X-Custom-Header': 'value'
            }
        },
        parsingMode: 'strict'
    });

    const uuid = await registerContentOrigin(contentOrigin);
    return loadDataModel(uuid);
}

Manage Multiple Data Sources

Data Source Registry Pattern

class AttackDataManager {
    private dataSources: Map<string, string> = new Map();

    async registerSource(name: string, config: any): Promise<string> {
        const dataSource = new DataSource(config);
        const uuid = await registerContentOrigin(contentOrigin);
        this.dataSources.set(name, uuid);
        return uuid;
    }

    getModel(name: string) {
        const uuid = this.dataSources.get(name);
        if (!uuid) {
            throw new Error(`Data source '${name}' not registered`);
        }
        return loadDataModel(uuid);
    }

    async setupCommonSources() {
        // Enterprise latest
        await this.registerSource('enterprise-latest', {
            source: 'mitre',
            domain: 'enterprise-attack',
            parsingMode: 'relaxed'
        });

        // Enterprise v15.0
        await this.registerSource('enterprise-v15', {
            source: 'mitre',
            domain: 'enterprise-attack',
            version: '15.0',
            parsingMode: 'relaxed'
        });

        // Mobile latest
        await this.registerSource('mobile-latest', {
            source: 'mitre',
            domain: 'mobile-attack',
            parsingMode: 'relaxed'
        });
    }
}

// Usage
const manager = new AttackDataManager();
await manager.setupCommonSources();

const enterpriseModel = manager.getModel('enterprise-latest');
const mobileModel = manager.getModel('mobile-latest');

Handle Data Source Errors

Graceful Fallback Pattern

async function loadWithFallback() {
    const fallbackSources = [
        // Try latest first
        {
            source: 'mitre',
            domain: 'enterprise-attack',
            parsingMode: 'relaxed'
        },
        // Fallback to specific version
        {
            source: 'mitre',
            domain: 'enterprise-attack',
            version: '15.1',
            parsingMode: 'relaxed'
        },
        // Final fallback to local file
        {
            source: 'file',
            filePath: './backup-enterprise-attack.json',
            parsingMode: 'relaxed'
        }
    ];

    for (const config of fallbackSources) {
        try {
            const dataSource = new DataSource(config);
            const uuid = await registerContentOrigin(contentOrigin);
            const model = loadDataModel(uuid);

            console.log(`Successfully loaded from source: ${config.source}`);
            return model;

        } catch (error) {
            console.warn(`Failed to load from ${config.source}:`, error.message);
            continue;
        }
    }

    throw new Error('All data sources failed');
}

Validate Data Source Before Use

async function validateDataSource(config: any): Promise<boolean> {
    try {
        const dataSource = new DataSource(config);
        const uuid = await registerContentOrigin(contentOrigin);
        const model = loadDataModel(uuid);

        // Basic validation checks
        const hasMinimumTechniques = model.techniques.length > 100;
        const hasBasicTactics = model.tactics.length > 5;
        const hasGroups = model.groups.length > 0;

        if (hasMinimumTechniques && hasBasicTactics && hasGroups) {
            console.log('✅ Data source validation passed');
            return true;
        } else {
            console.warn('⚠️  Data source seems incomplete');
            return false;
        }

    } catch (error) {
        console.error('❌ Data source validation failed:', error);
        return false;
    }
}

Cache and Performance

Implement Simple Caching

class CachedDataManager {
    private cache: Map<string, any> = new Map();

    private getCacheKey(config: any): string {
        return JSON.stringify(config);
    }

    async loadData(config: any) {
        const cacheKey = this.getCacheKey(config);

        // Check cache first
        if (this.cache.has(cacheKey)) {
            console.log('📋 Loading from cache');
            return this.cache.get(cacheKey);
        }

        // Load fresh data
        console.log('🌐 Loading fresh data');
        const dataSource = new DataSource(config);
        const uuid = await registerContentOrigin(contentOrigin);
        const model = loadDataModel(uuid);

        // Cache the result
        this.cache.set(cacheKey, model);

        return model;
    }

    clearCache() {
        this.cache.clear();
        console.log('🗑️  Cache cleared');
    }
}

Production Configuration

Environment-Based Data Sources

function getDataSourceConfig(): any {
    const environment = process.env.NODE_ENV || 'development';

    switch (environment) {
        case 'production':
            return {
                source: 'mitre',
                domain: 'enterprise-attack',
                version: '15.1', // Pin version in production
                parsingMode: 'strict' // Strict validation in production
            };

        case 'staging':
            return {
                source: 'url',
                url: process.env.STAGING_ATTACK_URL,
                parsingMode: 'relaxed'
            };

        case 'development':
        default:
            return {
                source: 'file',
                filePath: './dev-data/enterprise-attack.json',
                parsingMode: 'relaxed'
            };
    }
}

// Usage
const config = getDataSourceConfig();
const dataSource = new DataSource(config);

Monitoring and Logging

Add Data Source Monitoring

async function loadWithMonitoring(config: any) {
    const startTime = Date.now();

    try {
        console.log('📡 Starting data source load:', config.source);

        const dataSource = new DataSource(config);
        const uuid = await registerContentOrigin(contentOrigin);
        const model = loadDataModel(uuid);

        const loadTime = Date.now() - startTime;
        console.log(`✅ Load completed in ${loadTime}ms`);
        console.log(`📊 Loaded: ${model.techniques.length} techniques, ${model.groups.length} groups`);

        return model;

    } catch (error) {
        const loadTime = Date.now() - startTime;
        console.error(`❌ Load failed after ${loadTime}ms:`, error.message);
        throw error;
    }
}

Key Takeaways

  • Version Management: Pin specific versions in production, use latest for development
  • Fallback Strategy: Always have backup data sources for reliability
  • Validation: Verify data quality after loading, especially with custom sources
  • Caching: Implement caching for frequently accessed data to improve performance
  • Monitoring: Log data source performance and success rates for troubleshooting
  • Environment Configuration: Use different data sources for different deployment environments

Related Guides