diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a6f3284 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,89 @@ +# SOFE Security Checklist + +Formal security checklist for anyone who deploys or operates SOFE (self-hosted +or the hosted SaaS). Built from a real audit of the SOFE stack (2026-09). + +Severity legend: πŸ”΄ Critical Β· 🟠 High Β· 🟑 Medium Β· 🟒 Good practice + +--- + +## 1. Repositories & Supply Chain + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 1.1 | No secrets committed: `AKIA*`, `sk_*`, `AIzaSy*`, private keys, `.env`, `.dev.vars`, `wrangler.toml` `[vars]` with credentials | πŸ”΄ | | +| 1.2 | No secrets in git history (`git log -p` / `trufflehog` / `gitleaks`) | πŸ”΄ | | +| 1.3 | `build/`, `.terraform/`, `*.tfstate*`, `lambda.zip`, `node_modules/`, `.wrangler/`, `.DS_Store` are gitignored and NOT tracked | 🟑 | | +| 1.4 | CI workflows use least-privilege `permissions:` and no plaintext secrets | 🟠 | | +| 1.5 | Release artifacts (PyPI, GoReleaser, Docker) publish `checksums.txt` / SBOM | 🟒 | | +| 1.6 | Install scripts verify checksum/signature instead of bare `curl | bash` | 🟠 | | +| 1.7 | Only necessary collaborators with admin access on public repos | 🟑 | | + +## 2. Authentication + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 2.1 | Bearer tokens (Firebase ID tokens / JWTs) are **signature-verified** (JWKS) with `iss`/`aud`/`exp` checks β€” never just base64-decoded | πŸ”΄ | | +| 2.2 | API keys stored as SHA-256 hashes, not plaintext (even in a field named `keyHash`) | πŸ”΄ | | +| 2.3 | Internal secrets (Lambda ↔ Worker) use a shared secret header and the backend rejects requests without it | πŸ”΄ | | +| 2.4 | Device-code flow completes on the backend; browser never writes `device_codes`/`api_keys` via unauthenticated REST | πŸ”΄ | | +| 2.5 | Admin-only endpoints check role via a trusted path (backend/service account), not client-supplied claims | 🟠 | | +| 2.6 | Rate limiting applied at every public entry point (incl. raw backend URLs, not just the edge) | 🟠 | | + +## 3. Data Store (Firestore / DB) + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 3.1 | Security rules require `request.auth != null` β€” no anonymous reads/writes with only a public API key | πŸ”΄ | | +| 3.2 | Owner-scoped collections (`users/{uid}/**`, `connectedAccounts`, `evaluations`) only readable/writable by owner or backend | πŸ”΄ | | +| 3.3 | `api_keys` and `device_codes` (post-approval) never readable by other users | πŸ”΄ | | +| 3.4 | Backend Workers authenticate with a service account (OAuth2 token), never the web API key | πŸ”΄ | | +| 3.5 | Sensitive user fields (third-party AI API keys, external IDs) encrypted at rest | 🟠 | | +| 3.6 | Rules have a default-deny catch-all; `get()`/`exists()` used only for trusted checks | 🟒 | | + +## 4. Compute (Lambda / Workers / Pages) + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 4.1 | No publicly reachable backend URL without auth (API Gateway catch-all `$default` + unauthenticated Lambda = πŸ”΄) | πŸ”΄ | | +| 4.2 | Lambda/IAM role follows least privilege: no `sts:AssumeRole Resource="*"`, no `List*`/`Describe*` beyond what's needed | πŸ”΄ | | +| 4.3 | Cross-account endpoints (`/connect/test`, Bedrock proxy) validate ARN format and require an external ID | 🟠 | | +| 4.4 | Workers reject requests to internal endpoints unless a valid `x-sofe-internal-secret` is present | 🟠 | | +| 4.5 | `reload=True` and dev flags disabled in production | 🟑 | | +| 4.6 | WAF / throttling on public endpoints; function URLs not exposed | 🟒 | | + +## 5. Secrets Management + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 5.1 | All credentials stored as Cloudflare secrets / AWS SSM / env β€” never in committed `[vars]`/`wrangler.toml` | πŸ”΄ | | +| 5.2 | Rotation plan: API keys, external IDs, cron secrets, web API keys on any exposure | πŸ”΄ | | +| 5.3 | `.dev.vars.example` committed instead of `.dev.vars` | 🟒 | | +| 5.4 | Service accounts scoped to one project with `datastore` (or narrower) access | 🟒 | | +| 5.5 | **Follow-up**: migrate Workers to Workload Identity Federation (OIDC) to avoid long-lived service-account keys (Google-recommended) | 🟑 | | + +## 6. Client / CLI + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 6.1 | CLI config file (`~/.sofe/config.yaml`) written `0600` and API keys stored securely (or OS keychain) | 🟑 | | +| 6.2 | Remote templates/fixtures fetched over HTTPS and validated (no `http.Get` of untrusted URLs) | 🟠 | | +| 6.3 | CLI updater verifies artifact checksum | 🟠 | | + +## 7. Documentation & Disclosure + +| # | Check | Sev | Status | +|---|-------|-----|--------| +| 7.1 | No real credentials, internal URLs, account IDs, or external IDs in docs/README/blog | 🟠 | | +| 7.2 | `SECURITY.md` with a private disclosure contact and response expectations | 🟒 | | +| 7.3 | Public-facing blog posts about audits are anonymized (no real endpoints/secrets/account IDs) | 🟑 | | + +--- + +## Remediation priority (if you're starting from πŸ”΄) + +1. **Firestore rules** β†’ require auth; owner-scoped collections; default deny. +2. **Rotate** every exposed credential (API keys, external IDs, web API key, cron secret). +3. **Workers** β†’ verify JWT signatures, hash API keys, use a service account for Firestore. +4. **Lambda** β†’ require the internal secret, restrict `sts:AssumeRole`, remove the public catch-all. +5. **Supply chain** β†’ checksummed installs, gitignore hygiene, no secrets in git history. \ No newline at end of file diff --git a/build/lib/sofe/__init__.py b/build/lib/sofe/__init__.py deleted file mode 100644 index 8ab64e3..0000000 --- a/build/lib/sofe/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""SOFE β€” Stairway Open FinOps Engine. Policies as Code for AWS.""" -__version__ = "0.1.0" diff --git a/build/lib/sofe/cli.py b/build/lib/sofe/cli.py deleted file mode 100644 index 54a273b..0000000 --- a/build/lib/sofe/cli.py +++ /dev/null @@ -1,121 +0,0 @@ -"""SOFE CLI β€” sofe evaluate, sofe validate.""" - -import json -import click -from .loader import load_policies, validate_policies -from .engine import evaluate -from .models import Finding - - -@click.group() -@click.version_option(version="0.1.0") -def main(): - """SOFE β€” Stairway Open FinOps Engine. Policies as Code for AWS.""" - pass - - -@main.command() -@click.option("--policies", "-p", required=True, help="Path to policies directory or file") -@click.option("--format", "-f", "fmt", default="table", type=click.Choice(["table", "json", "markdown"])) -@click.option("--min-severity", default=None, type=click.Choice(["critical", "high", "medium", "low", "info"])) -@click.option("--fail-on", default=None, type=click.Choice(["critical", "high", "medium", "low"])) -@click.option("--profile", default=None, help="AWS profile to use") -@click.option("--dry-run", is_flag=True, help="Show what would be evaluated without calling AWS") -def evaluate_cmd(policies, fmt, min_severity, fail_on, profile, dry_run): - """Evaluate policies against live AWS resources.""" - from .collectors import collect_all - - click.echo(f"πŸ“‹ Loading policies from: {policies}") - policy_list = load_policies(policies) - click.echo(f" Found {len(policy_list)} policies") - - if dry_run: - click.echo("\nπŸ” DRY RUN β€” would evaluate:") - for p in policy_list: - click.echo(f" β€’ {p.metadata.name} ({p.spec.severity.value}) β†’ {p.spec.scope.resource_types}") - return - - click.echo(f"\n☁️ Scanning AWS resources (profile: {profile or 'default'})...") - resources = collect_all(profile=profile, resource_types=_get_required_types(policy_list)) - click.echo(f" Found {len(resources)} resources") - - click.echo(f"\n⚑ Evaluating {len(policy_list)} policies against {len(resources)} resources...") - findings = evaluate(policy_list, resources) - - # Filter by severity - if min_severity: - severity_order = ["critical", "high", "medium", "low", "info"] - min_idx = severity_order.index(min_severity) - findings = [f for f in findings if severity_order.index(f.severity.value) <= min_idx] - - # Output - if fmt == "json": - click.echo(json.dumps([f.model_dump(mode="json") for f in findings], indent=2, default=str)) - elif fmt == "markdown": - _output_markdown(findings) - else: - _output_table(findings) - - # Exit code - if fail_on and findings: - severity_order = ["critical", "high", "medium", "low"] - fail_idx = severity_order.index(fail_on) - blocking = [f for f in findings if severity_order.index(f.severity.value) <= fail_idx] - if blocking: - raise SystemExit(1) - - -@main.command() -@click.option("--policies", "-p", required=True, help="Path to policies directory or file") -def validate(policies): - """Validate policy YAML files without evaluating.""" - results = validate_policies(policies) - all_valid = True - for r in results: - status = "βœ…" if r["valid"] else "❌" - click.echo(f" {status} {r['file']}") - if r["error"]: - click.echo(f" {r['error']}") - all_valid = False - - click.echo(f"\n{'All valid βœ…' if all_valid else 'Some invalid ❌'}") - if not all_valid: - raise SystemExit(1) - - -def _get_required_types(policies) -> list[str]: - types = set() - for p in policies: - types.update(p.spec.scope.resource_types) - return list(types) - - -def _output_table(findings: list[Finding]): - if not findings: - click.echo("\nβœ… No violations found!") - return - - click.echo(f"\n{'─'*80}") - click.echo(f"{'Severity':<10} {'Policy':<25} {'Resource':<20} {'Message':<25}") - click.echo(f"{'─'*80}") - icons = {"critical": "πŸ”΄", "high": "🟠", "medium": "🟑", "low": "πŸ”΅", "info": "βšͺ"} - for f in findings: - icon = icons.get(f.severity.value, "βšͺ") - click.echo(f"{icon} {f.severity.value:<8} {f.policy_name:<25} {f.resource_id:<20} {f.message:<25}") - - total_savings = sum(f.estimated_savings or 0 for f in findings) - click.echo(f"{'─'*80}") - click.echo(f"Summary: {len(findings)} findings | Potential savings: ${total_savings:.2f}/mo") - - -def _output_markdown(findings: list[Finding]): - click.echo(f"# SOFE Evaluation Results\n") - click.echo(f"**Findings:** {len(findings)}\n") - click.echo("| Severity | Policy | Resource | Message |") - click.echo("|----------|--------|----------|---------|") - for f in findings: - click.echo(f"| {f.severity.value} | {f.policy_name} | {f.resource_id} | {f.message} |") - - -if __name__ == "__main__": - main() diff --git a/build/lib/sofe/collectors/__init__.py b/build/lib/sofe/collectors/__init__.py deleted file mode 100644 index 3fbd5eb..0000000 --- a/build/lib/sofe/collectors/__init__.py +++ /dev/null @@ -1,113 +0,0 @@ -"""AWS resource collectors for SOFE β€” scan resources + fetch metrics.""" - -from __future__ import annotations -import boto3 -from datetime import datetime, timedelta -from ..models import Resource -from .aws import COLLECTORS, ALL_TYPES - - -def collect_all(profile: str = None, resource_types: list[str] = None, regions: list[str] = None) -> list[Resource]: - """Collect resources from AWS using the given profile and modular collectors.""" - session = boto3.Session(profile_name=profile) if profile else boto3.Session() - account_id = session.client('sts').get_caller_identity()['Account'] - target_regions = regions or [session.region_name or 'us-east-1'] - types = resource_types or ALL_TYPES - - resources: list[Resource] = [] - - for region in target_regions: - for rtype in types: - if rtype in COLLECTORS: - collector = COLLECTORS[rtype](session=session, region=region, account_id=account_id) - collected = collector.collect() - resources.extend(collected) - if collected: - print(f" βœ“ {rtype}: {len(collected)} resources in {region}") - - # Enrich with metrics - _enrich_metrics(session, resources, target_regions[0]) - - # Enrich with real costs from Cost Explorer - _enrich_costs(session, resources, account_id) - - # Tag-based metrics - for r in resources: - for key in ['owner', 'env', 'costCenter', 'Environment', 'Name']: - r.metrics[f'has_tag:{key}'] = 1.0 if key in r.tags else 0.0 - - return resources - - -def _enrich_metrics(session: boto3.Session, resources: list[Resource], region: str): - """Fetch CloudWatch metrics for resources (CPU, cost).""" - try: - cw = session.client('cloudwatch', region_name=region) - ce = session.client('ce', region_name='us-east-1') - now = datetime.utcnow() - start = now - timedelta(days=30) - - for r in resources: - if r.resource_type == 'aws.ec2': - try: - resp = cw.get_metric_statistics( - Namespace='AWS/EC2', MetricName='CPUUtilization', - Dimensions=[{'Name': 'InstanceId', 'Value': r.resource_id}], - StartTime=start, EndTime=now, Period=86400 * 30, Statistics=['Average'], - ) - if resp['Datapoints']: - r.metrics['avg_cpu_utilization'] = round(resp['Datapoints'][0]['Average'], 2) - except: - pass - - # Running days for EC2 - if r.resource_type == 'aws.ec2' and 'launch_time' in r.properties and r.properties['launch_time']: - try: - from dateutil.parser import parse - launch = parse(r.properties['launch_time']) - r.metrics['running_days'] = (now - launch.replace(tzinfo=None)).days - except: - pass - - # Monthly cost per resource β€” now handled by _enrich_costs() via CostCollector - # (keeping this as fallback for accounts without ce:GetCostAndUsageWithResources) - try: - s = (now.replace(day=1) - timedelta(days=1)).replace(day=1).strftime('%Y-%m-%d') - e = now.replace(day=1).strftime('%Y-%m-%d') - resp = ce.get_cost_and_usage(TimePeriod={'Start': s, 'End': e}, Granularity='MONTHLY', Metrics=['UnblendedCost']) - total = float(resp['ResultsByTime'][0]['Total']['UnblendedCost']['Amount']) - if resources and not any(r.metrics.get('monthly_cost') for r in resources): - # Only apply naΓ―ve distribution if CostCollector didn't run - per_resource = total / len(resources) - for r in resources: - r.metrics.setdefault('monthly_cost', round(per_resource, 2)) - except: - pass - except: - pass - - -def _enrich_costs(session: boto3.Session, resources: list[Resource], account_id: str): - """Enrich resources with real per-resource costs from Cost Explorer.""" - from .aws.cost import CostCollector - - try: - cost_collector = CostCollector(session=session, region="us-east-1", account_id=account_id) - cost_collector.collect() - cost_map = cost_collector.get_cost_map() - - if not cost_map: - return # No cost data available (permission denied or CE not enabled) - - enriched = 0 - for r in resources: - cost = cost_collector.get_cost_for_resource(r.resource_id) - if cost is not None: - r.metrics['monthly_cost'] = cost - enriched += 1 - - if enriched: - print(f" πŸ’° Cost data: {enriched}/{len(resources)} resources enriched (total: ${cost_collector.get_total_cost()}/mo)") - - except Exception: - pass # Graceful fallback β€” evaluation works without cost data diff --git a/build/lib/sofe/collectors/aws/__init__.py b/build/lib/sofe/collectors/aws/__init__.py deleted file mode 100644 index e81c2b6..0000000 --- a/build/lib/sofe/collectors/aws/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -"""AWS Collectors registry β€” all 18 collectors.""" - -from .ec2 import EC2Collector -from .s3 import S3Collector -from .lambda_ import LambdaCollector -from .rds import RDSCollector -from .ebs import EBSCollector -from .ecs import ECSCollector -from .eks import EKSCollector -from .elasticache import ElastiCacheCollector -from .redshift import RedshiftCollector -from .dynamodb import DynamoDBCollector -from .cloudfront import CloudFrontCollector -from .apigateway import APIGatewayCollector -from .natgateway import NATGatewayCollector -from .elb import ELBCollector -from .route53 import Route53Collector -from .secretsmanager import SecretsManagerCollector -from .sagemaker import SageMakerCollector -from .cost import CostCollector - -# Registry: resource_type β†’ Collector class -COLLECTORS = { - "aws.ec2": EC2Collector, - "aws.s3": S3Collector, - "aws.lambda": LambdaCollector, - "aws.rds": RDSCollector, - "aws.ebs": EBSCollector, - "aws.ecs": ECSCollector, - "aws.eks": EKSCollector, - "aws.elasticache": ElastiCacheCollector, - "aws.redshift": RedshiftCollector, - "aws.dynamodb": DynamoDBCollector, - "aws.cloudfront": CloudFrontCollector, - "aws.apigateway": APIGatewayCollector, - "aws.natgateway": NATGatewayCollector, - "aws.elb": ELBCollector, - "aws.route53": Route53Collector, - "aws.secretsmanager": SecretsManagerCollector, - "aws.sagemaker": SageMakerCollector, - "aws.cost": CostCollector, -} - -ALL_TYPES = list(COLLECTORS.keys()) diff --git a/build/lib/sofe/collectors/aws/apigateway.py b/build/lib/sofe/collectors/aws/apigateway.py deleted file mode 100644 index c553411..0000000 --- a/build/lib/sofe/collectors/aws/apigateway.py +++ /dev/null @@ -1,54 +0,0 @@ -"""AWS API Gateway collector β€” REST APIs with auth, logging, throttle metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class APIGatewayCollector(BaseCollector): - resource_type = "aws.apigateway" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('apigateway', region_name=self.region) - apis = client.get_rest_apis().get('items', []) - resources = [] - for api in apis: - api_id = api['id'] - name = api.get('name', api_id) - endpoint_type = ','.join(api.get('endpointConfiguration', {}).get('types', ['REGIONAL'])) - - # Check stages for throttle + logging - throttle_configured = False - logging_enabled = False - try: - stages = client.get_stages(restApiId=api_id).get('item', []) - for stage in stages: - settings = stage.get('methodSettings', {}).get('*/*', {}) - if settings.get('throttlingRateLimit', 0) > 0: - throttle_configured = True - if stage.get('accessLogSettings'): - logging_enabled = True - except: - pass - - # Tags - tags = api.get('tags', {}) - - resources.append(self._make_resource( - resource_id=name, - tags=tags, - properties={ - 'api_id': api_id, - 'endpoint_type': endpoint_type, - 'description': api.get('description', ''), - }, - metrics={ - 'throttle_configured': 1.0 if throttle_configured else 0.0, - 'logging_enabled': 1.0 if logging_enabled else 0.0, - 'endpoint_type_edge': 1.0 if 'EDGE' in endpoint_type else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ API Gateway scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/base.py b/build/lib/sofe/collectors/aws/base.py deleted file mode 100644 index 26b13f1..0000000 --- a/build/lib/sofe/collectors/aws/base.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Base collector class β€” all AWS collectors inherit from this.""" - -from __future__ import annotations -from abc import ABC, abstractmethod -import boto3 -from ...models import Resource - - -class BaseCollector(ABC): - """Abstract base for all AWS resource collectors.""" - - resource_type: str = "" # e.g., "aws.ec2" - - def __init__(self, session: boto3.Session, region: str, account_id: str): - self.session = session - self.region = region - self.account_id = account_id - - @abstractmethod - def collect(self) -> list[Resource]: - """Collect resources from AWS. Must be implemented by subclasses.""" - ... - - def _make_resource(self, resource_id: str, tags: dict = None, properties: dict = None, metrics: dict = None) -> Resource: - """Helper to create a Resource with common fields pre-filled.""" - return Resource( - resource_id=resource_id, - resource_type=self.resource_type, - region=self.region, - account_id=self.account_id, - tags=tags or {}, - properties=properties or {}, - metrics=metrics or {}, - ) diff --git a/build/lib/sofe/collectors/aws/cloudfront.py b/build/lib/sofe/collectors/aws/cloudfront.py deleted file mode 100644 index 938e8d7..0000000 --- a/build/lib/sofe/collectors/aws/cloudfront.py +++ /dev/null @@ -1,50 +0,0 @@ -"""AWS CloudFront collector β€” distributions with HTTPS, WAF, compression metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class CloudFrontCollector(BaseCollector): - resource_type = "aws.cloudfront" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('cloudfront', region_name='us-east-1') # CloudFront is global - distributions = client.list_distributions().get('DistributionList', {}).get('Items', []) - resources = [] - for dist in distributions: - dist_id = dist['Id'] - default_behavior = dist.get('DefaultCacheBehavior', {}) - viewer_policy = default_behavior.get('ViewerProtocolPolicy', '') - compress = default_behavior.get('Compress', False) - waf_id = dist.get('WebACLId', '') - price_class = dist.get('PriceClass', 'PriceClass_All') - - # Tags - tags = {} - try: - tag_resp = client.list_tags_for_resource(Resource=dist['ARN']) - tags = {t['Key']: t['Value'] for t in tag_resp.get('Tags', {}).get('Items', [])} - except: - pass - - resources.append(self._make_resource( - resource_id=dist_id, - tags=tags, - properties={ - 'domain': dist.get('DomainName'), - 'status': dist.get('Status'), - 'price_class': price_class, - 'viewer_protocol_policy': viewer_policy, - }, - metrics={ - 'compression_enabled': 1.0 if compress else 0.0, - 'https_only': 1.0 if viewer_policy in ('redirect-to-https', 'https-only') else 0.0, - 'waf_enabled': 1.0 if waf_id else 0.0, - 'price_class_all': 1.0 if price_class == 'PriceClass_All' else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ CloudFront scan failed: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/dynamodb.py b/build/lib/sofe/collectors/aws/dynamodb.py deleted file mode 100644 index c8fa8f7..0000000 --- a/build/lib/sofe/collectors/aws/dynamodb.py +++ /dev/null @@ -1,47 +0,0 @@ -"""AWS DynamoDB collector β€” tables with billing mode, size, item count.""" - -from .base import BaseCollector -from ...models import Resource - - -class DynamoDBCollector(BaseCollector): - resource_type = "aws.dynamodb" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('dynamodb', region_name=self.region) - tables = client.list_tables().get('TableNames', []) - resources = [] - for name in tables: - desc = client.describe_table(TableName=name).get('Table', {}) - billing = desc.get('BillingModeSummary', {}).get('BillingMode', 'PROVISIONED') - item_count = desc.get('ItemCount', 0) - table_size = desc.get('TableSizeBytes', 0) - - # Tags - tags = {} - try: - tag_resp = client.list_tags_of_resource(ResourceArn=desc['TableArn']) - tags = {t['Key']: t['Value'] for t in tag_resp.get('Tags', [])} - except: - pass - - resources.append(self._make_resource( - resource_id=name, - tags=tags, - properties={ - 'billing_mode': billing, - 'item_count': item_count, - 'table_size_bytes': table_size, - 'status': desc.get('TableStatus'), - }, - metrics={ - 'billing_mode_provisioned': 1.0 if billing == 'PROVISIONED' else 0.0, - 'item_count': float(item_count), - 'table_size_mb': round(table_size / 1_048_576, 2), - }, - )) - return resources - except Exception as e: - print(f" ⚠️ DynamoDB scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/ebs.py b/build/lib/sofe/collectors/aws/ebs.py deleted file mode 100644 index 983579d..0000000 --- a/build/lib/sofe/collectors/aws/ebs.py +++ /dev/null @@ -1,40 +0,0 @@ -"""AWS EBS collector β€” volumes with type, size, attachment metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class EBSCollector(BaseCollector): - resource_type = "aws.ebs" - - def collect(self) -> list[Resource]: - try: - ec2 = self.session.client('ec2', region_name=self.region) - volumes = ec2.describe_volumes().get('Volumes', []) - resources = [] - for vol in volumes: - tags = {t['Key']: t['Value'] for t in vol.get('Tags', [])} - attached = len(vol.get('Attachments', [])) > 0 - volume_type = vol.get('VolumeType', 'gp2') - - resources.append(self._make_resource( - resource_id=vol['VolumeId'], - tags=tags, - properties={ - 'volume_type': volume_type, - 'size_gb': vol.get('Size', 0), - 'iops': vol.get('Iops'), - 'state': vol.get('State'), - 'encrypted': vol.get('Encrypted', False), - }, - metrics={ - 'attached': 1.0 if attached else 0.0, - 'size_gb': float(vol.get('Size', 0)), - 'volume_type_gp2': 1.0 if volume_type == 'gp2' else 0.0, - 'encrypted': 1.0 if vol.get('Encrypted', False) else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ EBS scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/ec2.py b/build/lib/sofe/collectors/aws/ec2.py deleted file mode 100644 index d80c882..0000000 --- a/build/lib/sofe/collectors/aws/ec2.py +++ /dev/null @@ -1,47 +0,0 @@ -"""AWS EC2 collector β€” running instances with extended metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class EC2Collector(BaseCollector): - resource_type = "aws.ec2" - - def collect(self) -> list[Resource]: - try: - ec2 = self.session.client('ec2', region_name=self.region) - resp = ec2.describe_instances(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}]) - resources = [] - for res in resp.get('Reservations', []): - for inst in res.get('Instances', []): - tags = {t['Key']: t['Value'] for t in inst.get('Tags', [])} - instance_type = inst.get('InstanceType', '') - # Determine instance generation (t2=old, t3/m5=current, t4/m6/m7=latest) - family = instance_type.split('.')[0] if instance_type else '' - gen_num = int(''.join(c for c in family if c.isdigit()) or '0') - generation = 'latest' if gen_num >= 6 else ('current' if gen_num >= 3 else 'old') - - resources.append(self._make_resource( - resource_id=inst['InstanceId'], - tags=tags, - properties={ - 'instance_type': instance_type, - 'launch_time': str(inst.get('LaunchTime', '')), - 'state': inst.get('State', {}).get('Name'), - 'purchase_option': 'spot' if inst.get('InstanceLifecycle') == 'spot' else 'on-demand', - 'instance_generation': generation, - 'ebs_optimized': inst.get('EbsOptimized', False), - 'public_ip': inst.get('PublicIpAddress'), - 'platform': inst.get('Platform', 'linux'), - }, - metrics={ - 'purchase_option_spot': 1.0 if inst.get('InstanceLifecycle') == 'spot' else 0.0, - 'instance_generation_old': 1.0 if generation == 'old' else 0.0, - 'ebs_optimized': 1.0 if inst.get('EbsOptimized', False) else 0.0, - 'public_ip_attached': 1.0 if inst.get('PublicIpAddress') else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ EC2 scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/ecs.py b/build/lib/sofe/collectors/aws/ecs.py deleted file mode 100644 index 43ead18..0000000 --- a/build/lib/sofe/collectors/aws/ecs.py +++ /dev/null @@ -1,44 +0,0 @@ -"""AWS ECS collector β€” services with running/desired count, launch type.""" - -from .base import BaseCollector -from ...models import Resource - - -class ECSCollector(BaseCollector): - resource_type = "aws.ecs" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('ecs', region_name=self.region) - clusters = client.list_clusters().get('clusterArns', []) - resources = [] - for cluster_arn in clusters: - services = client.list_services(cluster=cluster_arn).get('serviceArns', []) - if not services: - continue - described = client.describe_services(cluster=cluster_arn, services=services[:10]).get('services', []) - for svc in described: - running = svc.get('runningCount', 0) - desired = svc.get('desiredCount', 0) - launch_type = svc.get('launchType', 'EC2') - - resources.append(self._make_resource( - resource_id=svc['serviceName'], - tags={t['key']: t['value'] for t in svc.get('tags', [])}, - properties={ - 'cluster': cluster_arn.split('/')[-1], - 'launch_type': launch_type, - 'running_count': running, - 'desired_count': desired, - 'status': svc.get('status'), - }, - metrics={ - 'running_count': float(running), - 'desired_count': float(desired), - 'launch_type_fargate': 1.0 if launch_type == 'FARGATE' else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ ECS scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/eks.py b/build/lib/sofe/collectors/aws/eks.py deleted file mode 100644 index 92f8299..0000000 --- a/build/lib/sofe/collectors/aws/eks.py +++ /dev/null @@ -1,40 +0,0 @@ -"""AWS EKS collector β€” Kubernetes clusters with version, access metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class EKSCollector(BaseCollector): - resource_type = "aws.eks" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('eks', region_name=self.region) - clusters = client.list_clusters().get('clusters', []) - resources = [] - for name in clusters: - detail = client.describe_cluster(name=name).get('cluster', {}) - version = detail.get('version', '0') - access_config = detail.get('resourcesVpcConfig', {}) - endpoint_public = access_config.get('endpointPublicAccess', True) - logging_enabled = bool(detail.get('logging', {}).get('clusterLogging', [{}])[0].get('enabled', False)) if detail.get('logging', {}).get('clusterLogging') else False - - resources.append(self._make_resource( - resource_id=name, - tags=detail.get('tags', {}), - properties={ - 'version': version, - 'status': detail.get('status'), - 'platform_version': detail.get('platformVersion'), - 'endpoint_public': endpoint_public, - }, - metrics={ - 'endpoint_public_access': 1.0 if endpoint_public else 0.0, - 'logging_enabled': 1.0 if logging_enabled else 0.0, - 'version_outdated': 1.0 if float(version or '0') < 1.28 else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ EKS scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/elasticache.py b/build/lib/sofe/collectors/aws/elasticache.py deleted file mode 100644 index a3f5c11..0000000 --- a/build/lib/sofe/collectors/aws/elasticache.py +++ /dev/null @@ -1,41 +0,0 @@ -"""AWS ElastiCache collector β€” clusters with engine version, encryption metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class ElastiCacheCollector(BaseCollector): - resource_type = "aws.elasticache" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('elasticache', region_name=self.region) - clusters = client.describe_cache_clusters(ShowCacheNodeInfo=True).get('CacheClusters', []) - resources = [] - for cluster in clusters: - cluster_id = cluster['CacheClusterId'] - engine = cluster.get('Engine', '') - engine_version = cluster.get('EngineVersion', '') - num_nodes = cluster.get('NumCacheNodes', 0) - encrypted = cluster.get('AtRestEncryptionEnabled', False) - transit_encrypted = cluster.get('TransitEncryptionEnabled', False) - - resources.append(self._make_resource( - resource_id=cluster_id, - properties={ - 'engine': engine, - 'engine_version': engine_version, - 'node_type': cluster.get('CacheNodeType'), - 'num_nodes': num_nodes, - 'status': cluster.get('CacheClusterStatus'), - }, - metrics={ - 'at_rest_encryption': 1.0 if encrypted else 0.0, - 'transit_encryption': 1.0 if transit_encrypted else 0.0, - 'num_nodes': float(num_nodes), - }, - )) - return resources - except Exception as e: - print(f" ⚠️ ElastiCache scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/elb.py b/build/lib/sofe/collectors/aws/elb.py deleted file mode 100644 index 08fd2e9..0000000 --- a/build/lib/sofe/collectors/aws/elb.py +++ /dev/null @@ -1,37 +0,0 @@ -"""AWS ELB/ALB collector β€” load balancers with target, WAF, scheme metrics.""" -from .base import BaseCollector -from ...models import Resource - -class ELBCollector(BaseCollector): - resource_type = "aws.elb" - def collect(self) -> list[Resource]: - try: - client = self.session.client('elbv2', region_name=self.region) - lbs = client.describe_load_balancers().get('LoadBalancers', []) - resources = [] - for lb in lbs: - arn = lb['LoadBalancerArn'] - tgs = client.describe_target_groups(LoadBalancerArn=arn).get('TargetGroups', []) - has_targets = any(tg.get('TargetType') for tg in tgs) - scheme = lb.get('Scheme', 'internal') - # Check WAF - waf_enabled = False - try: - waf = self.session.client('wafv2', region_name=self.region) - waf_resp = waf.get_web_acl_for_resource(ResourceArn=arn) - waf_enabled = bool(waf_resp.get('WebACL')) - except: - pass - resources.append(self._make_resource( - resource_id=lb['LoadBalancerName'], - properties={'type': lb.get('Type'), 'scheme': scheme, 'state': lb.get('State', {}).get('Code')}, - metrics={ - 'has_targets': 1.0 if has_targets else 0.0, - 'waf_enabled': 1.0 if waf_enabled else 0.0, - 'internet_facing': 1.0 if scheme == 'internet-facing' else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ ELB scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/lambda_.py b/build/lib/sofe/collectors/aws/lambda_.py deleted file mode 100644 index 74952bb..0000000 --- a/build/lib/sofe/collectors/aws/lambda_.py +++ /dev/null @@ -1,53 +0,0 @@ -"""AWS Lambda collector β€” functions with runtime, memory, timeout metrics.""" - -from .base import BaseCollector -from ...models import Resource - -# Deprecated runtimes as of 2026 -DEPRECATED_RUNTIMES = {'python3.7', 'python3.8', 'nodejs14.x', 'nodejs16.x', 'dotnetcore3.1', 'ruby2.7', 'java8', 'go1.x'} - - -class LambdaCollector(BaseCollector): - resource_type = "aws.lambda" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('lambda', region_name=self.region) - functions = client.list_functions().get('Functions', []) - resources = [] - for fn in functions: - runtime = fn.get('Runtime', '') - memory = fn.get('MemorySize', 128) - timeout = fn.get('Timeout', 3) - code_size = fn.get('CodeSize', 0) - - # Tags - tags = {} - try: - tag_resp = client.list_tags(Resource=fn['FunctionArn']) - tags = tag_resp.get('Tags', {}) - except: - pass - - resources.append(self._make_resource( - resource_id=fn['FunctionName'], - tags=tags, - properties={ - 'runtime': runtime, - 'memory': memory, - 'timeout': timeout, - 'code_size': code_size, - 'handler': fn.get('Handler', ''), - 'last_modified': fn.get('LastModified', ''), - }, - metrics={ - 'memory_size_mb': float(memory), - 'timeout_seconds': float(timeout), - 'runtime_deprecated': 1.0 if runtime in DEPRECATED_RUNTIMES else 0.0, - 'code_size_mb': round(code_size / 1_048_576, 2), - }, - )) - return resources - except Exception as e: - print(f" ⚠️ Lambda scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/natgateway.py b/build/lib/sofe/collectors/aws/natgateway.py deleted file mode 100644 index 155c600..0000000 --- a/build/lib/sofe/collectors/aws/natgateway.py +++ /dev/null @@ -1,37 +0,0 @@ -"""AWS NAT Gateway collector β€” gateways with connectivity, state metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class NATGatewayCollector(BaseCollector): - resource_type = "aws.natgateway" - - def collect(self) -> list[Resource]: - try: - ec2 = self.session.client('ec2', region_name=self.region) - gateways = ec2.describe_nat_gateways( - Filter=[{'Name': 'state', 'Values': ['available']}] - ).get('NatGateways', []) - resources = [] - for gw in gateways: - tags = {t['Key']: t['Value'] for t in gw.get('Tags', [])} - connectivity = gw.get('ConnectivityType', 'public') - - resources.append(self._make_resource( - resource_id=gw['NatGatewayId'], - tags=tags, - properties={ - 'state': gw.get('State'), - 'connectivity_type': connectivity, - 'subnet_id': gw.get('SubnetId'), - 'vpc_id': gw.get('VpcId'), - }, - metrics={ - 'connectivity_public': 1.0 if connectivity == 'public' else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ NAT Gateway scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/rds.py b/build/lib/sofe/collectors/aws/rds.py deleted file mode 100644 index 50cedef..0000000 --- a/build/lib/sofe/collectors/aws/rds.py +++ /dev/null @@ -1,48 +0,0 @@ -"""AWS RDS collector β€” DB instances with multi-AZ, encryption, backup metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class RDSCollector(BaseCollector): - resource_type = "aws.rds" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('rds', region_name=self.region) - instances = client.describe_db_instances().get('DBInstances', []) - resources = [] - for db in instances: - identifier = db['DBInstanceIdentifier'] - - # Tags - tags = {} - try: - tag_resp = client.list_tags_for_resource(ResourceName=db['DBInstanceArn']) - tags = {t['Key']: t['Value'] for t in tag_resp.get('TagList', [])} - except: - pass - - resources.append(self._make_resource( - resource_id=identifier, - tags=tags, - properties={ - 'instance_class': db.get('DBInstanceClass'), - 'engine': db.get('Engine'), - 'engine_version': db.get('EngineVersion'), - 'multi_az': db.get('MultiAZ', False), - 'storage_type': db.get('StorageType'), - 'allocated_storage_gb': db.get('AllocatedStorage'), - 'status': db.get('DBInstanceStatus'), - }, - metrics={ - 'multi_az': 1.0 if db.get('MultiAZ', False) else 0.0, - 'storage_encrypted': 1.0 if db.get('StorageEncrypted', False) else 0.0, - 'backup_retention_days': float(db.get('BackupRetentionPeriod', 0)), - 'publicly_accessible': 1.0 if db.get('PubliclyAccessible', False) else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ RDS scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/redshift.py b/build/lib/sofe/collectors/aws/redshift.py deleted file mode 100644 index 58907ae..0000000 --- a/build/lib/sofe/collectors/aws/redshift.py +++ /dev/null @@ -1,41 +0,0 @@ -"""AWS Redshift collector β€” clusters with encryption, node count metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class RedshiftCollector(BaseCollector): - resource_type = "aws.redshift" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('redshift', region_name=self.region) - clusters = client.describe_clusters().get('Clusters', []) - resources = [] - for cluster in clusters: - cluster_id = cluster['ClusterIdentifier'] - num_nodes = cluster.get('NumberOfNodes', 0) - encrypted = cluster.get('Encrypted', False) - publicly_accessible = cluster.get('PubliclyAccessible', False) - - tags = {t['Key']: t['Value'] for t in cluster.get('Tags', [])} - - resources.append(self._make_resource( - resource_id=cluster_id, - tags=tags, - properties={ - 'node_type': cluster.get('NodeType'), - 'num_nodes': num_nodes, - 'status': cluster.get('ClusterStatus'), - 'db_name': cluster.get('DBName'), - }, - metrics={ - 'encrypted': 1.0 if encrypted else 0.0, - 'publicly_accessible': 1.0 if publicly_accessible else 0.0, - 'num_nodes': float(num_nodes), - }, - )) - return resources - except Exception as e: - print(f" ⚠️ Redshift scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/route53.py b/build/lib/sofe/collectors/aws/route53.py deleted file mode 100644 index ea0cefa..0000000 --- a/build/lib/sofe/collectors/aws/route53.py +++ /dev/null @@ -1,36 +0,0 @@ -"""AWS Route53 collector β€” hosted zones with record count metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class Route53Collector(BaseCollector): - resource_type = "aws.route53" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('route53') # Route53 is global - zones = client.list_hosted_zones().get('HostedZones', []) - resources = [] - for zone in zones: - zone_id = zone['Id'].split('/')[-1] - name = zone['Name'].rstrip('.') - record_count = zone.get('ResourceRecordSetCount', 0) - private = zone.get('Config', {}).get('PrivateZone', False) - - resources.append(self._make_resource( - resource_id=name, - properties={ - 'zone_id': zone_id, - 'record_count': record_count, - 'private_zone': private, - }, - metrics={ - 'record_count': float(record_count), - 'private_zone': 1.0 if private else 0.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ Route53 scan failed: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/s3.py b/build/lib/sofe/collectors/aws/s3.py deleted file mode 100644 index 50c3fd1..0000000 --- a/build/lib/sofe/collectors/aws/s3.py +++ /dev/null @@ -1,79 +0,0 @@ -"""AWS S3 collector β€” buckets with encryption, versioning, lifecycle metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class S3Collector(BaseCollector): - resource_type = "aws.s3" - - def collect(self) -> list[Resource]: - try: - s3 = self.session.client('s3', region_name=self.region) - buckets = s3.list_buckets().get('Buckets', []) - resources = [] - for b in buckets: - name = b['BucketName'] - metrics = {} - properties = {'creation_date': str(b.get('CreationDate', ''))} - - # Encryption - try: - s3.get_bucket_encryption(Bucket=name) - metrics['encryption_enabled'] = 1.0 - except: - metrics['encryption_enabled'] = 0.0 - - # Lifecycle - try: - rules = s3.get_bucket_lifecycle_configuration(Bucket=name).get('Rules', []) - metrics['has_lifecycle_rules'] = 1.0 if rules else 0.0 - except: - metrics['has_lifecycle_rules'] = 0.0 - - # Versioning - try: - ver = s3.get_bucket_versioning(Bucket=name) - metrics['versioning_enabled'] = 1.0 if ver.get('Status') == 'Enabled' else 0.0 - except: - metrics['versioning_enabled'] = 0.0 - - # Public access block - try: - pab = s3.get_public_access_block(Bucket=name) - config = pab.get('PublicAccessBlockConfiguration', {}) - all_blocked = all([ - config.get('BlockPublicAcls', False), - config.get('IgnorePublicAcls', False), - config.get('BlockPublicPolicy', False), - config.get('RestrictPublicBuckets', False), - ]) - metrics['public_access_blocked'] = 1.0 if all_blocked else 0.0 - except: - metrics['public_access_blocked'] = 0.0 - - # Logging - try: - logging = s3.get_bucket_logging(Bucket=name) - metrics['logging_enabled'] = 1.0 if logging.get('LoggingEnabled') else 0.0 - except: - metrics['logging_enabled'] = 0.0 - - # Tags - tags = {} - try: - tag_resp = s3.get_bucket_tagging(Bucket=name) - tags = {t['Key']: t['Value'] for t in tag_resp.get('TagSet', [])} - except: - pass - - resources.append(self._make_resource( - resource_id=name, - tags=tags, - properties=properties, - metrics=metrics, - )) - return resources - except Exception as e: - print(f" ⚠️ S3 scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/sagemaker.py b/build/lib/sofe/collectors/aws/sagemaker.py deleted file mode 100644 index 219641c..0000000 --- a/build/lib/sofe/collectors/aws/sagemaker.py +++ /dev/null @@ -1,50 +0,0 @@ -"""AWS SageMaker collector β€” endpoints with instance count, variant metrics.""" - -from .base import BaseCollector -from ...models import Resource - - -class SageMakerCollector(BaseCollector): - resource_type = "aws.sagemaker" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('sagemaker', region_name=self.region) - endpoints = client.list_endpoints(StatusEquals='InService').get('Endpoints', []) - resources = [] - for ep in endpoints: - name = ep['EndpointName'] - # Get endpoint config for instance details - total_instances = 0 - try: - detail = client.describe_endpoint(EndpointName=name) - variants = detail.get('ProductionVariants', []) - total_instances = sum(v.get('CurrentInstanceCount', 0) for v in variants) - except: - pass - - # Tags - tags = {} - try: - tag_resp = client.list_tags(ResourceArn=ep['EndpointArn']) - tags = {t['Key']: t['Value'] for t in tag_resp.get('Tags', [])} - except: - pass - - resources.append(self._make_resource( - resource_id=name, - tags=tags, - properties={ - 'status': ep.get('EndpointStatus'), - 'creation_time': str(ep.get('CreationTime', '')), - 'instance_count': total_instances, - }, - metrics={ - 'instance_count': float(total_instances), - 'endpoint_active': 1.0, - }, - )) - return resources - except Exception as e: - print(f" ⚠️ SageMaker scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/collectors/aws/secretsmanager.py b/build/lib/sofe/collectors/aws/secretsmanager.py deleted file mode 100644 index d0db3da..0000000 --- a/build/lib/sofe/collectors/aws/secretsmanager.py +++ /dev/null @@ -1,49 +0,0 @@ -"""AWS Secrets Manager collector β€” secrets with rotation and access metrics.""" - -from datetime import datetime, timezone -from .base import BaseCollector -from ...models import Resource - - -class SecretsManagerCollector(BaseCollector): - resource_type = "aws.secretsmanager" - - def collect(self) -> list[Resource]: - try: - client = self.session.client('secretsmanager', region_name=self.region) - secrets = client.list_secrets().get('SecretList', []) - resources = [] - now = datetime.now(timezone.utc) - for s in secrets: - rotation_enabled = s.get('RotationEnabled', False) - last_rotated = s.get('LastRotatedDate') - last_accessed = s.get('LastAccessedDate') - - days_since_rotation = 999 - if last_rotated: - days_since_rotation = (now - last_rotated).days - - days_since_access = 999 - if last_accessed: - days_since_access = (now - last_accessed).days - - tags = {t['Key']: t['Value'] for t in s.get('Tags', [])} - - resources.append(self._make_resource( - resource_id=s.get('Name', s.get('ARN', '')), - tags=tags, - properties={ - 'rotation_enabled': rotation_enabled, - 'last_rotated': str(last_rotated) if last_rotated else None, - 'last_accessed': str(last_accessed) if last_accessed else None, - }, - metrics={ - 'rotation_enabled': 1.0 if rotation_enabled else 0.0, - 'days_since_last_rotation': float(days_since_rotation), - 'days_since_last_access': float(days_since_access), - }, - )) - return resources - except Exception as e: - print(f" ⚠️ SecretsManager scan failed in {self.region}: {e}") - return [] diff --git a/build/lib/sofe/engine/__init__.py b/build/lib/sofe/engine/__init__.py deleted file mode 100644 index c4e10fb..0000000 --- a/build/lib/sofe/engine/__init__.py +++ /dev/null @@ -1,225 +0,0 @@ -from __future__ import annotations -"""Evaluation engine β€” applies policies against resources.""" - -import uuid -from datetime import datetime -from ..models import Policy, Resource, Finding, Operator, Severity -from .architecture import ArchitectureContext - - -def evaluate(policies: list[Policy], resources: list[Resource], context: ArchitectureContext = None) -> list[Finding]: - """Evaluate all policies against all resources. Returns findings (violations). - - If context is provided, architecture-aware policies can access related resources. - """ - findings: list[Finding] = [] - - # Build context if not provided (auto-infer relationships) - if context is None: - context = ArchitectureContext.from_resources(resources) - - for policy in policies: - matching = _filter_by_scope(resources, policy) - for resource in matching: - violation = _check_rule(resource, policy, context) - if violation: - findings.append(violation) - - # Add architecture insights as findings - _add_architecture_findings(context, findings) - - return findings - - -def evaluate_architecture(policies: list[Policy], resources: list[Resource]) -> dict: - """Evaluate with full architecture analysis. Returns findings + insights.""" - context = ArchitectureContext.from_resources(resources) - findings = evaluate(policies, resources, context) - - # Compute architecture insights - insights = { - "blast_radius": {}, - "single_points_of_failure": context.single_points_of_failure(threshold=3), - "team_costs": {}, - "relationships_count": len(context.relationships), - } - - # Blast radius for resources with findings - resource_ids_with_findings = set(f.resource_id for f in findings) - for rid in resource_ids_with_findings: - affected = context.blast_radius(rid) - if affected: - insights["blast_radius"][rid] = { - "affected_count": len(affected), - "affected_resources": affected[:10], # Cap at 10 - "cost_impact": context.cost_chain(rid), - } - - # Team costs - owners = set() - for r in resources: - owner = r.tags.get("owner", r.tags.get("Owner", "")) - if owner: - owners.add(owner) - for owner in owners: - insights["team_costs"][owner] = context.team_cost(owner) - - return {"findings": findings, "insights": insights, "context": context} - - -def _filter_by_scope(resources: list[Resource], policy: Policy) -> list[Resource]: - """Filter resources by policy scope.""" - scope = policy.spec.scope - result = [] - - for r in resources: - # Check resource type - if not _matches_list(r.resource_type, scope.resource_types): - continue - # Check region - if not _matches_list(r.region, scope.regions): - continue - # Check account - if not _matches_list(r.account_id, scope.accounts): - continue - # Check environment tag - if scope.environments != ["*"]: - env = r.tags.get("Environment", r.tags.get("env", "")) - if not _matches_list(env, scope.environments): - continue - # Check exclude - if scope.exclude: - exclude_tags = scope.exclude.get("tags", {}) - if any(r.tags.get(k) == v for k, v in exclude_tags.items()): - continue - if r.resource_id in scope.exclude.get("resource_ids", []): - continue - - result.append(r) - - return result - - -def _check_rule(resource: Resource, policy: Policy, context: ArchitectureContext = None) -> Finding | None: - """Check if a resource violates the policy rule. Uses context for cross-resource checks.""" - rule = policy.spec.rule - metric_value = resource.metrics.get(rule.metric) - - if metric_value is None: - return None # No data for this metric β€” skip - - # Apply operator - violated = _apply_operator(metric_value, rule.operator, rule.threshold) - if not violated: - return None - - # Check additional conditions (AND logic) - for cond in rule.additional_conditions: - cond_value = resource.metrics.get(cond.field) or resource.properties.get(cond.field) - if cond_value is None: - return None # Missing data β€” skip - if not _apply_operator(float(cond_value), cond.operator, float(cond.value)): - return None # Condition not met - - # Check cross-resource context conditions (if policy has context spec) - if hasattr(policy.spec, 'context') and policy.spec.context and context: - ctx_spec = policy.spec.context - # Check requires_relationship - if ctx_spec.get("requires_relationship"): - rel_type = ctx_spec["requires_relationship"] - related = context.get_related(resource.resource_id, rel_type=rel_type) - if not related: - return None # Relationship doesn't exist β€” skip - - # Check related_check (condition on related resource) - if ctx_spec.get("related_check"): - check = ctx_spec["related_check"] - check_met = False - for rel_resource in related: - if check.get("resource_type") and rel_resource.resource_type != check["resource_type"]: - continue - rel_metric = rel_resource.metrics.get(check.get("metric", "")) - if rel_metric is not None: - if _apply_operator(rel_metric, Operator(check["operator"]), float(check["value"])): - check_met = True - break - if not check_met: - return None # Related check not met - - # Build finding - recommendation = None - estimated_savings = None - for action in policy.spec.actions: - if action.type == "recommend" and action.suggestion: - recommendation = action.suggestion - if action.estimated_savings == "calc": - estimated_savings = resource.metrics.get("monthly_cost") - - # Enrich with blast radius if context available - blast_info = "" - if context: - affected = context.blast_radius(resource.resource_id) - if affected: - blast_info = f" [blast radius: {len(affected)} resources]" - - return Finding( - id=str(uuid.uuid4())[:8], - policy_name=policy.metadata.name, - severity=policy.spec.severity, - resource_id=resource.resource_id, - resource_type=resource.resource_type, - region=resource.region, - account_id=resource.account_id, - message=f"{rule.metric} = {metric_value} (threshold: {rule.operator.value}{rule.threshold}){blast_info}", - metric_name=rule.metric, - metric_value=metric_value, - threshold=rule.threshold, - estimated_savings=estimated_savings, - recommendation=recommendation, - remediation_eligible=policy.spec.remediation.auto_eligible if policy.spec.remediation else False, - timestamp=datetime.utcnow(), - ) - - -def _add_architecture_findings(context: ArchitectureContext, findings: list[Finding]): - """Add findings for architectural issues (single points of failure, etc.).""" - # Single points of failure - spofs = context.single_points_of_failure(threshold=3) - for rid in spofs: - resource = context.resources.get(rid) - if not resource: - continue - fan = context.fan_in(rid) - findings.append(Finding( - id=str(uuid.uuid4())[:8], - policy_name="architecture-single-point-of-failure", - severity=Severity.medium, - resource_id=rid, - resource_type=resource.resource_type, - region=resource.region, - account_id=resource.account_id, - message=f"Single point of failure: {fan} resources depend on this (fan-in >= 3)", - metric_name="fan_in", - metric_value=float(fan), - threshold=3.0, - estimated_savings=None, - recommendation="Consider redundancy or multi-AZ for this critical resource", - remediation_eligible=False, - timestamp=datetime.utcnow(), - )) - - -def _apply_operator(value: float, operator: Operator, threshold: float) -> bool: - if operator == Operator.lt: return value < threshold - elif operator == Operator.gt: return value > threshold - elif operator == Operator.lte: return value <= threshold - elif operator == Operator.gte: return value >= threshold - elif operator == Operator.eq: return value == threshold - elif operator == Operator.ne: return value != threshold - return False - - -def _matches_list(value: str, allowed: list[str]) -> bool: - if "*" in allowed: - return True - return value in allowed diff --git a/build/lib/sofe/loader/__init__.py b/build/lib/sofe/loader/__init__.py deleted file mode 100644 index a959fe6..0000000 --- a/build/lib/sofe/loader/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Load and validate YAML policy files.""" - -import os -import yaml -from pathlib import Path -from typing import Optional -from ..models import Policy - - -def load_policies(path: str) -> list[Policy]: - """Load all .yaml policy files from a directory or single file.""" - policies = [] - p = Path(path) - - if p.is_file(): - pol = _load_single(p) - if pol: - policies.append(pol) - elif p.is_dir(): - for f in sorted(p.glob("*.yaml")): - pol = _load_single(f) - if pol: - policies.append(pol) - for f in sorted(p.glob("*.yml")): - pol = _load_single(f) - if pol: - policies.append(pol) - else: - raise FileNotFoundError(f"Policy path not found: {path}") - - return policies - - -def _load_single(path: Path) -> Optional[Policy]: - """Load and validate a single policy file. Returns None if invalid.""" - try: - with open(path) as f: - data = yaml.safe_load(f) - if not data: - return None - return Policy(**data) - except Exception: - return None # Skip invalid policies silently - - -def validate_policies(path: str) -> list[dict]: - """Validate policies without evaluating. Returns list of {file, valid, error}.""" - results = [] - p = Path(path) - files = [p] if p.is_file() else list(p.glob("*.yaml")) + list(p.glob("*.yml")) - - for f in files: - try: - _load_single(f) - results.append({"file": f.name, "valid": True, "error": None}) - except Exception as e: - results.append({"file": f.name, "valid": False, "error": str(e)}) - - return results diff --git a/build/lib/sofe/models/__init__.py b/build/lib/sofe/models/__init__.py deleted file mode 100644 index 967637e..0000000 --- a/build/lib/sofe/models/__init__.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Core models for SOFE β€” Policy, Resource, Finding.""" - -from pydantic import BaseModel -from typing import Optional, Union -from enum import Enum -from datetime import datetime - - -class Severity(str, Enum): - critical = "critical" - high = "high" - medium = "medium" - low = "low" - info = "info" - - -class Operator(str, Enum): - lt = "<" - gt = ">" - lte = "<=" - gte = ">=" - eq = "==" - ne = "!=" - - -class AdditionalCondition(BaseModel): - field: str - operator: Operator - value: Union[float, str] - - -class Scope(BaseModel): - environments: list[str] = ["*"] - resource_types: list[str] - regions: list[str] = ["*"] - accounts: list[str] = ["*"] - exclude: Optional[dict] = None - - -class Rule(BaseModel): - metric: str - period: str = "30d" - operator: Operator - threshold: float - additional_conditions: list[AdditionalCondition] = [] - - -class Action(BaseModel): - type: str # finding, notify, recommend, block - channel: Optional[str] = None - suggestion: Optional[str] = None - estimated_savings: Optional[str] = None - - -class Remediation(BaseModel): - auto_eligible: bool = False - action: Optional[str] = None - requires_approval_if: Optional[dict] = None - - -class PolicyMetadata(BaseModel): - name: str - description: str - author: Optional[str] = None - created: Optional[str] = None - tags: list[str] = [] - - -class PolicySpec(BaseModel): - scope: Scope - rule: Rule - severity: Severity - actions: list[Action] = [] - remediation: Optional[Remediation] = None - - -class Policy(BaseModel): - apiVersion: str = "sofe/v1" - kind: str = "Policy" - metadata: PolicyMetadata - spec: PolicySpec - - -class Resource(BaseModel): - resource_id: str - resource_type: str - region: str - account_id: str - tags: dict[str, str] = {} - properties: dict = {} - metrics: dict[str, float] = {} - - -class Finding(BaseModel): - id: str - policy_name: str - severity: Severity - resource_id: str - resource_type: str - region: str - account_id: str - message: str - metric_name: str - metric_value: float - threshold: float - estimated_savings: Optional[float] = None - recommendation: Optional[str] = None - remediation_eligible: bool = False - timestamp: datetime