dev.env.example

# JVM options passed to the Java process running the application.
JAVA_TOOL_OPTIONS = "-XX:ActiveProcessorCount=1 -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=2m -XX:MaxRAMPercentage=80.0 -XX:+ExitOnOutOfMemoryError -XX:MaxHeapFreeRatio=30 -XX:MinHeapFreeRatio=20 -XX:+UseCompactObjectHeaders -XX:+UnlockExperimentalVMOptions"

# Logical stream identifier used in logging, metrics, and stream context resolution.
STREAMCONTEXT__STREAM_ID = "dynamodb-test-stream"

# Stream implementation kind. For DynamoDB this is typically PullStream.
STREAMCONTEXT__STREAM_KIND = "PullStream"

# Source S3 credentials used by the source reader.
AWS_ACCESS_KEY_ID = "X"
AWS_SECRET_ACCESS_KEY = "X"
AWS_REGION = "eu-central-1"
AWS_ENDPOINT_URL_S3 = "https://example.s3.endpoint.com"

# Iceberg table storage / FileIO settings.
# These are used by Iceberg's S3FileIO when reading and writing table data files
# and metadata files in the S3-compatible object store.
ARCANE_FRAMEWORK__S3_CATALOG_ENDPOINT = "https://example.s3.endpoint.com"
ARCANE_FRAMEWORK__S3_CATALOG_ACCESS_KEY_ID = "X"
ARCANE_FRAMEWORK__S3_CATALOG_SECRET_ACCESS_KEY = "X"
ARCANE_FRAMEWORK__S3_CATALOG_REGION = "eu-central-1"

# Iceberg REST catalog authentication settings.
# These are used to authenticate to the Iceberg REST catalog service itself,
# for example Lakekeeper. They are not the S3 object-store credentials.
ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_URI = "https://example.auth.com/token"
ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_ID = "X"
ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_SECRET = "X"
ARCANE_FRAMEWORK__S3_CATALOG_AUTH_SCOPE = "openid profile X/.default"

# Trino/JDBC credentials used by the merge service when issuing SQL against
# the target query engine. These are separate from both S3 FileIO credentials
# and Iceberg REST catalog credentials.
ARCANE_FRAMEWORK__JDBC_MERGE_SERVICE_USER = "user"
ARCANE_FRAMEWORK__JDBC_MERGE_SERVICE_PASSWORD = "password"

# For up-to-date configuration refer to .helm/templates/crd-dynamodb-stream.yaml
STREAMCONTEXT__SPEC = '''
{
  "observability": {
    "metricTags": {}
  },
  "sink": {
    "icebergCatalog": {
      "catalogProperties": {},
      "catalogUri": "https://example.catalog.com/catalog",
      "maxCatalogInstanceLifetime": "3500 second",
      "namespace": "example_namespace",
      "warehouse": "example-warehouse"
    },
    "maintenanceSettings": {
      "targetAnalyzeSettings": {
        "batchThreshold": 60,
        "includedColumns": []
      },
      "targetOptimizeSettings": {
        "batchThreshold": 60,
        "fileSizeThreshold": "512MB"
      },
      "targetOrphanFilesExpirationSettings": {
        "batchThreshold": 60,
        "retentionThreshold": "6h"
      },
      "targetSnapshotExpirationSettings": {
        "batchThreshold": 60,
        "retentionThreshold": "6h"
      }
    },
    "mergeServiceClient": {
      "connectionUrl": "jdbc:trino://example.trino.endpoint.com",
      "credentialType": {
        "basic": {}
      },
      "extraConnectionParameters": {
        "SSL": "true",
        "clientTags": "tag1,tag2",
        "explicitPrepare": "false",
        "sessionProperties": "min_input_size_per_task:256MB;min_input_rows_per_task:100000"
      },
      "queryRetryBaseDuration": "1 second",
      "queryRetryMaxAttempts": 3,
      "queryRetryMode": {
        "always": {}
      },
      "queryRetryOnMessageContents": [
        "Encountered too many errors talking to a worker node",
        "Failed to load view",
        "No worker nodes available",
        "splitsInfo already set to",
        "Query killed"
      ],
      "queryRetryScaleFactor": 3.0
    },
    "targetTableFullName": "lakehouse_test.example_schema.example_table",
    "targetTableProperties": {
      "format": "PARQUET",
      "parquetBloomFilterColumns": [],
      "sortedBy": []
    }
  },
  "source": {
    "buffering": {
      "enabled": false,
      "strategy": {}
    },
    "configuration": {
      "avroSchemaString": "{ \"name\": \"GeneratedAvroSchemaTest\", \"namespace\": \"com.group.GeneratedAvroSchemaTest\", \"doc\": \"Example JSON schema\", \"type\": \"record\", \"fields\": [ { \"name\": \"id\", \"type\": [ \"null\", \"string\" ], \"default\": null }, { \"name\": \"value\", \"type\": [ \"null\", \"string\" ], \"default\": null } ] }",
      "jsonArrayPointers": {},
      "primaryKeys": [
        "id"
      ],
      "s3": {
        "endpoint": "https://example.s3.endpoint.com",
        "maxResultsPerPage": 1000,
        "region": "eu-central-1",
        "retryBaseDelay": "100 millisecond",
        "retryMaxAttempts": 5,
        "retryMaxDelay": "1 second",
        "usePathStyle": true
      },
      "sourcePath": "s3a://example-bucket/path/to/json",
      "tempStoragePath": "/tmp"
    },
    "fieldSelectionRule": {
      "essentialFields": [],
      "isServerSide": false,
      "rule": {
        "all": {}
      }
    }
  },
  "staging": {
    "icebergCatalog": {
      "catalogProperties": {},
      "catalogUri": "https://example.catalog.com/catalog",
      "maxCatalogInstanceLifetime": "3500 second",
      "namespace": "example_namespace",
      "warehouse": "staging"
    },
    "table": {
      "isUnifiedSchema": false,
      "maxRowsPerFile": 10000,
      "stagingCatalogName": "example_catalog",
      "stagingSchemaName": "example_schema",
      "stagingTablePrefix": "example_table"
    }
  },
  "streamMode": {
    "backfill": {
      "backfillBehavior": "Overwrite",
      "backfillStartDate": "2026-01-01T00:00:00Z"
    },
    "changeCapture": {
      "changeCaptureInterval": "60 second",
      "changeCaptureJitterSeed": 1,
      "changeCaptureJitterVariance": 0.15
    }
  },
  "throughput": {
    "advisedBurst": 1000,
    "advisedChunkSize": 10,
    "advisedRate": "1000 per 1 second",
    "shaperImpl": {
      "memoryBound": {
        "chunkCostMax": 10,
        "chunkCostScale": 1,
        "fallbackStringTypeSizeEstimate": 50,
        "objectTypeSizeEstimate": 4096,
        "tableRowCountWeight": 0.05,
        "tableSizeScaleFactor": 1,
        "tableSizeWeight": 0.05
      }
    }
  }
}
'''