Added cloudwatch datasource to grafana

Author: dementii-priadko

.gitignore

@@ -28,9 +28,10 @@ pids
 *.seed
 *.pid.lock
 
-# Generated config files (these are created by the sources-generator)
+# Generated config files (these are created by the sources-generator and datasource-generator)
 config/pgwatch-postgres/sources.yml
 config/pgwatch-prometheus/sources.yml
+config/grafana/provisioning/datasources/datasources.processed.yml
 
 # Volume data (if accidentally committed)
 data/

README.md

@@ -184,6 +184,11 @@ Get a complete monitoring setup with demo data in under 2 minutes.
 
 # Health check
 ./postgres_ai health
+
+# AWS CloudWatch integration (optional)
+./postgres_ai add-aws-credentials <access_key> <secret_key> [region]
+./postgres_ai show-aws-credentials
+./postgres_ai remove-aws-credentials
 ```
 
 ## 🌐 Access points
@@ -206,6 +211,32 @@ Technical URLs (for advanced users):
 ## 🔑 PostgresAI access token
 Get your access token at [PostgresAI](https://postgres.ai) for automated report uploads and advanced analysis.
 
+## ☁️ AWS CloudWatch integration (optional)
+
+If you're monitoring AWS RDS Postgres instances, you can enable CloudWatch datasource to correlate RDS metrics with postgres_ai monitoring data.
+
+**Enable CloudWatch datasource:**
+
+```bash
+./postgres_ai add-aws-credentials <YOUR_AWS_ACCESS_KEY> <YOUR_AWS_SECRET_KEY> us-east-1
+./postgres_ai restart
+```
+
+The CloudWatch datasource is disabled by default and will only be activated when AWS credentials are configured. Your credentials are stored securely in `.pgwatch-config` (which is git-ignored).
+
+**Manage AWS credentials:**
+
+```bash
+# View current configuration (credentials are masked)
+./postgres_ai show-aws-credentials
+
+# Remove AWS credentials (disables CloudWatch datasource)
+./postgres_ai remove-aws-credentials
+./postgres_ai restart
+```
+
+**Note:** AWS credentials are optional and only needed if you want to view AWS RDS CloudWatch metrics alongside postgres_ai monitoring data in Grafana.
+
 ## 🛣️ Roadmap
 
 - Host stats for on-premise and managed Postgres setups

config/grafana/provisioning/datasources/datasources.yml

@@ -26,6 +26,9 @@ datasources:
       queryTimeout: '5s'
       timeInterval: '5s'
       httpMethod: 'POST'
+  
+# CloudWatch datasource will be added here if AWS credentials are configured
+~CLOUDWATCH_DATASOURCE~
 
   - name: Infinity
     type: yesoreyeram-infinity-datasource

docker-compose.yml

@@ -18,6 +18,45 @@ services:
       echo 'Generated sources.yml files for both postgres and prometheus'
       "
 
+  # Datasource Generator - Generates datasources.yml with optional CloudWatch configuration
+  datasource-generator:
+    image: alpine:3.22.0
+    container_name: datasource-generator
+    working_dir: /app
+    volumes:
+      - ./.pgwatch-config:/app/.pgwatch-config:ro
+      - ./config/grafana/provisioning/datasources/datasources.yml:/app/datasources.template:ro
+      - ./config/grafana/provisioning/datasources:/app/output
+    command: >
+      sh -c "
+      echo 'Processing Grafana datasources configuration...' &&
+      if [ -f /app/.pgwatch-config ] && grep -q '^aws_access_key=' /app/.pgwatch-config && grep -q '^aws_secret_key=' /app/.pgwatch-config; then
+        AWS_ACCESS_KEY=$$(grep '^aws_access_key=' /app/.pgwatch-config | cut -d'=' -f2-) &&
+        AWS_SECRET_KEY=$$(grep '^aws_secret_key=' /app/.pgwatch-config | cut -d'=' -f2-) &&
+        AWS_REGION=$$(grep '^aws_region=' /app/.pgwatch-config | cut -d'=' -f2-) &&
+        AWS_REGION=$${AWS_REGION:-us-east-1} &&
+        echo 'AWS credentials found, enabling CloudWatch datasource' &&
+        grep -B 9999 '~CLOUDWATCH_DATASOURCE~' /app/datasources.template | grep -v '~CLOUDWATCH_DATASOURCE~' > /app/output/datasources.processed.yml &&
+        echo '  - name: CloudWatch-RDS' >> /app/output/datasources.processed.yml &&
+        echo '    type: cloudwatch' >> /app/output/datasources.processed.yml &&
+        echo '    access: proxy' >> /app/output/datasources.processed.yml &&
+        echo '    jsonData:' >> /app/output/datasources.processed.yml &&
+        echo '      authType: keys' >> /app/output/datasources.processed.yml &&
+        echo \"      defaultRegion: $$AWS_REGION\" >> /app/output/datasources.processed.yml &&
+        echo '      customMetricsNamespaces: AWS/RDS' >> /app/output/datasources.processed.yml &&
+        echo '    secureJsonData:' >> /app/output/datasources.processed.yml &&
+        echo \"      accessKey: $$AWS_ACCESS_KEY\" >> /app/output/datasources.processed.yml &&
+        echo \"      secretKey: $$AWS_SECRET_KEY\" >> /app/output/datasources.processed.yml &&
+        echo '    isDefault: false' >> /app/output/datasources.processed.yml &&
+        echo '    editable: true' >> /app/output/datasources.processed.yml &&
+        grep -A 9999 '~CLOUDWATCH_DATASOURCE~' /app/datasources.template | grep -v '~CLOUDWATCH_DATASOURCE~' >> /app/output/datasources.processed.yml
+      else
+        echo 'AWS credentials not configured, CloudWatch datasource disabled' &&
+        grep -v '~CLOUDWATCH_DATASOURCE~' /app/datasources.template > /app/output/datasources.processed.yml
+      fi &&
+      echo 'Datasources configuration generated successfully'
+      "
+
   # Target Database - The PostgreSQL database being monitored
   target-db:
     image: postgres:15
@@ -105,10 +144,12 @@ services:
       - "3000:3000"
     volumes:
       - grafana_data:/var/lib/grafana
-      - ./config/grafana/provisioning:/etc/grafana/provisioning
+      - ./config/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./config/grafana/provisioning/datasources/datasources.processed.yml:/etc/grafana/provisioning/datasources/datasources.yml:ro
       - ./config/grafana/dashboards:/var/lib/grafana/dashboards
       - ./config/grafana/provisioning/grafana.ini:/etc/grafana/grafana.ini
     depends_on:
+      - datasource-generator
       - sink-postgres
       - sink-prometheus
   flask-backend:

postgres_ai

@@ -78,6 +78,11 @@ show_help() {
     echo "  show-key                       Show current API key (masked)"
     echo "  remove-key                     Remove stored API key"
     echo ""
+    echo "AWS CLOUDWATCH MANAGEMENT:"
+    echo "  add-aws-credentials <access_key> <secret_key> [region]    Add AWS credentials for CloudWatch datasource"
+    echo "  show-aws-credentials           Show current AWS credentials (masked)"
+    echo "  remove-aws-credentials         Remove stored AWS credentials"
+    echo ""
     echo "GRAFANA PASSWORD MANAGEMENT:"
     echo "  generate-grafana-password      Generate secure password for Grafana"
     echo "  show-grafana-credentials       Show current Grafana credentials"
@@ -119,6 +124,11 @@ show_help() {
     echo "  $0 test-instance prod-db     # Test connection to 'prod-db' instance"
     echo "  $0 remove-instance old-db    # Remove 'old-db' instance"
     echo ""
+    echo "AWS CLOUDWATCH EXAMPLES:"
+    echo "  $0 add-aws-credentials AKIAIOSFODNN7EXAMPLE wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY us-east-1"
+    echo "  $0 show-aws-credentials      # Display masked AWS credentials"
+    echo "  $0 remove-aws-credentials    # Disable CloudWatch datasource"
+    echo ""
     echo "WORKFLOW:"
     echo "  QUICKSTART (RECOMMENDED):"
     echo "    • Run '$0 quickstart' for complete production setup (install + configure + start)"
@@ -681,6 +691,94 @@ get_api_key() {
     fi
 }
 
+# Add AWS credentials to configuration
+add_aws_credentials() {
+    local access_key="$1"
+    local secret_key="$2"
+    local region="${3:-us-east-1}"
+    
+    if [ -z "$access_key" ] || [ -z "$secret_key" ]; then
+        log_error "Please provide both AWS access key and secret key"
+        echo "Usage: $0 add-aws-credentials <access_key> <secret_key> [region]"
+        echo "  region defaults to us-east-1 if not specified"
+        exit 1
+    fi
+    
+    # Create config file if it doesn't exist
+    touch "$SCRIPT_DIR/.pgwatch-config"
+    
+    # Remove existing AWS credentials if present
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        grep -v "^aws_access_key=" "$SCRIPT_DIR/.pgwatch-config" | grep -v "^aws_secret_key=" | grep -v "^aws_region=" > "$SCRIPT_DIR/.pgwatch-config.tmp" || true
+        mv "$SCRIPT_DIR/.pgwatch-config.tmp" "$SCRIPT_DIR/.pgwatch-config"
+    fi
+    
+    # Add the new AWS credentials
+    echo "aws_access_key=$access_key" >> "$SCRIPT_DIR/.pgwatch-config"
+    echo "aws_secret_key=$secret_key" >> "$SCRIPT_DIR/.pgwatch-config"
+    echo "aws_region=$region" >> "$SCRIPT_DIR/.pgwatch-config"
+    
+    log_success "AWS credentials added successfully"
+    log_info "CloudWatch datasource will be enabled on next restart"
+    log_info "Region: $region"
+}
+
+# Show AWS credentials (masked for security)
+show_aws_credentials() {
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        local access_key=$(grep "^aws_access_key=" "$SCRIPT_DIR/.pgwatch-config" 2>/dev/null | cut -d'=' -f2)
+        local region=$(grep "^aws_region=" "$SCRIPT_DIR/.pgwatch-config" 2>/dev/null | cut -d'=' -f2)
+        
+        if [ -n "$access_key" ]; then
+            local masked_key="${access_key:0:4}$( printf '%*s' $((${#access_key} - 8)) '' | tr ' ' '*' )${access_key: -4}"
+            log_info "CloudWatch Configuration:"
+            echo "  AWS Access Key: $masked_key"
+            echo "  AWS Region:     ${region:-us-east-1}"
+            log_success "CloudWatch datasource is configured"
+        else
+            log_warning "No AWS credentials configured"
+            log_info "CloudWatch datasource is disabled"
+        fi
+    else
+        log_warning "No AWS credentials configured"
+        log_info "CloudWatch datasource is disabled"
+    fi
+}
+
+# Remove AWS credentials from configuration
+remove_aws_credentials() {
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        grep -v "^aws_access_key=" "$SCRIPT_DIR/.pgwatch-config" | grep -v "^aws_secret_key=" | grep -v "^aws_region=" > "$SCRIPT_DIR/.pgwatch-config.tmp" || true
+        mv "$SCRIPT_DIR/.pgwatch-config.tmp" "$SCRIPT_DIR/.pgwatch-config"
+        log_success "AWS credentials removed successfully"
+        log_info "CloudWatch datasource will be disabled on next restart"
+    else
+        log_warning "No AWS credentials configured"
+    fi
+}
+
+# Get AWS credentials from configuration
+get_aws_access_key() {
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        grep "^aws_access_key=" "$SCRIPT_DIR/.pgwatch-config" 2>/dev/null | cut -d'=' -f2
+    fi
+}
+
+get_aws_secret_key() {
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        grep "^aws_secret_key=" "$SCRIPT_DIR/.pgwatch-config" 2>/dev/null | cut -d'=' -f2
+    fi
+}
+
+get_aws_region() {
+    if [ -f "$SCRIPT_DIR/.pgwatch-config" ]; then
+        local region=$(grep "^aws_region=" "$SCRIPT_DIR/.pgwatch-config" 2>/dev/null | cut -d'=' -f2)
+        echo "${region:-us-east-1}"
+    else
+        echo "us-east-1"
+    fi
+}
+
 # Detect project directory
 detect_project_dir() {
     # Check if we're already in the project directory
@@ -2075,6 +2173,15 @@ main() {
         "remove-key")
             remove_api_key
             ;;
+        "add-aws-credentials")
+            add_aws_credentials "$2" "$3" "$4"
+            ;;
+        "show-aws-credentials")
+            show_aws_credentials
+            ;;
+        "remove-aws-credentials")
+            remove_aws_credentials
+            ;;
         "generate-grafana-password")
             generate_grafana_password
             ;;