Skip to main content

Logstash

Overview​

Logstash digunakan untuk sync data dari PostgreSQL ke OpenSearch.

Configuration​

Docker Compose​

logstash:
  image: clm-logstash:latest
  build:
    context: ./configs/logstash
    dockerfile: Dockerfile
  container_name: logstash
  depends_on:
    postgres:
      condition: service_healthy
    opensearch:
      condition: service_started
  environment:
    LS_JAVA_OPTS: "-Xms512m -Xmx512m"
  volumes:
    - ./configs/logstash/databases.json:/usr/share/logstash/config/databases.json:ro
    - ./configs/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
    - logstash-data:/usr/share/logstash/data
    - postgres-os-metrics:/usr/share/logstash/postgres-os-metrics:ro
  ports:
    - "5044:5044"
    - "9601:9600"
  mem_limit: 1g
  cpus: "0.50"

logstash.yml​

http.host: "0.0.0.0"
xpack.monitoring.enabled: false
pipeline.workers: 2
pipeline.batch.size: 125

databases.json​

[
  {
    "name": "clm_logs",
    "host": "postgres",
    "port": 5432,
    "database": "clm_logs",
    "username": "admin",
    "password": "postgres123",
    "tables": ["users", "tenants", "audit_logs"]
  }
]

Pipelines​

PostgreSQL Sync​

# config.d/postgresql-sync.conf
input {
  jdbc {
    jdbc_connection_string => "jdbc:postgresql://postgres:5432/clm_logs"
    jdbc_user => "admin"
    jdbc_password => "postgres123"
    jdbc_driver_library => "/usr/share/logstash/postgresql-42.6.0.jar"
    jdbc_driver_class => "org.postgresql.Driver"
    statement => "SELECT * FROM audit_logs WHERE updated_at > :sql_last_value"
    schedule => "*/5 * * * *"
    use_column_value => true
    tracking_column => "updated_at"
    last_run_metadata_path => "/usr/share/logstash/data/.logstash_jdbc_last_run"
  }
}

filter {
  mutate {
    add_field => { "source_type" => "postgresql" }
    add_field => { "app_slug" => "clm-backend" }
  }
  
  date {
    match => [ "created_at", "ISO8601" ]
    target => "@timestamp"
  }
}

output {
  opensearch {
    hosts => ["https://opensearch:9200"]
    user => "admin"
    password => "MyStrong@OpenSearch2026!"
    ssl => true
    ssl_certificate_verification => false
    index => "audit-log-%{+YYYY.MM.dd}"
  }
}

PostgreSQL Metrics​

# config.d/postgresql-metrics.conf
input {
  file {
    path => "/usr/share/logstash/postgres-os-metrics/*.json"
    start_position => "beginning"
    sincedb_path => "/usr/share/logstash/data/.sincedb_metrics"
  }
}

filter {
  json {
    source => "message"
  }
  
  mutate {
    add_field => { "source_type" => "metrics" }
    add_field => { "app_slug" => "postgresql" }
  }
}

output {
  opensearch {
    hosts => ["https://opensearch:9200"]
    user => "admin"
    password => "MyStrong@OpenSearch2026!"
    ssl => true
    ssl_certificate_verification => false
    index => "postgresql-metrics-%{+YYYY.MM.dd}"
  }
}

Monitoring​

Health Check​

curl http://localhost:9600/_node/stats

Pipeline Stats​

curl http://localhost:9600/_node/stats/pipelines

Troubleshooting​

Check Logs​

docker logs logstash --tail 100

Test Configuration​

docker exec logstash logstash --config.test_and_exit -f /usr/share/logstash/pipeline/

Next Steps​