• Menu

    io.yupiik.alveoli:loki-simple:1.0.27-SNAPSHOT

    Loki setup to aggregate service logs using promtail, the prometheus for logs.

    Warning
    		 Starting from 1.0.20, the Promtail descriptor moved from Deployment to DaemonSet.

    Maven Dependency

    <dependency>
  <groupId>io.yupiik.alveoli</groupId>
  <artifactId>loki-simple</artifactId>
  <version>1.0.27-SNAPSHOT</version>
</dependency>

    Default Configuration

    loki.yaml
    # this is the default local configuration for loki
auth_enabled: false
server:
  http_listen_port: 3100
ingester:
  wal:
    enabled: false
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
  max_transfer_retries: 0     # Chunk transfers disabled
schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h
storage_config:
  boltdb_shipper:
    active_index_directory: /tmp/loki/boltdb-shipper-active
    cache_location: /tmp/loki/boltdb-shipper-cache
    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
    shared_store: filesystem
  filesystem:
    directory: /tmp/loki/chunks
compactor:
  working_directory: /tmp/loki/boltdb-shipper-compactor
  shared_store: filesystem
limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
chunk_store_config:
  max_look_back_period: 0s
table_manager:
  retention_deletes_enabled: false
  retention_period: 0s
ruler:
  storage:
    type: local
    local:
      directory: /tmp/loki/rules
  rule_path: /tmp/loki/rules-temp
  alertmanager_url: http://localhost:9093
  ring:
    kvstore:
      store: inmemory
  enable_api: true
    promtail.yaml
    # See also https://clymene-project.github.io/docs/service-discovery/promtail-config/
server:
  log_level: info
  http_listen_port: 3101
clients:
  - url: {{loki-simple.loki.client_url:-http://loki:3100/loki/api/v1/push}}
positions:
  filename: /run/promtail/positions.yaml
scrape_configs:
  # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
  # Pods with a label 'app.kubernetes.io/name'
  - job_name: kubernetes-pods-app-kubernetes-io-name
    pipeline_stages:
      - {{loki-simple.loki.pipeline_stage:-cri}}: {}
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_name
        target_label: app
      - action: drop
        regex: ''
        source_labels:
          - app
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_component
        target_label: component
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - namespace
        - app
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
        - __meta_kubernetes_pod_container_name
        target_label: __path__
  # Pods with a label 'app'
  - job_name: kubernetes-pods-app
    pipeline_stages:
      - {{loki-simple.loki.pipeline_stage:-cri}}: {}
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Drop pods with label 'app.kubernetes.io/name'. They are already considered above
      - action: drop
        regex: .+
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_name
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_label_app
        target_label: app
      - action: drop
        regex: ''
        source_labels:
          - app
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_label_component
        target_label: component
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - namespace
        - app
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
        - __meta_kubernetes_pod_container_name
        target_label: __path__
  # Pods with direct controllers, such as StatefulSet
  - job_name: kubernetes-pods-direct-controllers
    pipeline_stages:
      - {{loki-simple.loki.pipeline_stage:-cri}}: {}
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Drop pods with label 'app.kubernetes.io/name' or 'app'. They are already considered above
      - action: drop
        regex: .+
        separator: ''
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_name
          - __meta_kubernetes_pod_label_app
      - action: drop
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
          - __meta_kubernetes_pod_controller_name
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_controller_name
        target_label: app
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - namespace
        - app
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
        - __meta_kubernetes_pod_container_name
        target_label: __path__
  # Pods with indirect controllers, such as Deployment
  - job_name: kubernetes-pods-indirect-controller
    pipeline_stages:
      - {{loki-simple.loki.pipeline_stage:-cri}}: {}
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Drop pods with label 'app.kubernetes.io/name' or 'app'. They are already considered above
      - action: drop
        regex: .+
        separator: ''
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_name
          - __meta_kubernetes_pod_label_app
      - action: keep
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
          - __meta_kubernetes_pod_controller_name
      - action: replace
        regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'
        source_labels:
          - __meta_kubernetes_pod_controller_name
        target_label: app
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - namespace
        - app
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
        - __meta_kubernetes_pod_container_name
        target_label: __path__
  # All remaining pods not yet covered
  - job_name: kubernetes-other
    pipeline_stages:
      - {{loki-simple.loki.pipeline_stage:-cri}}: {}
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Drop what has already been covered
      - action: drop
        regex: .+
        separator: ''
        source_labels:
          - __meta_kubernetes_pod_label_app_kubernetes_io_name
          - __meta_kubernetes_pod_label_app
      - action: drop
        regex: .+
        source_labels:
          - __meta_kubernetes_pod_controller_name
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_name
        target_label: app
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_label_component
        target_label: component
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - namespace
        - app
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
        - __meta_kubernetes_pod_container_name
        target_label: __path__

    Ports

    • Name: loki (loki)

      • Protocol: TCP

      • Port: 3100

      • Target Port: 3100

      • Node Port: 32100

    Tip
    		 on linux and with minikube you can access this service using http://$(minikube ip):32100 on your host.

    • Name: promtail (promtail-http)

      • Protocol: TCP

      • Port: 3101

      • Target Port: 3101

      • Node Port: 32101

    Tip
    		 on linux and with minikube you can access this service using http://$(minikube ip):32101 on your host.

    Sample Usage

    {
  "alveoli": [
    {
      "//": "My alveolus.",
      "name": "com.company:my-app:1.0.0",
      "descriptors": [],
      "dependencies": [
        {
          "name": "io.yupiik.alveoli:loki-simple:1.0.27-SNAPSHOT",
          "location": "io.yupiik.alveoli:loki-simple:1.0.27-SNAPSHOT",
        }
      ],
      "patches": [{
        "descriptorName": "io.yupiik.alveoli:loki-simple:1.0.27-SNAPSHOT",
        "interpolate": true
      }]
    }
  ]
}

    Configuration

    loki-simple.loki.client_url

    URL of the Loki API used to push data. Default value: http://loki:3100/loki/api/v1/push.

    loki-simple.loki.image

    The image to use for loki. Default value: grafana/loki:2.8.2.

    loki-simple.loki.pipeline_stage

    Pipeline stage to use, generally cri or docker. Default value: cri.

    loki-simple.loki.resources.limits.cpu

    The resource limits cpu for Loki API. Default value: 0.2.

    loki-simple.loki.resources.limits.memory

    The resource limits memory for Loki API. Default value: 128Mi.

    loki-simple.loki.resources.requests.cpu

    The resource requests cpu for Loki API. Default value: 0.1.

    loki-simple.loki.resources.requests.memory

    The resource requests memory for Loki API. Default value: 64Mi.

    loki-simple.namespace

    The Kubernetes namespace to use for the installation. Default value: ``.

    loki-simple.promtail.image

    The image to use for promtail. Default value: grafana/promtail:2.8.2.

    loki-simple.resources.limits.cpu

    The resource limits cpu for Promtail. Default value: 0.2.

    loki-simple.resources.limits.memory

    The resource limits memory for Promtail. Default value: 128Mi.

    loki-simple.resources.requests.cpu

    The resource requests cpu for Promtail. Default value: 0.1.

    loki-simple.resources.requests.memory

    The resource requests memory for Promtail. Default value: 64Mi.

    Navigation