Merge pull request #1 from Diesel-Net/development

Tested

.drone.yaml

@@ -0,0 +1,32 @@
+---
+kind: pipeline
+type: docker
+name: Build FluentD Image
+
+concurrency:
+  limit: 1
+
+clone:
+  depth: 1
+
+steps:
+  - name: docker build & push
+    image: plugins/docker
+    settings:
+      mirror: dockerhub.nexus.diesel.net
+      registry: docker.nexus.diesel.net
+      username: 
+        from_secret: nexus_user
+      password:
+        from_secret: nexus_pass
+      repo: docker.nexus.diesel.net/fluentd
+      tags:
+        - latest
+
+trigger:
+  branch:
+    - development
+    - stable
+  event:
+    - push
+...

Dockerfile

@@ -0,0 +1,8 @@
+FROM fluent/fluentd:v1.14.6-debian-1.0
+USER root
+RUN ["gem", "install", "fluent-plugin-grafana-loki"]
+RUN ["gem", "install", "fluent-plugin-rewrite-tag-filter"]
+RUN ["gem", "install", "fluent-plugin-record-reformer"]
+RUN ["gem", "install", "fluent-plugin-filter-docker_metadata"]
+RUN ["gem", "install", "fluent-plugin-detect-exceptions"]
+USER root

README.md

@@ -1,2 +1,9 @@
+[![Build Status](https://drone.kiwi-labs.net/api/badges/Diesel-Net/ansible-role-fluentd/status.svg)](https://drone.kiwi-labs.net/Diesel-Net/ansible-role-fluentd)
+
 # ansible-role-fluentd
-Fluentd log collector/shipper on Docker Swarm
+
+Log Collector on Docker Swarm
+
+- [Fluentd - Grafana Loki Docs](https://grafana.com/docs/loki/latest/clients/fluentd/)
+- [Fluentd Congiguration](https://docs.fluentd.org/configuration)
+    - When this IaC was written, the YAML config file format did not exist. Perhaps we convert to that config file format in the future?

defaults/main.yaml

@@ -0,0 +1,4 @@
+# defaults
+loki_url: https://loki.diesel.net
+loki_username: loki
+loki_password: loki

meta/main.yaml

@@ -0,0 +1,29 @@
+galaxy_info:
+  author: Thomas Daley
+  description: Installs fluentd agent for streaming Docker Swarm logs to Loki
+
+  min_ansible_vesion: 2.14
+  platforms:
+    - name: Ubuntu
+      versions:
+        - 18.04
+        - 20.04
+
+  galaxy_tags:
+    - ubuntu
+    - docker
+    - fluentd
+    - logging
+    - grafana
+
+  dependencies:
+
+    - name: application
+      scm: git
+      src: "[email protected]:Diesel-Net/ansible-role-application.git"
+      version: 2.1.0
+
+    - name: docker
+      scm: git
+      src: "[email protected]:Diesel-Net/ansible-role-docker.git"
+      version: 2.0.3

tasks/configure.yaml

@@ -0,0 +1,12 @@
+- include_role:
+    name: application
+    tasks_from: make_config_dir
+
+- include_role:
+    name: application
+    tasks_from: make_data_dir
+
+- name: Render loki.conf to {{ config_dir }}/fluent.conf
+  template:
+    src: loki.conf
+    dest: '{{ config_dir }}/fluent.conf'

tasks/deploy.yaml

@@ -0,0 +1,10 @@
+- include_role:
+    name: docker
+    tasks_from: stack_deploy
+
+# force reload
+- include_role:
+    name: docker
+    tasks_from: service_update
+  vars:
+    options: --force

tasks/main.yaml

@@ -0,0 +1,8 @@
+- include_role:
+    name: application
+
+- include_role:
+    name: docker
+
+- include_tasks: configure.yaml
+- include_tasks: deploy.yaml

templates/docker-compose.yaml

@@ -0,0 +1,25 @@
+version: '3.8'
+services: 
+
+  {{ docker_service }}:
+    image: {{ docker_registry }}/fluentd:latest
+    volumes:
+      - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt
+      - /etc/localtime:/etc/localtime
+      - {{ data_dir }}/:/fluentd/log/
+      - {{ config_dir }}/:/fluentd/etc/
+      - /var/lib/docker/containers/:/var/lib/docker/containers/
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      #https://github.com/fluent/fluentd-kubernetes-daemonset#run-as-root
+      - FLUENT_UID=0
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+      resources:
+          limits:
+            cpus: '2.0'
+          reservations:
+            # https://docs.fluentd.org/deployment/performance-tuning-single-process#multi-workers
+            cpus: '1.0'

templates/loki.conf

@@ -0,0 +1,110 @@
+<source>
+  @type tail
+  @id in_tail_container_logs
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /fluentd/log/containers.log.pos
+  tag docker.*
+  read_from_head false # only affects the startup behaviour from v1.14.3+
+  follow_inodes true # avoids reading rotated files duplicately
+  refresh_interval 5 # we want this pretty low for quicker pickup of repeated deployments
+  read_bytes_limit_per_second 33554432
+  <parse>
+    @type json
+    time_key time
+    time_format %iso8601
+  </parse>
+</source>
+
+<filter docker.var.lib.docker.containers.*.*.log>
+  @type docker_metadata
+  # https://rubygems.org/gems/fluent-plugin-filter-docker_metadata
+  # https://github.com/wshihadeh/fluent-plugin-filter-docker_metadata/blob/master/lib/fluent/plugin/filter_docker_metadata.rb
+  labels com.docker.stack.namespace:namespace,com.docker.swarm.service.name:service_name
+</filter>
+
+<match docker.var.lib.docker.containers.*.*.log>
+  @type rewrite_tag_filter
+  <rule>
+    key container_name
+    pattern {{ ignored_container_patterns | join('|') }}
+    tag ignore.${tag}
+  </rule>
+  <rule>
+    key service_name
+    pattern /^(.+)$/
+    tag raw.$1
+  </rule>
+  <rule>
+    key container_name
+    pattern /^(.+)$/
+    tag raw.adhoc
+  </rule>
+</match>
+
+<match ignore.**>
+  @type null
+</match>
+
+# use service_name for container_name for the rare case of adhoc containers (non-swarm services)
+# <filter raw.**>
+#   @type record_transformer
+#   enable_ruby
+#   <record>
+#     service_name ${record["service_name"] or record["container_name"]}
+#   </record>
+# </filter>
+
+# uncomment for debugging
+# <match **>
+#   @type stdout
+# </match>
+
+<match raw.**>
+  @type detect_exceptions
+  remove_tag_prefix raw
+  message log
+  languages python
+</match>
+
+<match **>
+  @type relabel
+  @label @LOGS_PARSED
+</match>
+
+<label @LOGS_PARSED>
+  <match **>
+    # https://grafana.com/docs/loki/latest/clients/fluentd/#output-format
+    @type loki
+    url {{ loki_url }}
+    ca_cert /etc/ssl/certs/ca-certificates.crt
+    username {{ loki_username }}
+    password {{ loki_password }}
+
+    <label>
+      namespace
+      service_name
+      container_name
+      stream
+    </label>
+
+    extra_labels {"environment":"{{ version }}", "vm_hostname":"{{ inventory_hostname }}", "job":"{{ loki_job }}"}
+
+    <buffer service_name>
+      @type file
+      path /fluentd/log/dlog/loki-buffer
+      flush_thread_count 8
+      flush_mode interval
+      flush_interval 1s
+      retry_timeout 72h
+      overflow_action throw_exception
+    </buffer>
+
+  </match>
+</label>
+
+# not sure what this does anymore or if this is needed...
+<label @FLUENT_LOG>
+  <match **>
+    @type null
+  </match>
+</label>

vars/main.yaml

@@ -0,0 +1,10 @@
+app_name: fluentd
+docker_service: daemon
+
+# container names matching any one of these patterns, will instruct fluentd to drop the logs
+ignored_container_patterns:
+  - (^.*fluentd_.+$) # fluentd collector on each host e.g. "fluentd_development_collector.f2wm1ivqj5x9o2rqmzt4rh1hb.nngvjdjhy67s9w2gfnhtid99u"
+  - (^.*grafana-agent.+$) # grafana-agent scraper on each host e.g. "grafana-agent_development_scraper.f2wm1ivqj5x9o2rqmzt4rh1hb.nngvjdjhy67s9w2gfnhtid99u"
+  - (^drone-[\w]{20}$) # drone build-step containers e.g. "drone-eijUWwBU8QPyv2pXGnJ1" (only works for Drone Enterprise, as OSS version does not follow this naming standard)
+
+loki_job: docker-swarm