Fix YAML/Jinja2 syntax (#243)

... and other problems that make our checks fail. I try hard to only use
configuration in Molecule or quick workarounds. Every issue I find will
go into it's own issue in this repository.


fixes #242

Author: widhalmt

.config/ansible-lint.yml

@@ -7,7 +7,6 @@ warn_list:
   - key-order[task]  # Ensure specific order of keys in mappings.
   - name[casing]
   - 'risky-shell-pipe'
-  - no-handler # backup of old certificates
   - var-naming[no-role-prefix] # remove when https://github.com/ansible/ansible-lint/discussions/3451 is done
 skip_list:
   - '106'
@@ -16,3 +15,4 @@ skip_list:
   - 'line-length'
   - 'package-latest'
   - yaml  # Violations reported by yamllint.
+  - no-handler # backup of old certificates

docs/logstash-pipelines.md

@@ -205,7 +205,7 @@ Every Output can have a `congestion:` option with a numerical value. If the Redi
 
 ### Unsafe shutdown ###
 
-If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. If you want better controll over which pipeline is allowed to shutdown unsafely, there are `ansible_input_unsafe_shutdown`and `ansible_forwarder_unsafe_shutdown` for default pipelines. And every pipeline has it's own `unsafe_shutdown` setting. All three default to the value of `logstash_pipeline_unsafe_shutdown` which by itself defaults to `false`. 
+If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`.
 
 ## Caveats ##
 

molecule/elasticstack_default/converge.yml

@@ -13,9 +13,10 @@
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
-    elasticsearch_heap: "1"
+    elasticsearch_heap: "2"
     elasticstack_full_stack: true
     logstash_pipeline_unsafe_shutdown: true
+    logstash_password_hash: false
     beats_filebeat_syslog_udp: true
     beats_filebeat_syslog_tcp: true
     beats_filebeat_modules:
@@ -28,10 +29,13 @@
         elasticstack_rpm_workaround: true
       when: ansible_os_family == 'RedHat' and ansible_distribution_major_version >= "9"
     - name: Update apt cache.
-      apt: update_cache=yes cache_valid_time=600
+      ansible.builtin.apt:
+        update_cache: yes
+        cache_valid_time: 600
+      changed_when: false
       when: ansible_os_family == 'Debian'
     - name: Install dependencies
-      package:
+      ansible.builtin.package:
         name:
           - curl
     - name: Include Redis

molecule/elasticstack_default/verify.yml

@@ -7,10 +7,6 @@
     elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords
   tasks:
 
-    - name: Give some time for tools to connect
-      ansible.builtin.wait_for:
-        timeout: 120
-
     - name: Run Logstash syntax check
       ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t"
       when: "'logstash' in group_names"

roles/elasticsearch/tasks/elasticsearch-security.yml

@@ -24,7 +24,7 @@
 
 - name: Set the ca expiration date in days
   ansible.builtin.set_fact:
-    elasticstack_ca_expiration_days: "{{ (( elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
+    elasticstack_ca_expiration_days: "{{ ((elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
   when: inventory_hostname == elasticstack_ca and elasticstack_ca_infos.skipped is not defined
 
 - name: Set ca will expire soon to true

roles/elasticsearch/tasks/main.yml

@@ -148,8 +148,8 @@
     mode: "2750"
   when: item.create | bool
   loop:
-    - {create: "{{elasticsearch_create_logpath}}", path: "{{ elasticsearch_logpath }}" }
-    - {create: "{{elasticsearch_create_datapath}}", path: "{{ elasticsearch_datapath }}" }
+    - {create: "{{ elasticsearch_create_logpath }}", path: "{{ elasticsearch_logpath }}" }
+    - {create: "{{ elasticsearch_create_datapath }}", path: "{{ elasticsearch_datapath }}" }
 
 - name: Import Tasks elasticsearch-security.yml
   ansible.builtin.import_tasks: elasticsearch-security.yml

roles/elasticsearch/templates/jvm.options.j2

@@ -73,15 +73,15 @@
 ################################################################
 
 ## GC configuration
-{% if elasticstack_release is version('7.6.0', '<') %}
--XX:+UseConcMarkSweepGC
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
-{% else %}
-8-13:-XX:+UseConcMarkSweepGC
-8-13:-XX:CMSInitiatingOccupancyFraction=75
-8-13:-XX:+UseCMSInitiatingOccupancyOnly
-{% endif %}
+#{% if elasticstack_release is version('7.6.0', '<') %}
+##-XX:+UseConcMarkSweepGC
+##-XX:CMSInitiatingOccupancyFraction=75
+##-XX:+UseCMSInitiatingOccupancyOnly
+#{% else %}
+#8-13:-XX:+UseConcMarkSweepGC
+#8-13:-XX:CMSInitiatingOccupancyFraction=75
+#8-13:-XX:+UseCMSInitiatingOccupancyOnly
+#{% endif %}
 
 ## G1GC Configuration
 # NOTE: G1 GC is only supported on JDK version 10 or later

roles/logstash/defaults/main.yml

@@ -17,7 +17,6 @@ logstash_config_path_logs: /var/log/logstash
 # pipeline configuration #
 logstash_manage_pipelines: true
 logstash_queue_type: persisted
-logstash_pipeline_unsafe_shutdown: false
 
 # this will deactivate all pipeline management
 logstash_no_pipelines: false

roles/logstash/templates/logstash.yml.j2

@@ -13,6 +13,9 @@ http.port: {{ logstash_http_port }}
 {% if logstash_global_ecs is defined %}
 pipeline.ecs_compatibility: {{ logstash_global_ecs }}
 {% endif %}
+{% if pipeline.unsafe_shutdown is defined %}
+pipeline.unsafe_shutdown: {{ logstash_pipeline_unsafe_shutdown }}
+{% endif %}
 {% if logstash_legacy_monitoring | bool and elasticstack_full_stack | bool and elasticstack_variant == "elastic" and elasticstack_release | int < 8 %}
 xpack.monitoring.enabled: true
 xpack.monitoring.elasticsearch.hosts: [ {% for host in logstash_elasticsearch %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %} ]

roles/logstash/templates/pipelines.yml.j2

@@ -19,29 +19,21 @@
 # Default beat input #
 # Autoconfigured Redis outputs: input
 
-- pipeline
-    id: ansible-input
-    unsafe_shutdown: {{ ansible_input_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }}
-  path
-    config: "/etc/logstash/conf.d/ansible-input/*.conf"
-  queue
-    type: {{ logstash_input_queue_type }}
-    max_bytes: {{ logstash_input_queue_max_bytes }}
+- pipeline.id: ansible-input
+  path.config: "/etc/logstash/conf.d/ansible-input/*.conf"
+  queue.type: {{ logstash_input_queue_type }}
+  queue.max_bytes: {{ logstash_input_queue_max_bytes }}
 
 {% endif %}
 {% if logstash_elasticsearch_output | bool %}
 
 # Default elasticsearch output #
 # Autoconfigured Redis input: forwarder
 
-- pipeline
-    id: ansible-forwarder
-    unsafe_shutdown: {{ ansible_forwarder_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }}
-  path
-    config: "/etc/logstash/conf.d/ansible-forwarder/*.conf"
-  queue
-    type: {{ logstash_forwarder_queue_type }}
-    max_bytes: {{ logstash_forwarder_queue_max_bytes }}
+- pipeline.id: ansible-forwarder
+  path.config: "/etc/logstash/conf.d/ansible-forwarder/*.conf"
+  queue.type: {{ logstash_forwarder_queue_type }}
+  queue.max_bytes: {{ logstash_forwarder_queue_max_bytes }}
 
 {% endif %}
 {% if logstash_pipelines is defined %}
@@ -72,14 +64,10 @@
 
 {% endif %}
 
-- pipeline
-    id: {{ item.name }}
-    unsafe_shutdown: {{ item.unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }}
-  path
-    config: "/etc/logstash/conf.d/{{ item.name }}/*.conf"
-  queue
-    type: {{ item.queue_type | default('memory') }}
-    max_bytes: {{ item.queue_max_bytes | default('1gb') }}
+- pipeline.id: {{ item.name }}
+  path.config: "/etc/logstash/conf.d/{{ item.name }}/*.conf"
+  queue.type: {{ item.queue_type | default('memory') }}
+  queue.max_bytes: {{ item.queue_max_bytes | default('1gb') }}
 
 {% endfor %}
 {% endif %}