ELK

ELK Stack: Elasticsearch, Logstash, Kibana

https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html

Elasticsearch

wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/9.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-9.x.list
sudo apt-get update && sudo apt-get install elasticsearch

vi elasticsearch.yml
cluster.name: blob2queue
network.host: 0.0.0.0
transport.host: 0.0.0.0

sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node

curl -k -X POST "https://localhost:9200/_security/api_key" -H "Content-Type: application/json" -u elastic:elastic_password -d '{
    "name": "vnet-indexing-key",
    "role_descriptors": {
      "vnet_writer": {
        "indices": [
          {
            "names": [ "vnet*" ],
            "privileges": [
              "create_index","create","index","write"
            ]
          }
        ]
      }
    }
  }'

sudo service elasticsearch start
sudo service elasticsearch status
sudo service elasticsearch stop

sudo tail -f /var/log/elasticsearch/elasticsearch.log

sudo mkdir /var/lib/elasticsearch
sudo chown elasticsearch:elasticsearch /var/lib/elasticsearch
sudo ls -la /var/lib/elasticsearch
sudo tail -f /var/log/elasticsearch/elasticsearch.log

https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/cluster-level-shard-allocation-routing-settings#cluster-shard-allocation-settings

cluster.routing.allocation.disk.watermark.flood_stage: 99%
cluster.routing.allocation.disk.watermark.flood_stage.max_headroom: 10MB
cluster.routing.allocation.disk.threshold_enabled: false

Kibana

sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token - -scope kibana
sudo -u kibana /usr/share/kibana/bin/kibana-verification-code

sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic

http://localhost:5601/

Logstash

docker run --rm -ti docker.elastic.co/logstash/logstash-oss:8.3.2 /bin/bash

update-alternatives --install /usr/bin/gem gem /usr/share/logstash/vendor/jruby/bin/gem 20
update-alternatives --install /usr/bin/jgem jgem /usr/share/logstash/vendor/jruby/bin/jgem 20
update-alternatives --install /usr/bin/jruby jruby /usr/share/logstash/vendor/jruby/bin/jruby 20

docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0

Update Logstash plugin

git clone https://github.com/Azure-Samples/storage-blobs-go-quickstart

VERSION=$(grep version logstash-input-azure_blob_storage.gemspec | cut -d"'" -f 2)
GEMPWD=$(pwd)
echo "Building ${VERSION}"
pushd /usr/share/logstash
sudo -u logstash /usr/share/logstash/bin/logstash-plugin remove logstash-input-azure_blob_storage
popd

sudo -u logstash gem build logstash-input-azure_blob_storage.gemspec
sudo -u logstash gem install logstash-input-azure_blob_storage-${VERSION}.gem

pushd /usr/share/logstash
sudo -u logstash /usr/share/logstash/bin/logstash-plugin install ${GEMPWD}/logstash-input-azure_blob_storage-${VERSION}.gem
popd

sudo -u logstash /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf --config.reload.automatic

vi CHANGELOG.md
vi README.md

git add CHANGELOG.md
git add logstash-input-azure_blob_storage.gemspec
git add lib/logstash/inputs/azure_blob_storage.rb
git commit -m "fixed"
git push

git tag 0.12.3
git push --tags

gem push logstash-input-azure_blob_storage

Update Bundle

/usr/share/logstash/bin/logstash-plugin update
/usr/share/logstash/bin/logstash-plugin generate --type input --name test --path .

sudo -u logstash bash
bundle -v
bundle update
bundle install
bundle exec rake vendor
bundle exec rspec
bundle exec rspec spec/inputs/
bundle exec rake publish_gem

gem env
bundle exec rspec spec/inputs/azure_blob_storage_spec.rb

jruby -S bundle install
jruby -S gem list

gem install bundler
/usr/share/logstash/bin/logstash-plugin install --development
gem install logstash-core
gem install logstash-core-plugin-api
gem install logstash-devutils
bundle exec rspec

gem uninstall logstash-input-azure_blob_storage:0.11.4

https://stackoverflow.com/questions/33523395/testing-custom-logstash-filters

Rubocop

gem install rubocop
rubocop --only-guide-cops -a
rubocop --only-guide-cops -a .\lib\logstash\inputs\azure_blob_storage.rb

JRUBY https://www.jruby.org/download

JDK https://docs.microsoft.com/en-us/java/openjdk/download

IMPLEMENTOR="Eclipse Adoptium" IMPLEMENTOR_VERSION="Temurin-11.0.15+10" JAVA_VERSION="11.0.15" JAVA_VERSION_DATE="2022-04-19"

Filebeat

TCP/5044 lumberjack v2

https://logz.io/blog/filebeat-vs-logstash/

Logz.io Opensearch Opensearch Dashboard