Splunk Day-2

/Volumes/SREE256GB/AgileBrains-Videos/Splunk/Splunk-Apr2021/splunk-lab
(base) SreeMacMin16GB-1280:splunk-lab sree$ cat md.sh 

echo "expose_extensions: pdf,docx,xlsx,txt" > .madness.yml 
docker rm -f md
docker run -d --rm -it -v $PWD:/docs --name md -p 8888:3000 dannyben/madness server --auth sree:sree123

index="tutorial_data" sourcetype="access*" productId > 0 | lookup prices_lookup productId AS productId OUTPUT price AS price product_name AS productName




index="tutorial_data" sourcetype=access_combined_wcookie | top limit=10 productName
index="tutorial_data" sourcetype=access_combined_wcookie | stats count by productName
index="tutorial_data" sourcetype=access_combined_wcookie | top limit=10 productName

index="tutorial_data" sourcetype="access*" productId > 0 | lookup product_name_price productId AS productId OUTPUT price AS price product_name AS productName

docker-compose exec web-uf bash
root@61818e91f900:/opt/splunkforwarder# sudo nano etc/apps/search/local/inputs.conf

[monitor:///var/log]
disabled = false
sourcetype = access_combined
index = web-uf_index

sudo ./bin/splunk restart


curl localhost:7000
curl localhost:7000/aaa
docker-compose exec web-uf bash
root@7bb0b0a5c708:/var/www/html# echo EEE > sree.txt

index="web-uf_index" status=404

[ansible@e82b8cd20dec splunkforwarder]$ sudo ./bin/splunk list monitor
Splunk username: admin
Password: 
Monitored Directories:
     $SPLUNK_HOME/var/log/splunk
         /opt/splunkforwarder/var/log/splunk/audit.log
         /opt/splunkforwarder/var/log/splunk/btool.log
         /opt/splunkforwarder/var/log/splunk/conf.log
         /opt/splunkforwarder/var/log/splunk/first_install.log
         /opt/splunkforwarder/var/log/splunk/license_usage.log
         /opt/splunkforwarder/var/log/splunk/mongod.log
         /opt/splunkforwarder/var/log/splunk/remote_searches.log
         /opt/splunkforwarder/var/log/splunk/scheduler.log
         /opt/splunkforwarder/var/log/splunk/searchhistory.log
         /opt/splunkforwarder/var/log/splunk/splunkd-utility.log
         /opt/splunkforwarder/var/log/splunk/splunkd_access.log
         /opt/splunkforwarder/var/log/splunk/splunkd_stderr.log
         /opt/splunkforwarder/var/log/splunk/splunkd_stdout.log
         /opt/splunkforwarder/var/log/splunk/splunkd_ui_access.log
     $SPLUNK_HOME/var/log/splunk/metrics.log
         /opt/splunkforwarder/var/log/splunk/metrics.log
     $SPLUNK_HOME/var/log/splunk/splunkd.log
         /opt/splunkforwarder/var/log/splunk/splunkd.log
     $SPLUNK_HOME/var/spool/splunk/...stash_new
     /var/log
         /var/log/alternatives.log
         /var/log/apt
         /var/log/apt/eipp.log.xz
         /var/log/apt/history.log
         /var/log/apt/term.log
         /var/log/bootstrap.log
         /var/log/btmp
         /var/log/dpkg.log
         /var/log/faillog
         /var/log/lastlog
         /var/log/nginx
         /var/log/nginx/access.log
         /var/log/nginx/error.log
         /var/log/tallylog
         /var/log/wtmp
Monitored Files:
     $SPLUNK_HOME/etc/splunk.version
[ansible@e82b8cd20dec splunkforwarder]$ ping splunk
PING splunk (192.168.32.6): 56 data bytes
64 bytes from 192.168.32.6: seq=0 ttl=64 time=10.204 ms
64 bytes from 192.168.32.6: seq=1 ttl=64 time=0.479 ms
^C
--- splunk ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.479/5.341/10.204 ms

root@5b0a57eba1dc:/opt/splunkforwarder# sudo ./bin/splunk list forward-server
Active forwards:
     splunk:9997
Configured but inactive forwards:
     None

[ansible@e82b8cd20dec splunkforwarder]$ sudo ./bin/splunk add forward-server splunk:9997
Added forwarding to: splunk:9997.
[ansible@e82b8cd20dec splunkforwarder]$

[ansible@e82b8cd20dec splunkforwarder]$ sudo cat etc/system/local/outputs.conf

[indexAndForward]
index = false

[tcpout]
defaultGroup = default-autolb-group

[tcpout:default-autolb-group]
server = splunk:9997

[tcpout-server://splunk:9997]

[ansible@e82b8cd20dec splunkforwarder]$ sudo ./bin/splunk list forward-server
Active forwards:
    splunk:9997
Configured but inactive forwards:
    None

docker-compose exec collectd bash

nano /etc/collectd/collectd.conf


Hostname "dd15a3fc994c"


FQDNLookup false
Interval 10
Timeout 2
ReadThreads 5
WriteThreads 5


#Sree enabled these
LoadPlugin cpu
LoadPlugin interface
LoadPlugin load
LoadPlugin memory


TypesDB "/usr/share/collectd/types.db"
Include "/etc/collectd/collectd.d/*.conf"




<Plugin write_http>
    <Node "node1">
        URL "https://splunk:8088/services/collector/raw?channel=4609eb39-f258-435c-9a75-cfc2ea1303d4"
        Header "Authorization: Splunk edc2b152-2e32-41db-ad62-72f3a9ae7c5b"
        Format "JSON"
        VerifyPeer false
        VerifyHost false
        Metrics true
        StoreRates true
    </Node>
</Plugin>

# SEE ALREADY INCLUDING THIS
LoadPlugin write_http
<Plugin write_http>
  <Node "node-http-1">
    URL "http://splunk:8088/services/collector/raw?channel={{SPLUNK_TOKEN}}"
    Header "Authorization: Splunk {{SPLUNK_TOKEN}}"
    Format "JSON"
    Metrics true
    StoreRates true
  </Node>
</Plugin>

root@dd15a3fc994c:/etc# service collectd restart     
root@dd15a3fc994c:/etc# service collectd status 
 * collectd is running




docker-compose exec splunk sudo ./bin/splunk restart

CHECK collectd_index !!!! 

docker-compose exec collectd bash
find / -type f -exec grep -l abcd {} \;


SIMPLE DOING THIS RETURNS SUCCESS
YES
(base) SreeMacMin16GB-1199:lab sree$ curl -k https://localhost:8088/services/collector/event -H "Authorization: Splunk edc2b152-2e32-41db-ad62-72f3a9ae7c5b" -d '{"event": "hello world"}'
{"text":"Success","code":0}

YES THIS WORKS
curl -k https://localhost:8088/services/collector/raw?sourcetype=collectd_http \
-H "Authorization: Splunk edc2b152-2e32-41db-ad62-72f3a9ae7c5b" -d '[{"values":[164.9196798931339196],"dstypes":["derive"],"dsnames":["value"],"time":1505356687.894,"interval":10.000,"host":"collectd","plugin":"protocols","plugin_instance":"IpExt","type":"protocol_counter","type_instance":"InOctets"}]'
{"text":"Success","code":0}


NOTHING TO DO
(base) SreeMacMin16GB-1199:lab sree$ docker-compose exec splunk bash
[ansible@86e8ec3f2f07 splunk]$ sudo cat /opt/splunk/etc/apps/search/local/indexes.conf
[web-uf_index]
coldPath = $SPLUNK_DB/web-uf_index/colddb
enableDataIntegrityControl = 0
enableTsidxReduction = 0
homePath = $SPLUNK_DB/web-uf_index/db
maxTotalDataSizeMB = 512000
thawedPath = $SPLUNK_DB/web-uf_index/thaweddb

[collectd_index]
coldPath = $SPLUNK_DB/collectd_index/colddb
datatype = metric
enableDataIntegrityControl = 0
enableTsidxReduction = 0
homePath = $SPLUNK_DB/collectd_index/db
maxTotalDataSizeMB = 512000
thawedPath = $SPLUNK_DB/collectd_index/thaweddb

| msearch index="collectd_index"
| mcatalog values(metric_name) WHERE index="collectd_index"
| mstats avg(_value) WHERE index="collectd_index" metric_name=cpu.idle.valu
| mstats avg(_value) WHERE index="collectd_index" metric_name=cpu.idle.value span=5s
| mstats avg(_value) where index="collectd_index" metric_name=cpu.idle.value span=10m

| mstats avg(_value) where index="collectd_index" metric_name="cpu.*" span=10m  prestats=true | stats avg(_value) by metric_name

| mstats avg(_value) WHERE index="collectd_index" metric_name=memory.free.value span=1d

msearch index="collectd_metrics"  =>>> search msearch index="collectd_metrics"
(DEPRECATED) | msearch index="collectd_metrics"
| mpreview index="collectd_metrics"

| mpreview index="collectd_index"

| mstats avg(_value) WHERE index="collectd_index" metric_name=cpu.idle.value span=5s

find / -type f -exec grep -l abcd {} \;

index="tutorial_data" sourcetype="access*" action="purchase" status=200 | stats sum(price) AS "total" | fieldformat total="$" + total + " USD”

index="tutorial_data" sourcetype="access*" action="purchase" status=200 | stats sum(productPrice) AS "total" | fieldformat total="$" + total + " USD”


ROUNDING

index="tutorial_data" sourcetype="access*" action=purchase status=200 | stats sum(price) AS "total" | eval total=round(total, 0)  | fieldformat total="$" + total + " USD"

Panel 2 (top, right; Line Chart; connected to time picker)
index="tutorial_data" sourcetype=access_* action=purchase status=200 | timechart span=1d sum(productPrice) as "Sales"
NEW OCT 2021
index="tutorial_data" sourcetype=access_* action=purchase status=200 | timechart span=1d sum(price) as “Sales"

index="tutorial_data" sourcetype="access*" action=purchase status=200 | timechart span=1d sum(price) as "Sales"

Panel 3 (bottom, left; Pie Chart; connected to time picker)
index="tutorial_data" sourcetype="access*" action="purchase" status=200 | top productName showperc=0

Panel 4 (bottom, right; Table; connected to time picker)
index="tutorial_data" sourcetype="access*" productId!="" action=purchase | rename _time as "Date", clientip as "ClientIP" | table Date, productId, productName, ClientIP
index="tutorial_data" sourcetype="access_*" action="purchase" status=200 productId != "" | rename _time as Date, clientip as clientIP | fieldformat Date = strftime(Date, "%b-%d-%Y %H:%M:%S") | table Date, productId, productName, clientIP

CLIENT IP FIELD EXTRACTION
^(?P<clientIP>\d+\.\d+\.\d+\d+)

USE THIS
index="tutorial_data" sourcetype="access_*" action="purchase" status=200 productId != "" | rename _time as Date, clientip as clientIP | fieldformat Date = strftime(Date, "%b-%d-%Y %H:%M:%S") | table Date, productId, productName, clientIP

index="tutorial_data" sourcetype="access_*" action="purchase" status=200 productId != "" productName="$productNameToken$" | rename _time as Date, clientip as clientIP | fieldformat Date = strftime(Date, "%b-%d-%Y %H:%M:%S") | table Date, productId, productName, clientIP

<init>
  <set token="productNameToken">*</set>
</init>

<form version="1.1" theme="dark">
   <init>
        <set token="productName">*</set>
    </init>
  <label>My Sales Dashboard V1.1</label>
  <fieldset submitButton="false" autoRun="true">
    <input type="time" token="time_picker1" searchWhenChanged="true">
      <label>Period</label>
      <default>
        <earliest>-7d@h</earliest>
        <latest>now</latest>
      </default>
    </input>
  </fieldset>
  <row>
    <panel>
      <title>Top 10 Products</title>
      <chart>
        <title>$productName$</title>
        <search>
          <query>index="tutorial_data" sourcetype="access*" action="purchase" status=200 | top productName showperc=0 | rename productName as "Product Name"</query>
          <earliest>$time_picker1.earliest$</earliest>
          <latest>$time_picker1.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">pie</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
        <drilldown>
          <set token="productName">$click.value$</set>
        </drilldown>
      </chart>
    </panel>
    <panel>
      <title>Total Sales</title>
      <single>
        <search>
          <query>index="tutorial_data" sourcetype="access*" action=purchase status=200 productName="$productName$" | stats sum(price) AS "total"  | eval total=round(total, 0)  | fieldformat total="$$" + total + " USD"</query>
          <earliest>$time_picker1.earliest$</earliest>
          <latest>$time_picker1.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
      </single>
    </panel>
  </row>
  <row>
    <panel>
      <title>Sales by Product</title>
      <table>
        <search>
          <query>index="tutorial_data" sourcetype="access*" status &lt; 299 | stats sum(price) as Sales by productName | fieldformat Sales="$$" + tostring(Sales, "commas") + "USD" | sort - Sales | rename productName as "Product Name"</query>
          <earliest>$time_picker1.earliest$</earliest>
          <latest>$time_picker1.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
    </panel>
    <panel>
      <title>Sales Data</title>
      <table>
        <search>
          <query>index="tutorial_data" sourcetype="access_*" action="purchase" status=200 productId != "" | rename _time as Date, clientip as clientIP | fieldformat Date = strftime(Date, "%b-%d-%Y %H:%M:%S") | table Date, productId, productName, clientIP</query>
          <earliest>$time_picker1.earliest$</earliest>
          <latest>$time_picker1.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
    </panel>
  </row>
</form>