diff --git a/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.png b/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.png new file mode 100644 index 0000000000000000000000000000000000000000..eae84627576ac6d52326211cae93a6af0bb3328b Binary files /dev/null and b/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.png differ diff --git a/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.xml b/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.xml new file mode 100644 index 0000000000000000000000000000000000000000..9244eed435d8c3fecbe4b261c830bd30a2905b3a --- /dev/null +++ b/docs/understanding/images/logs_data_flow_diagram.app.diagrams.net.xml @@ -0,0 +1 @@ +<mxfile host="app.diagrams.net" modified="2021-04-01T03:29:15.169Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" etag="D8lnvlAFDhG5Sz-aWDvn" version="14.5.3" type="device"><diagram id="w6dHBhk2DzmvdH_unnzh" name="Page-1">1VpXk6M+Ev80U3f3MC6iwY/GBueAM37ZkkEEGxBDcPr0JxGc8OzM3nr/teeqGUuN1Gp1/1rqbvzGNrxjKwSBPUAGdN8Yyji+sc03hqEZgcdfhHLKKAxdUKzQMfJRV8LUOcOcSOXUxDFgdDcwRsiNneCeqCPfh3p8RwNhiA73w0zk3q8aACtfkboSpjpwYWnY0jFiO6OK/M3oNnQsu1iZpvInHigG54TIBgY63JBY+Y1thAjFWcs7NqBLtFfoJZunfPL0IlgI/fg7E36sko+tfBju99awu0hakj+uv+dc9sBN8g3nwsanQgMhSnwDEibUGysdbCeG0wDo5OkBGx3T7NhzcY/GzbJQxQowjOHxhpQL2YLIg3F4wkPyp+80xWZzTgWBr+WgOdyYoJqPsm/UzzD5QJCb3bqwv2oGN3Ll/IKixCeKqrp4WWmDGxZp9JwN8EFBxqtcnlyGho+UKAB+Qds5PpagTrwFBi46eUSHV263Q7/HsBCIciLiNjYkDhAiP4Z4IYZqzTupR4RkXw48OL51GeYiK0phgEJIxjr+5ZHsgih29CkEoW7jfsNNohiGn4tUkv1TgNFfA8wFG+hKQN9Z6bQGcrH8bNNHPh4vAdexfNx1oUmWIqBzsC/Xc3KMgkxFOt5rPx3T5F6DWo6tPYCW44QSaBmWKYNWoP4QZtmvMZsbM8qNWQUe0ZC/iYKfouwW3FfYwntm1L8bGBnIw60JjFAS6vA/ZZazFFT51HcUwBBg0L1TxCDoAXpQ311G4K4PPBilSMHKSyF9Aa+FDfwOd0+E0guZwlym+yXyubfcK7izROEuvXosiMeFaZOgOEinJkEqwc+cI1sDOxtZGpn4X+YFIIZm4kYwJu5GX/xRnqZXSD7XR+QOLDb5MOqARbuOqjwz2l/lcDR1Jc3I84zyCh+khdLFQZV8kOWosg/S7J9yQuZrJ1TwIz/eOPGvOpwBoId8DJ7yxBue2fEPMnCngYgTBAQxDfKMDEgiWAyxURSPSeiCRUZu4hGf8LDh4wLyRQdLtAeYiZIxRVcXIizwVy/ZwNCHccqaYPMC4Hy52E4FKxgGyLmuoiPXJeHcw32UOg5IlY4PB0IJE9/P/DWVAFzWIpuLULFDPfXOCN6fElGx3J3bPnGh+uN+bjRPQT9KQnjx3xvF5wfYEzkhRvUpl7Xylzko9+iftde4J11cdD91T6YIQe/iOuoF7ik3Tn3rw/O4d1v6sVaG67W++U4AjAP3gDQdL80VpPS7HgVZukHOLlB0TOdIzHZRc5/YZowiJ3YQUfcGxeTy+cwO+QpNO45JelMnW2OUEBwqlhPbyQY7Tqin8Vtc0cklpmRouzTeU9wpIq1zNcaoGcIGQAANztBpUxdqpkhVOY7jWYrlBJrdkMjMx3i6nV6J9tZr7M1ypeNYrJaPY7ZSBO23Fr8hv9zmz47kv8zmYA9iEEbUp4ZP8F9VqOKwEwujEA2lPpIHcPvXBbYCVfJakStbUaw889sb8sut+Cy6/cKKX5rok7P0jWHlqoL96lMrltBQBkxhaANbFts56zJKQC6FhrOQRpMD1WtZqI4/w+nclucWbjUHpH9u1DXyXUWz+gehdmRXVhcTJMGVhplQ0UybHNrimOrinurX6tx86KCPpbxT6YZHd9hjvf+xaHFUt2E5/Z4jWpA+Lo5qa6q6fbk72/amp9Zg0oqbvfMkkam+vBXqW28XMtXYYE8Nh22P2SY7Zs0R5Z3pw7hvbqCO1/KN9RsjCWfcHA4w3qS1qosHm5iUb567SqK16DM3M1IjN3c1cBjLdM0TOp21MxB7CEewUk+qDY9jjAhpB5vS0NSAykXNqHloB2fpXKf7anPFQK4z3ozOI80SWy3QYrrmrL8zW21qL/Vl307AlGngiKKJccso89pQhac2UDHPjR1bfV04rHC7tepqwWChtil/N+aD2bLX5+dnGsLm2RAT9bCtea6qWB1np8zd7cB2NxavqVSjSRCqd02dPfdDDylIdT7kFj0S4oMc+o29rQO23fb1xcw7hSo+txS2OwDiBqOhO62P9ibfGe3UTlSVdr3lZjMNGLehbJaaMGttqZMyPovJqFeXOxaznRyFszAAHeF06oPkuJTcVotaSe6U1ux5nPTOPY05tPW1J+9WfWrdnJui19/jvSl0TVyHe1Ga19ud5u64M8mGnfXKEkc91VyIrs+v7U1c261Mz6o6S/WjO2F2SnvkDkRVHo+FGNtI0oa1Xn2nW1Dexd5cm1fNxqx7nG+1DgcdvLGap2/p3pLlxCO/tkR0mBxEoznxx635+MCKdgvZ1eW0GXhdGq4oobP14NGX6sOuOtgpi3HCaB9Bz9RqQD7LItXdWWBiu2G81E7q8eRNFSG0ln1ZNEl8OZ0k6mZfX3radKxKg5OmHK1xZ4xgr+7OxQFF9YIPk5W2A8ZdUL1+M1ZZn/mALQDW+klxjuwgUaYNnjP2Huoiro9W3EfUmvL2bDal+I7hnp0J0L0FCUyUWWvIGMbB6rRVSwulj95ysO4Ng5XdDYKttKA3iToN230IRnw3W2mC1TXXtUgENXk9mgm8tem6Hw04a1PCXmY62vEcqfvmQSRuAapJA7SaUKjbthDzW5+Rw+7C6tMfW/+scmzXn9KO6Hf2nITmXaF3xP+lgXiUx/RpZcfzxaZWGw02MiJYrNtGG3/JK2Y5XJsHztvRmjEZaIvpXJOqcwoMvG1PcaWIjO1FVds/9nnPllqGumMSS2k16Lrd7C15qptQ59FU2En46JRk8WPQVvsjRRzutVE1GIGZKw0icvLUp/PFaNLjG1qnQ07El9wtVZYvFU3Y8t3yJF97RbomUtz7YkyHqx+75k7fz3Xvh/Q0HnxI10CIw6QQpBtJI/zfL/jds/n1St/MhiG8zfRwsLDHdnHSZOOAY4i0/HCXwhzyOXfpFEhJ8EjurbTwl6ZHKImDJE/GnPAhd4pinNWB0MjGVdLU8FHmziUNTaIEL3i6ZGE6iGDaAXGqiseZ749iZzlVLkAmbgy9AGGDOCnfQ+jEOFAqxLvPUxUSRQHHh2GaxTkh3idKLZmlZSCt2pRS18rXpTEiaP2tyLFDGF/UHyDX0VOwhJAY5VqMzXdhhmlBKonyCixK8+wkFSXLeHMx/pUJHZFSVFGaeiZHVlN7oikDujCzK7b+NWO/yVOvI9IKk4f28GJp4KO8+HVXc3pacvp/KP1WHxPdG8IkP2uYl73U+Fbyy4kVofok/6X/VHmq+nUUDX2jTt6k4Z7ugihy9Hu7YK2Ep1Ue5aYdjXQqHC8UhObx9nHzdNsbw9DBe4FhTvxU2zghsmD8dakNGsUrvU9scqNw/sndUtBC6ILY2d+/CHxmgHyFcV7AKgDBP1qcq9XumWQF6Hze1ZDfYMXUHgCRKafEKsXEZevfgkncVn8kp7lpTs4x0EBAGQvrSco8pMtAwYpPU6t7fBSeqWODEiuXfNNzDIPwkEIYOWewSfkRKKQlwXSDvPTGPw86forqR/+8vCfOF3m7fRV7h5HfNH6hsFNhwHsGyDTT2uFvGerpzoUX+vM7VaGo6oNT08xLnfovcVa29vjOmRMfbPZdZ+VovlKrUtcP9xXnP+y75ZB2+AQUf43vFgj+33238EGM1lIJ8iVo4SoPRqXZivCPuTj9jZr1jY/n8ZEBIvsSad1YmNDHAEfMoZ9SGCqt/8ch2l1+8XJ/y1f4uyPhq+Pg6MTZNC47S3BfK3ji9nUa6dzOejxDtokX5D8Qqv7sSMlf6375XvzLeEL8q44onq0+YPnyJvNXjyiG+wfjiQUd2MpA/tGT1nxbqNFRozZ5diahGKbpGlsOLH4pTTAd170p3Bo8FI0rpG+eiMyGrV6R9OTAKYHr83dbYinYY8vhffUJOpgXRPZPVfwkZLuouHxg/P0qrvKlYtGT10kvUjF57Xb5ZV4G+usPHFn5vw==</diagram></mxfile> \ No newline at end of file diff --git a/docs/understanding/logs_data_flow_diagram.md b/docs/understanding/logs_data_flow_diagram.md new file mode 100644 index 0000000000000000000000000000000000000000..a4a3cefd6db2b400955a17fb26349ad5dd1f2665 --- /dev/null +++ b/docs/understanding/logs_data_flow_diagram.md @@ -0,0 +1,14 @@ +# Goals of this Diagram: +* Help new users understand the data flow of pod logs + +# Kubernetes Pod Logs Data Flow Diagram: + + +| Line Number | Protocol | Port | Description | +| --- | --- | --- | --- | +| N1 | Volume Mount | NA | Fluentbit reads pod logs from a host node volume mount | +| N2 | HTTPS | TCP:9200 | Fluentbit sends logs to Elastic Search over the URL:Â https://logging-ek-es-http:9200 (This URL is only exposed over the Kubernetes Inner Cluster Network, and because Fluentbit and ElasticSearch have Istio Envoy Proxy sidecar containers the network traffic is protected by the service mesh.) | + +## Notes: +1. The fluentbit log shipper is configured to send pod logs to the ElasticSearch Cluster in the logstash data format. Logstash_Format On +2. By default: The log index logstash-%Y.%m.%d will create a new log index everyday, because %d will increment by one everyday. There are no default Index Lifecycle Management Policies that are created or applied to these indexes. It is recommended that customers create a Index Lifecycle policy to prevent disk space from filling up. (Example: Archive to s3 and then delete from PVC logs older than N days.)