1\uff0c\u9996\u5148\u683c\u5f0f\u5316nginx\u65e5\u5fd7\u683c\u5f0f<\/p>\n\n\n\n
log_format main '$time_iso8601 $remote_addr - $remote_user \"$request\" '\n '$status $body_bytes_sent $http_referer '\n '$http_x_forwarded_for $upstream_addr $upstream_response_time $request_time \"$http_cookie\" '\n '\"$http_user_agent\"';<\/pre>\n\n\n\n2\uff0c\u8f93\u5165ingest_pipeline\u81ea\u5b9a\u4e49\u6a21\u677f<\/p>\n\n\n\n
PUT \/_ingest\/pipeline\/nginxaccess\n{\n \"description\": \"nginx access log pipeline\",\n \"processors\": [{\n \"grok\": {\n \"field\": \"message\",\n \"patterns\": [\"%{TIMESTAMP_ISO8601:timestamp} (%{IPORHOST:client_ip}|-) (%{USER:ident}|-) (%{USER:auth}|-) \\\"(?:%{WORD:verb} %{NOTSPACE:request} (?:HTTP\/%{NUMBER:http_version})?|-)\\\" (?:%{NUMBER:status}|-) (?:%{NUMBER:bytes}|-) %{NOTSPACE:request_body} (%{IPORHOST:forwardedFor}|-) (%{URIHOST:upstream_host}|-) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE16FLOAT:request_time}) \\\"(%{DATA:http_cookei}|-)\\\" \\\"%{DATA:user_agent}\\\"\"]\n }\n },\n {\n \"date\": {\n \"field\": \"timestamp\",\n \"target_field\": \"@timestamp\",\n \"formats\": [\n \"yyyy-MM-dd HH:mm:ss.SSS\",\n \"ISO8601\"\n ],\n \"timezone\": \"Asia\/Shanghai\",\n \"ignore_failure\": true\n }\n },\n {\n \"date_index_name\": {\n \"field\": \"@timestamp\",\n \"index_name_prefix\": \"nginx-\",\n \"index_name_format\": \"yyyy.MM.dd\",\n \"date_rounding\": \"d\",\n \"timezone\": \"Asia\/Shanghai\",\n \"ignore_failure\": true\n }\n },\n {\n \"geoip\":{\n \"field\": \"client_ip\",\n \"target_field\": \"geoip\",\n \"ignore_missing\": true\n }\n },\n {\n \"user_agent\": {\n \"field\": \"user_agent\",\n \"target_field\": \"useragent\"\n } \n }\n ],\n \"on_failure\": [{\n \"set\": {\n \"field\": \"parse_err_message\",\n \"value\": \"{{ _ingest.on_failure_message }}\"\n }\n },\n {\n \"set\": {\n \"field\": \"_index\",\n \"value\": \"splog-parse-failed\"\n }\n }\n ]\n}<\/pre>\n\n\n\n3\uff0c\u5c06\u81ea\u5b9a\u4e49template\u5f55\u5165es<\/p>\n\n\n\n
PUT \/_template\/nginxaccess\n{\n \"order\": 0,\n \"version\": 1,\n \"index_patterns\": [\n \"nginx-*\"\n ],\n \"settings\": {\n \"index\": {\n \"refresh_interval\": \"30s\",\n \"number_of_shards\": \"1\",\n \"translog\": {\n \"sync_interval\": \"5s\",\n \"durability\": \"async\"\n },\n \"merge\": {\n \"scheduler\": {\n \"max_thread_count\": \"1\"\n }\n },\n \"number_of_replicas\": \"0\"\n }\n },\n \"mappings\": {\n \"dynamic_templates\": [{\n \"strings_as_keywords\": {\n \"match_mapping_type\": \"string\",\n \"mapping\": {\n \"type\": \"keyword\"\n }\n }\n }],\n \"properties\": {\n \"status\": {\n \"type\": \"long\"\n },\n \"bytes\": {\n \"type\": \"long\"\n },\n \"request_time\": {\n \"type\": \"float\"\n },\n \"upstream_response_time\": {\n \"type\": \"float\"\n },\n \"forwardedFor\": {\n \"type\": \"text\"\n },\n \"geoip\" : {\n \"properties\" : {\n \"location\": {\n \"type\": \"geo_point\"\n }\n }\n }\n }\n },\n \"aliases\": {}\n}<\/pre>\n\n\n\n4\uff0c\u5b89\u88c5\u5e76\u7f16\u8f91filebeat\u914d\u7f6e<\/p>\n\n\n\n
\u4f20\u9001\u95e8:https:\/\/www.elastic.co\/cn\/downloads\/past-releases#filebeat<\/a><\/p>\n\n\n\n
rpm -ivh filebeat–%{[observer.version]}-x86_64.rpm<\/p>\n\n\n\n
\u7f16\u8f91\u542f\u52a8\u811a\u672c<\/p>\n\n\n\n
vi \/usr\/lib\/systemd\/system\/filebeat.service\n[Unit]\nDescription=Filebeat sends log files to Logstash or directly to Elasticsearch.\nDocumentation=https:\/\/www.elastic.co\/products\/beats\/filebeat\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\n\nEnvironment=\"BEAT_LOG_OPTS=\"\nEnvironment=\"BEAT_CONFIG_OPTS=-c \/etc\/filebeat\/filebeat.yml\"\nEnvironment=\"BEAT_PATH_OPTS=-path.home \/usr\/share\/filebeat -path.config \/etc\/filebeat -path.data \/var\/lib\/filebeat -path.logs \/var\/log\/filebeat\"\nExecStart=\/usr\/share\/filebeat\/bin\/filebeat $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS\nRestart=always\n\n[Install]\nWantedBy=multi-user.target<\/pre>\n\n\n\n\u7f16\u8f91\u914d\u7f6e\u6587\u4ef6<\/p>\n\n\n\n
filebeat.inputs:\n- type: log\n # Change to true to enable this input configuration.\n enabled: true\n # Paths that should be crawled and fetched. Glob based paths.\n paths:\n - \/var\/log\/nginx\/*.log\n #- c:\\programdata\\elasticsearch\\logs\\*\n # Exclude lines. A list of regular expressions to match. It drops the lines that are\n # matching any regular expression from the list.\n #exclude_lines: ['^DBG']\n # Include lines. A list of regular expressions to match. It exports the lines that are\n # matching any regular expression from the list.\n #include_lines: ['^ERR', '^WARN']\n # Exclude files. A list of regular expressions to match. Filebeat drops the files that\n # are matching any regular expression from the list. By default, no files are dropped.\n #exclude_files: ['.gz$']\n # Optional additional fields. These fields can be freely picked\n # to add additional information to the crawled log files for filtering\n #fields:\n # level: debug\n # review: 1\n ### Multiline options\n # Multiline can be used for log messages spanning multiple lines. This is common\n # for Java Stack Traces or C-Line Continuation\n # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [\n #multiline.pattern: ^\\[\n multiline.pattern: '^\\d{4}-\\d{2}-\\d{2}(\\s\\d{2}:\\d{2}:\\d{2})?|^{|^(\\d{1,3}\\.){3}\\d{1,3}|^\\d{2}-\\d{2}\\s+'\n # Defines if the pattern set under pattern should be negated or not. Default is false.\n #multiline.negate: false\n multiline.negate: true\n # Match can be set to \"after\" or \"before\". It is used to define if lines should be append to a pattern\n # that was (not) matched before or after or as long as a pattern is not matched based on negate.\n # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash\n multiline.match: after\nfilebeat.config.modules:\n # Glob pattern for configuration loading\n path: ${path.config}\/modules.d\/*.yml\n # Set to true to enable config reloading\n reload.enabled: false\n # Period on which files under path should be checked for changes\n #reload.period: 10s\n #index.codec: best_compression\n #_source.enabled: false\nsetup.ilm.enabled: false\nsetup.kibana:\n # Kibana Host\n # Scheme and port can be left out and will be set to the default (http and 5601)\n # In case you specify and additional path, the scheme is required: http:\/\/localhost:5601\/path\n # IPv6 addresses should always be defined as: https:\/\/[2001:db8::1]:5601\n host: \"10.33.200.8:5601\"\n username: \"elastic\"\n password: \"xxxxxxxxxxxxxxxxx\"\n # Kibana Space ID\n # ID of the Kibana Space into which the dashboards should be loaded. By default,\n # the Default Space will be used.\n #space.id:\noutput.elasticsearch:\n # Array of hosts to connect to.\n hosts: [\"10.33.100.9:9200\",\"10.33.100.14:9200\",\"10.33.100.3:9200\"]\n indices:\n - index: \"nginx-%{+yyyy.MM.dd}\"\n # Protocol - either `http` (default) or `https`.\n #protocol: \"https\"\n # Authentication credentials - either API key or username\/password.\n #api_key: \"id:api_key\"\n username: \"elastic\"\n password: \"xxxxxxxxxxxxxxxx\"\n worker: 2\n bulk_max_size: 256\n pipeline: nginxaccess\n # The Logstash hosts\n #hosts: [\"localhost:5044\"]\n # Optional SSL. By default is off.\n # List of root certificates for HTTPS server verifications\n #ssl.certificate_authorities: [\"\/etc\/pki\/root\/ca.pem\"]\n # Certificate for SSL client authentication\n #ssl.certificate: \"\/etc\/pki\/client\/cert.pem\"\n # Client Certificate Key\n #ssl.key: \"\/etc\/pki\/client\/cert.key\"\nprocessors:\n - add_host_metadata: ~\n - add_cloud_metadata: ~\n - add_docker_metadata: ~\n - add_kubernetes_metadata: ~\nmonitoring.enabled: true\nmonitoring.elasticsearch:\n hosts: [\"10.33.100.9:9200\",\"10.33.100.14:9200\",\"10.33.100.3:9200\"]\n username: \"elastic\"\n password: \"xxxxxxxxxxxxxxxx\"<\/pre>\n\n\n\n\u542f\u52a8\u670d\u52a1<\/p>\n\n\n\n
systemctl enable filebeat\nsystemctl restart filebeat<\/pre>\n","protected":false},"excerpt":{"rendered":"1\uff0c\u9996\u5148\u683c\u5f0f\u5316nginx\u65e5\u5fd7\u683c\u5f0f 2\uff0c\u8f93\u5165ingest_pipeline\u81ea\u5b9a\u4e49\u6a21 … \u7ee7\u7eed\u9605\u8bfb