es使用笔记

发表于2022年8月21日由midas

记录es常用命令

#请求es的4种方法
curl -XPOST -u ${es_username}:${es_password} -H 'Content-Type: application/json' http://$ip:9200/$ -d ''
curl -XPUT -u ${es_username}:${es_password} -H 'Content-Type: application/json' http://$ip:9200/$ -d ''
curl -XGET -u ${es_username}:${es_password} -H 'Content-Type: application/json' http://$ip:9200/$
curl -XDELETE -u ${es_username}:${es_password} -H 'Content-Type: application/json' http://$ip:9200/$


#设置es支持批量删除索引
PUT _cluster/settings
{"persistent": {"action.destructive_requires_name":"false"}}

#打开关闭长期不用的索引，提高性能
POST xxxxx*/_close
POST xxxxx*/_open


#注册一个本地文件系统或nfs的快照仓库
#如果是集群需要所有的节点都可以访问这个目录
curl -XPUT 'http://localhost:9200/_snapshot/my_backup' -H "Content-Type: application/json"  -d 
'{
    "type": "fs",
    "settings": {
        "location": "/data/snap_data",
        "chunk_size": "500mb",
        "compress": true
    }
}'

#注册一个基于腾讯云cos的仓库
PUT _snapshot/cos_backup
{
    "type": "cos",
    "settings": {
        "app_id": "xxxxxxxxx",
        "access_key_id": "xxxxxxxxxxxxxxxxxxx",
        "access_key_secret": "xxxxxxxxxxxxxxxxxx",
        "bucket": "es-bakcup",
        "region": "ap-xxx",
        "compress": true,
        "chunk_size": "500mb",
        "base_path": "/xxxx"
    }
}

#创建快照备份
PUT _snapshot/my_xxx_xxxx_backup/snapshot_20220816
{
    "indices": "xxx*"
}

curl -XPUT -u ${es_username}:${es_password} http://$ip:9200/_snapshot/my_xxxx_xxxx_backup/snapshot?wait_for_completion=true -d '{
  "indices": "xxxx*"
}

#查看所有快照
GET _snapshot/my_backup/_all

#创建单独大索引快照
PUT _snapshot/my_xxx_xxxx_backup/snapshot_20220816_bigindex
{
    "indices": "xxxxxxx"
}

#查看正在恢复的索引
GET _cat/recovery?v=true&h=i,s,t,ty,st,rep,snap,f,fp,b,bp

#在CDZ中恢复快照
POST _snapshot/my_xxxx_xxxx_backup/snapshot_20220817/_restore 
{
  "indices": "xxxxxxx*,xxxxxxxx*,xxxxxx*",
  "ignore_unavailable": true
}

##在CDZ中恢复单独大索引快照
POST _snapshot/my_xxxx_xxxx_backup/snapshot_20220816/_restore -d '
{
  "indices": "xxxxxxxxxxx",
  "ignore_unavailable": true
}'

#增加集群node最大shards
PUT /_cluster/settings
{
  "persistent" : {
      "cluster.max_shards_per_node": "2000"
  }
}

#查看系统中共有哪些template
GET _cat/templates
#查看所有template详情
GET _template
#查看单个template详情
GET _template/nginxaccess
#新增或更新template
PUT _template/nginxaccess
{
      "order" : 1,
      "index_patterns" : [
        "nginx-*"
      ],
      "settings" : {
        "index" : {
          "lifecycle" : {
            "name" : "nginx-180-days"
          },
          "routing" : {
            "allocation" : {
              "include" : {
                "_tier_preference" : "data_hot"
              }
            }
          },
          "refresh_interval" : "10s",
          "number_of_shards" : "5",
          "number_of_replicas" : "1"
        }
      },
      "mappings" : {
        "properties" : {
          "referer" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "agent" : {
            "properties" : {
              "os" : {
                "properties" : {
                  "name" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "version" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "full" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "name" : {
                "ignore_above" : 256,
                "type" : "keyword"
              },
              "device" : {
                "properties" : {
                  "name" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "version" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "auth" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "log" : {
            "type" : "object",
            "properties" : {
              "file" : {
                "type" : "object",
                "properties" : {
                  "path" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              }
            }
          },
          "ident" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "xforwardedFor-ip3" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "xforwardedFor-ip4" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "xforwardedFor-ip1" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "xforwardedFor-ip2" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "request_time" : {
            "type" : "float"
          },
          "host" : {
            "properties" : {
              "name" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "client_ip" : {
            "type" : "ip"
          },
          "event" : {
            "type" : "object",
            "properties" : {
              "original" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "user_agent" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "timestamp" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "geoip" : {
            "properties" : {
              "geo" : {
                "properties" : {
                  "region_iso_code" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "city_name" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "country_iso_code" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "timezone" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "country_name" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "continent_code" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  },
                  "location" : {
                    "properties" : {
                      "lon" : {
                        "type" : "float"
                      },
                      "lat" : {
                        "type" : "float"
                      }
                    }
                  },
                  "region_name" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "ip" : {
                "type" : "ip"
              },
              "coordinates" : {
                "type" : "geo_point"
              }
            }
          },
          "http_cookei" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "xforwardedFor" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "verb" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "http_version" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "url" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "tags" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "input" : {
            "type" : "object"
          },
          "@timestamp" : {
            "type" : "date"
          },
          "request_body" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "upstream_host" : {
            "ignore_above" : 256,
            "type" : "keyword"
          },
          "upstream_response_time" : {
            "type" : "float"
          },
          "bytes" : {
            "type" : "long"
          },
          "fields" : {
            "type" : "object",
            "properties" : {
              "appname" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "status" : {
            "ignore_above" : 256,
            "type" : "keyword"
          }
        }
      },
      "aliases" : { }
}

#logstash 条件判断语句
使用条件来决定filter和output处理特定的事件。logstash条件类似于编程语言。条件支持if、else if、else语句，可以嵌套。 
比较操作有： 
相等: ==, !=, <, >, <=, >= 
正则: =~(匹配正则), !~(不匹配正则) 
包含: in(包含), not in(不包含) 
布尔操作： 
and(与), or(或), nand(非与), xor(非或) 
一元运算符： 
!(取反) 
()(复合表达式), !()(对复合表达式结果取反) 


2、if[foo] in "String"在执行这样的语句是出现错误原因是没有找到叫做foo的field，无法把该字段值转化成String类型。所以最好要加field if exist判断。
判断字段是否存在，代码如下：


if ["foo"] {
  mutate {
    add_field => { "bar" => "%{foo}"}
  }
}

#elasticsearch and minio,官方文档:https://www.elastic.co/guide/en/elasticsearch/reference/master/repository-s3.html
#注册仓库
#并非特别安全，可在api里查到AK信息，需要在jvm中添加-Des.allow_insecure_settings=true并重启生效
PUT _snapshot/minio_backup
{
    "type": "s3",
    "settings": {
        "access_key": "xxxxxxxx",
        "secret_key": "xxxxxxxxxxxxxxx",
        "bucket": "es-backup",
        "compress": true,
        "chunk_size": "500mb",
        "base_path": "/prod",
        "protocol": "http",
        "endpoint": "10.x.x.x:9000"
    }
}

#官方推荐，将默认密码写入elasticsearch-keystore
#下面是交互添加AK信息的指令，添加完需要逐个重启
#docker exec -i elasticsearch bin/elasticsearch-keystore add s3.client.default.access_key
#docker exec -i elasticsearch bin/elasticsearch-keystore add s3.client.default.secret_key
PUT _snapshot/minio_backup
{
    "type": "s3",
    "settings": {
        "bucket": "es-backup",
        "compress": true,
        "chunk_size": "500mb",
        "base_path": "/prod",
        "protocol": "http",
        "disable_chunked_encoding":"true",
        "endpoint": "10.x.x.x:9000"
    }
}

#当node节点存在大量shard，重启优化，官方文档:https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html
#集群分片策略官方文档:https://www.elastic.co/guide/en/elasticsearch/reference/6.8/shards-allocation.html#_shard_allocation_settings
#ES滚动重启
#将es路由策略改为，并增加默认shard恢复限制（默认为2，非常的慢，具体值建议与node节点相同，更优化的值需要自己测试）
PUT _cluster/settings
{
  "persistent": {
    "cluster.routing.allocation.enable": "primaries",
    "cluster.routing.allocation.node_concurrent_incoming_recoveries": "12",
    "cluster.routing.allocation.node_concurrent_outgoing_recoveries": "12"
  }
}

#将内存中的数据刷新到磁盘
POST /_flush

#重启之后，等待主分片均为正常状态并且集群从red改为yellow，等待集群状态变为green即可进行下一个节点恢复，我12个节点默认2个为一组重启，重复到所有节点重启完毕即可
PUT _cluster/settings
{
  "persistent": {
    "cluster.routing.allocation.enable": null
  }
}

#elastic白金版切换成basic版
POST /_xpack/license/start_basic?acknowledge=true

此条目发表在ELK日志服务器分类目录。将固定链接加入收藏夹。

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注