efk使用心得之自动化工具编写

#!/bin/bash
URL=$1
USERNAME=$2
PASSWORD=$3
VARI_ABLE=$4
FLAG=$5
CURL_LOG='tempfile'
API_NAME=''
JSON_FILENAME="${VARI_ABLE}"


function pipeline(){
API_NAME="_ingest/pipeline/${VARI_ABLE}"
_execute
}


function template(){
API_NAME="_template/${VARI_ABLE}"
    _execute
}


function policy(){
API_NAME="_ilm/policy/${VARI_ABLE}"
JSON_FILENAME='policy'
   _execute
}


function _execute(){
httpcode=$(curl -k --cacert cert/elastic-ca.pem -H'Content-Type: application/json' -H'WWW-Authenticate:Basic realm=security charset=UTF-8' -s -u ${USERNAME}:${PASSWORD} -XPUT ${URL}/${API_NAME} -d "@template/${JSON_FILENAME}.json" -w %{http_code} -o "$CURL_LOG")
#echo "$httpcode"
if [ "$httpcode" = "200" ]; then
rm "$CURL_LOG"
exit 1
else
cat "$CURL_LOG" > dev/stderr
rm "$CURL_LOG"
fi
}


if [ "$FLAG" = "pipeline" ]; then
pipeline
elif [ "$FLAG" = "template" ]; then
template
elif [ "$FLAG" = "policy" ]; then
policy
else
echo "invalid parameter"
fi

#!/bin/bash


URL=$1
USERNAME=$2
PASSWORD=$3
INDEXPATTERN=$4
FLAG=$5


function create(){
curl -k --cacert cert/elastic-ca.pem -H'Content-Type: application/json' -H'kbn-xsrf: true' -s -u ${USERNAME}:${PASSWORD} -XPOST ${URL}/api/saved_objects/index-pattern/${INDEXPATTERN}?overwrite=true -d'
{
"attributes":
{
"title": "'"$INDEXPATTERN"'",
"timeFieldName": "@timestamp"
}
}
'
}


function delete(){
curl -k --cacert cert/elastic-ca.pem -H'Content-Type: application/json' -H'kbn-xsrf: true' -s -u ${USERNAME}:${PASSWORD} -XDELETE ${URL}/api/saved_objects/index-pattern/${INDEXPATTERN}
}


if [ "$FLAG" = "create" ]; then
create
elif [ "$FLAG" = "delete" ]; then
delete
else
echo "invalid parameter"
fi

public static async Task CreateIndexAsync(IndexDto dto)
{
var stuffix = dto.Environment + "-" + dto.IndexName;
//别名
var aliasName = stuffix + "-alias";
//策略
var policyId = stuffix + "-policy";
//模板
var templateName = stuffix + "-template";
//索引名
var indexName = dto.RollUpType == 0 ? "<" + stuffix + "-{now/d}-1>" : stuffix + "-000001";
// kibana index-pattern
var kibanaIndexPattern = stuffix + "-*";


//1.先创建策略
var getlifecycleRequest = new GetLifecycleRequest(policyId);
var lifeCycleGetRespOnse= await EsClient.Client.IndexLifecycleManagement.GetLifecycleAsync(getlifecycleRequest);
if (lifeCycleGetResponse.ApiCall.HttpStatusCode == 404)
  {
var result = await EsScriptClient.ExecutePolicyAsync(new PolicyDto(policyId));
if (!result.Item1)
{
LogCenterManager.Log(LogLevel.Error, $"索引生命周期策略 { policyId} 创建失败，返回响应值：{result.Item2}", nameof(CreateIndexAsync));
return;
      }
      LogCenterManager.Log(LogLevel.Info, $"索引生命周期策略 {policyId} 创建成功", nameof(CreateIndexAsync));     
  }
//2.先创建模板，优先级为1，高于通用模板
var indexExistsTemplateRequest = new IndexTemplateExistsRequest(templateName);
if (!(await EsClient.Client.Indices.TemplateExistsAsync(indexExistsTemplateRequest)).Exists)
{
var putIndexTemplateRequest = new PutIndexTemplateRequest(templateName)
{
          Order = 1,
IndexPatterns = new[] { $"{stuffix}-*" },
Settings = new IndexSettings(new Dictionary<string, object>
{
{"lifecycle.name", policyId},
{"lifecycle.rollover_alias", aliasName}
})
};
var putTemplateRespOnse= await EsClient.Client.Indices.PutTemplateAsync(putIndexTemplateRequest);
if (!putTemplateResponse.IsValid)
{
LogCenterManager.Log(LogLevel.Error, $"索引模板 {templateName} 创建失败", nameof(CreateIndexAsync), putTemplateResponse.OriginalException);
return;
        }
LogCenterManager.Log(LogLevel.Info, $"索引模板 {templateName} 创建成功，应用 lifecycle.name：{policyId}，lifecycle.rollover_alias：{aliasName}", nameof(CreateIndexAsync));
    }
//3.创建初始写索引
if (!(await EsClient.Client.Indices.ExistsAsync(indexName)).Exists)
{
     var createIndexResponse = await EsClient.Client.Indices.CreateAsync(indexName, request =>
{
return request.Aliases(aliases => aliases.Alias(aliasName, alias => alias.IsWriteIndex()));
});
if (!createIndexResponse.IsValid)
{
LogCenterManager.Log(LogLevel.Error, $"索引 {indexName} 创建失败", nameof(CreateIndexAsync), createIndexResponse.OriginalException);
return;
}
    }
//4.创建kibana index-pattern
var createKibanaIndexPatternRespOnse= await KibanaClient.CreateIndexPatternAsync(new IndexPatternDto(kibanaIndexPattern, dto.Flag));
if (!createKibanaIndexPatternResponse.Item1)
{
LogCenterManager.Log(LogLevel.Error, $"kibana index-pattern {kibanaIndexPattern} 失败，返回响应值：{createKibanaIndexPatternResponse.Item2}", nameof(CreateIndexAsync));
return;
  }
//5.生成 filebeat 配置文件
var filebeatInputFile = Path.Combine(Consts.FilebeatOutputFilePath, dto.IndexName + ".yml");
var cOntents= await File.ReadAllTextAsync(Consts.FilebeatFilePath, Encoding.UTF8);
cOntents= contents
.Replace("${current_path}", filebeatInputFile)
.Replace("${create_time}", DateTime.Now.ToString("yyyy-MM-dd"))
.Replace("${log_type}", dto.IndexName)
.Replace("${logName}", dto.IndexName.Replace("-", "."))
.Replace("${source}", dto.IndexName.Split('-')[0]);


await File.WriteAllTextAsync(filebeatInputFile, contents);


LogCenterManager.Log(LogLevel.Info, Environment.NewLine + contents + Environment.NewLine + Environment.NewLine + $"索引 {dto.IndexName} 创建成功，{dto.IndexName}.yml 文件可以试着手动进行二次确认，内容如上，没问题的话，可执行 -push 命令往服务器推送", nameof(CreateIndexAsync));
}

# 自动生成
# ${current_path} ${create_time}
- type: log
enabled: true
paths:
    - /var/logs/${logName}/*.log
fields_under_root: true
fields:
log_type: ${log_type}
     source: ${source}
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: false
  json.message_key: 'message'
multiline.pattern: '^\{"date":"\d{4}-\d{1,2}-\d{1,2}'
multiline.negate: true
  multiline.match: after

#!/bin/bash
USERNAME=$1
SOURCENAME=$2
TARGETNAME=$3
TARGETFILE="/usr/local/filebeat/input.d"
ssh ${USERNAME} "mkdir ${TARGETFILE}"
scp template/output/${SOURCENAME}.yml ${USERNAME}:${TARGETFILE}/${TARGETNAME}.yml
r=$?
if [ "$r" = 0 ]; then
echo "success"
else
echo "error:$r"
fi