分类目录归档:未分类

logstash 解析nginx error日志

input {
#    beats {
#        host => "0.0.0.0"
#        port => 5400
#    }

stdin { }
}

filter {
 grok {
   patterns_dir => "/etc/logstash/patterns"
   #match => [ "message" , "%{NGINXACCESS}"]
   match => [ "message" , "%{DATA:timestr} \[%{DATA:error_level}\] (?<nginx_message>(.|\r|\n)*)(?:, client: %{IPORHOST:clientip})(?:, server: %{IPORHOST:nginx_server})(?:, request: \"%{DATA:nginx_request}\")?(?:, upstream: \"%{DATA:nginx_upstream}\")?(?:, host: \"%{DATA:nginx_host}\")?(?:, referrer: \"%{DATA:nginx_referrer}\")?"]
 }

      if [http_x_forwarded_for] == "-" or [http_x_forwarded_for] == "null" {
         mutate {
            update => { "http_x_forwarded_for" => "" }
         }
      }

      if [referer] == "-" or [referer] == "null" {
         mutate {
            update => { "referer" => "" }
         }
      }

    geoip {
      source => "clientip"
    }

    useragent {
      source => "agent"
      target => "agent_fields"
    }

  date {
    match => [ "timestr", "yyyy/MM/dd HH:mm:ss" ]
    timezone => "Asia/Shanghai"
    #target => "newtimestr"
    #locale => "en"
  }

        ruby {
                code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y%m%d'))"
        }


}

output {
 elasticsearch {
   hosts => ["127.0.0.1:9200"]
   #index => "tek-%{+YYYY.MM.dd}"
   index => "tek-%{index_day}"
   document_type => "nginx_logs"
   template_name => "ta"
 }
 stdout { codec => rubydebug }
}

防止一天的日志分在了两个index中

        ruby {
                code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y%m%d'))"
        }

kvm网络隔离禁止虚拟机之间通信

nwfilter xml文件默认路径: /etc/libvirt/nwfilter/

cat deny-test.xml
<filter name='deny-test' chain='ipv4' priority='-700'>
  <uuid>fce8ae34-e69e-83bf-262e-30786c1f8079</uuid>
  <rule action='drop' direction='out' priority='200'>
    <ip srcipaddr='172.21.13.102' dstipaddr='172.21.13.107' dstipmask='32'/>
  </rule>
</filter>


virsh nwfilter-define deny-test.xml
virsh nwfilter-list #确认是否添加成功


virsh edit xxx

<interface type='bridge'>
      <mac address='52:54:00:7c:17:86'/>
      <source bridge='br0'/>
      <model type='virtio'/>
      <filterref filter='deny-test'/> #add
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>

重启虚拟机

宿主机上执行:

ebtables -t nat -L


确认规则有没有添加成功

Bridge table: nat

Bridge chain: PREROUTING, entries: 1, policy: ACCEPT
-i vnet46 -j libvirt-I-vnet46

Bridge chain: OUTPUT, entries: 0, policy: ACCEPT

Bridge chain: POSTROUTING, entries: 0, policy: ACCEPT

Bridge chain: libvirt-I-vnet46, entries: 1, policy: ACCEPT
-p IPv4 -j I-vnet46-ipv4

Bridge chain: I-vnet46-ipv4, entries: 1, policy: ACCEPT
-p IPv4 --ip-src 172.21.13.102 --ip-dst 172.21.13.107 -j DROP 

简单 SHELL 

#!/bin/bash
# usage ./1.sh 172.21.13.102 deny-test 
tmpxml=$(mktemp /tmp/ifcfg.XXX)
macaddr="$(virsh domiflist $1 | awk "/bridge\s/ {print \$NF}")"
if [ -z "$macaddr" ]; then 
    echo "vm not exist" 
    exit 2
fi
if [ -z "$2" ]; then 
    echo "nwfilter name is null" 
    exit 2
fi

cat > "$tmpxml" <<EOF
<interface type='bridge'>
    <mac address='$macaddr'/>
    <source bridge='br0'/>
    <model type='virtio'/>
    <filterref filter='$2'/>
    <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
EOF
virsh update-device "$1" "$tmpxml" --live --persistent --config
rm "$tmpxml"

判断一个IP是否能ping通

package main

import (
	"time"
	"net"
	"fmt"
)

func isping(ip string) (bool) {
	recvBuf1 := make([]byte, 2048)
	payload:=[]byte{0x08,0x00,0x4d,0x4b,0x00,0x01,0x00,0x10,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69}
	Time, _ := time.ParseDuration("3s")
	conn, err := net.DialTimeout("ip4:icmp", ip,Time)
	if err !=nil {
		fmt.Println("bibi")
		return false
	}
	_,err=conn.Write(payload)
	if err !=nil {
		return false
	}
	conn.SetReadDeadline(time.Now().Add(time.Second * 2))
	num, err := conn.Read(recvBuf1[0:])
	if err !=nil {
		//check 80 3389 443 22 port
		Timetcp, _ := time.ParseDuration("1s")
		conn1, err := net.DialTimeout("tcp", ip+":80",Timetcp)
		if err == nil {
			defer conn1.Close()
			return true
		}

		conn2, err := net.DialTimeout("tcp", ip+":443",Timetcp)
		if err == nil {
			defer conn2.Close()
			return true
		}

		conn3, err := net.DialTimeout("tcp", ip+":3389",Timetcp)
		if err == nil {
			defer conn3.Close()
			return true
		}

		conn4, err := net.DialTimeout("tcp", ip+":22",Timetcp)
		if err == nil {
			defer conn4.Close()
			return true
		}

		return false
	}
	conn.SetReadDeadline(time.Time{})
	if string(recvBuf1[0:num]) !="" {
		return  true
	}
	return false

}

func main()  {
	ip := "172.8.47.213"
	fmt.Println(isping(ip))
}

遍历中国所有IP地址

#main

package main

import (
	"libmy"
	"fmt"
)

var iplistchan chan string
var iplistsuccess chan string
var hostsuccess chan string

func insertintochan(iplist []string,iplistchan chan string)  {
	for _,ipcidr := range iplist {
		hosts, _ := libmy.Hosts(ipcidr)
		for _, ip := range hosts {
			iplistchan <- ip
		}
	}
	close(iplistchan)
	iplistsuccess <- "good"

}

func worker(iplistchan chan string)  {
	for {
		if elem, ok := <-iplistchan; ok {
			fmt.Println(elem)
		} else {
			break
		}
	}
	hostsuccess <- "good"
}


func main()  {
	iplistchan=make(chan string ,1000)
	iplistsuccess=make(chan string)
	hostsuccess=make(chan string)
	iplist:=libmy.ReadList("cn1.zone")
	go insertintochan(iplist,iplistchan)

	for i:=0;i<13;i++ {
		go worker(iplistchan)
	}
	<-iplistsuccess
	for j:=0;j<13;j++ {
		<-hostsuccess
	}
}

#lib.go

package libmy

import "net"
import "os"
import "bufio"
import "fmt"
import "strings"

func Hosts(cidr string) ([]string, error) {
	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return nil, err
	}

	var ips []string
	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
		ips = append(ips, ip.String())
	}
	return ips[1 : len(ips)-1], nil
}

func inc(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

func ReadList(fileName string) ( [] string) {
	ipListFile, err := os.Open(fileName)
	if err != nil {
		fmt.Println("ERR::" + err.Error())
		os.Exit(1)
	}
	defer ipListFile.Close()
	ipList:=make([]string,0)
	scanner := bufio.NewScanner(ipListFile)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ipinfo := strings.TrimSpace(scanner.Text())
		ipList = append(ipList, ipinfo)
	}
	return ipList
}

通过cidr遍历IP地址

package main

import (
	"net"
	"fmt"
)

func hosts(cidr string) ([]string, error) {
ip, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
return nil, err
}

var ips []string
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
ips = append(ips, ip.String())
}
return ips[1 : len(ips)-1], nil
}

func inc(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

func main()  {
	hosts, _ := hosts("192.168.11.9/27")
	for _, ip := range hosts {
		 fmt.Println("sent: " + ip)
	}
}

检测一个端口是否为HTTPS

package main

import (
	"fmt"
	"time"
	"net"
	"strconv"
	"os"
)

func main(){
	t:=[]byte{0x16,0x03,0x01,0x00,0xb5,0x01,0x00,0x00,0xb1,0x03,0x03,0xb2,0xd3,0x4d,0xfd,0x63,0xbe,0x89,0xdb,0xe5,0x46,0xcc,0xaf,0x39,0x6e,0xba,0x63,0x63,0x75,0xce,0x30,0xda,0xe0,0x4f,0xab,0xa2,0x3e,0x50,0xea,0x41,0x20,0x10,0xc4,0x00,0x00,0x18,0xc0,0x2b,0xc0,0x2f,0xc0,0x2c,0xc0,0x30,0xc0,0x13,0xc0,0x14,0x00,0x9c,0x00,0x9d,0x00,0x2f,0x00,0x35,0x00,0x0a,0x00,0xff,0x01,0x00,0x00,0x70,0x00,0x00,0x00,0x15,0x00,0x13,0x00,0x00,0x10,0x77,0x77,0x77,0x2e,0x73,0x6f,0x2d,0x63,0x6f,0x6f,0x6c,0x73,0x2e,0x63,0x6f,0x6d,0x00,0x0b,0x00,0x04,0x03,0x00,0x01,0x02,0x00,0x0a,0x00,0x06,0x00,0x04,0x00,0x17,0x00,0x18,0x00,0x23,0x00,0x00,0x00,0x0d,0x00,0x20,0x00,0x1e,0x06,0x01,0x06,0x02,0x06,0x03,0x05,0x01,0x05,0x02,0x05,0x03,0x04,0x01,0x04,0x02,0x04,0x03,0x03,0x01,0x03,0x02,0x03,0x03,0x02,0x01,0x02,0x02,0x02,0x03,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x0f,0x00,0x01,0x01,0x00,0x10,0x00,0x0b,0x00,0x09,0x08,0x68,0x74,0x74,0x70,0x2f,0x31,0x2e,0x31}

	Target:="115.239.210.27"
	port:=443
	Time, _ := time.ParseDuration("1s")
	conn, err := net.DialTimeout("tcp", Target+":"+strconv.Itoa(port), Time )

	if err != nil {
		fmt.Println("ERR::" + strconv.Itoa(port) + ">" + err.Error())
		os.Exit(1)
	}
	conn.Write(t)
	recvBuf := make([]byte, 2048)
	conn.SetReadDeadline(time.Now().Add(time.Second * 2))
	_, err = conn.Read(recvBuf[:])
	conn.SetReadDeadline(time.Time{})
	fmt.Println("tlsinfo:")
	fmt.Println( string(recvBuf[:]))
	if string(recvBuf[0:4]) == string([] byte {22,3,3,0}) {
		fmt.Println("this is tls ^_^")
	}else{
		fmt.Println("this is not tls")
	}
	conn.Close()
}