From a6a208c7d7ab9adb1d088d7aab438367876d506a Mon Sep 17 00:00:00 2001 From: zu1k Date: Sat, 18 Jul 2020 09:52:59 +0800 Subject: [PATCH] zx ipv6 test --- go.sum | 1 + internal/app/parse.go | 42 +++++++++++--- internal/iptools/ipparser.go | 16 ++++++ pkg/common/struct.go | 11 +++- pkg/qqwry/qqwry.go | 34 +++++------- pkg/zxipv6wry/update.go | 5 ++ pkg/zxipv6wry/zxipv6wry.go | 103 ++++++++++++++++++----------------- 7 files changed, 132 insertions(+), 80 deletions(-) create mode 100644 pkg/zxipv6wry/update.go diff --git a/go.sum b/go.sum index ae053a3..373347b 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/app/parse.go b/internal/app/parse.go index 17cea7b..68cd81a 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" + "github.com/zu1k/nali/pkg/zxipv6wry" + "github.com/zu1k/nali/constant" "github.com/zu1k/nali/internal/ipdb" "github.com/zu1k/nali/internal/iptools" @@ -13,26 +15,38 @@ import ( ) var ( - db ipdb.IPDB + db []ipdb.IPDB qqip qqwry.QQwry geoip geoip2.GeoIP ) // init ip db content func InitIPDB(ipdbtype ipdb.IPDBType) { + db = make([]ipdb.IPDB, 1) switch ipdbtype { case ipdb.GEOIP2: - db = geoip2.NewGeoIP(filepath.Join(constant.HomePath, "GeoLite2-City.mmdb")) + db[0] = geoip2.NewGeoIP(filepath.Join(constant.HomePath, "GeoLite2-City.mmdb")) case ipdb.QQIP: - db = qqwry.NewQQwry(filepath.Join(constant.HomePath, "qqwry.dat")) + db[0] = qqwry.NewQQwry(filepath.Join(constant.HomePath, "qqwry.dat")) + db = append(db, zxipv6wry.NewZXwry(filepath.Join(constant.HomePath, "ipv6wry.db"))) } } // parse several ips func ParseIPs(ips []string) { + db0 := db[0] + var db1 ipdb.IPDB + if len(db) > 1 { + db1 = db[1] + } else { + db1 = nil + } for _, ip := range ips { if iptools.ValidIP4(ip) { - result := db.Find(ip) + result := db0.Find(ip) + fmt.Println(formatResult(ip, result)) + } else if iptools.ValidIP6(ip) && db1 != nil { + result := db1.Find(ip) fmt.Println(formatResult(ip, result)) } else { fmt.Println(ReplaceInString(ip)) @@ -41,10 +55,24 @@ func ParseIPs(ips []string) { } func ReplaceInString(str string) (result string) { + db0 := db[0] + var db1 ipdb.IPDB + if len(db) > 1 { + db1 = db[1] + } else { + db1 = nil + } + result = str - ips := iptools.GetIP4FromString(str) - for _, ip := range ips { - info := db.Find(ip) + ip4s := iptools.GetIP4FromString(str) + for _, ip := range ip4s { + info := db0.Find(ip) + result = strings.ReplaceAll(result, ip, formatResult(ip, info)) + } + + ip6s := iptools.GetIP6FromString(str) + for _, ip := range ip6s { + info := db1.Find(ip) result = strings.ReplaceAll(result, ip, formatResult(ip, info)) } return diff --git a/internal/iptools/ipparser.go b/internal/iptools/ipparser.go index a362cd4..2796974 100644 --- a/internal/iptools/ipparser.go +++ b/internal/iptools/ipparser.go @@ -8,11 +8,17 @@ import ( var ( ipv4re0 *regexp.Regexp ipv4re *regexp.Regexp + + ipv6re0 *regexp.Regexp + ipv6re *regexp.Regexp ) func init() { ipv4re0 = regexp.MustCompile(`^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$`) ipv4re = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + + ipv6re0 = regexp.MustCompile(`^fe80:(:[0-9a-fA-F]{1,4}){0,4}(%\w+)?|([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?$`) + ipv6re = regexp.MustCompile(`fe80:(:[0-9a-fA-F]{1,4}){0,4}(%\w+)?|([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?`) } func ValidIP4(str string) bool { @@ -20,7 +26,17 @@ func ValidIP4(str string) bool { return ipv4re0.MatchString(str) } +func ValidIP6(str string) bool { + str = strings.Trim(str, " ") + return ipv6re0.MatchString(str) +} + func GetIP4FromString(str string) []string { str = strings.Trim(str, " ") return ipv4re.FindAllString(str, -1) } + +func GetIP6FromString(str string) []string { + str = strings.Trim(str, " ") + return ipv6re.FindAllString(str, -1) +} diff --git a/pkg/common/struct.go b/pkg/common/struct.go index 6cf5d9a..1669509 100644 --- a/pkg/common/struct.go +++ b/pkg/common/struct.go @@ -39,12 +39,12 @@ func (db *IPDB) ReadString(offset uint32) []byte { } // readData 从文件中读取数据 -func (db *IPDB) ReadData(length int, offset ...uint32) (rs []byte) { +func (db *IPDB) ReadData(length uint32, offset ...uint32) (rs []byte) { if len(offset) > 0 { db.SetOffset(offset[0]) } - readLength := uint32(length) - end := db.Offset + readLength + + end := db.Offset + length dataNum := uint32(len(db.Data.Data)) if db.Offset > dataNum { return nil @@ -82,3 +82,8 @@ func (db *IPDB) ReadArea(offset uint32) []byte { } return db.ReadString(offset) } + +func (db *IPDB) GetMiddleOffset(start uint32, end uint32) uint32 { + records := ((end - start) / db.IndexLen) >> 1 + return start + records*db.IndexLen +} diff --git a/pkg/qqwry/qqwry.go b/pkg/qqwry/qqwry.go index e72544f..d7b6862 100644 --- a/pkg/qqwry/qqwry.go +++ b/pkg/qqwry/qqwry.go @@ -13,16 +13,12 @@ import ( "golang.org/x/text/encoding/simplifiedchinese" ) -const ( - IndexLen = 7 // index length -) - type QQwry struct { common.IPDB } -// new db from path -func NewQQwry(filePath string) common.IPDB { +// NewQQwry new db from path +func NewQQwry(filePath string) QQwry { var tmpData []byte var fileInfo common.FileInfo @@ -58,9 +54,12 @@ func NewQQwry(filePath string) common.IPDB { start := binary.LittleEndian.Uint32(buf[:4]) end := binary.LittleEndian.Uint32(buf[4:]) - return common.IPDB{ - Data: &fileInfo, - IPNum: uint32((end-start)/IndexLen + 1), + return QQwry{ + IPDB: common.IPDB{ + Data: &fileInfo, + IndexLen: 7, + IPNum: (end-start)/7 + 1, + }, } } @@ -81,6 +80,7 @@ func (q QQwry) Find(ip string) (res string) { var gbkArea []byte mode := q.ReadMode(offset + 4) + // [IP][0x01][国家和地区信息的绝对偏移地址] if mode == common.RedirectMode1 { countryOffset := q.ReadUInt24() mode = q.ReadMode(countryOffset) @@ -116,18 +116,18 @@ func (q *QQwry) searchIndex(ip uint32) uint32 { start := binary.LittleEndian.Uint32(header[:4]) end := binary.LittleEndian.Uint32(header[4:]) - buf := make([]byte, IndexLen) + buf := make([]byte, q.IndexLen) mid := uint32(0) _ip := uint32(0) for { - mid = q.getMiddleOffset(start, end) - buf = q.ReadData(IndexLen, mid) + mid = q.GetMiddleOffset(start, end) + buf = q.ReadData(q.IndexLen, mid) _ip = binary.LittleEndian.Uint32(buf[:4]) - if end-start == IndexLen { + if end-start == q.IndexLen { offset := common.ByteToUInt32(buf[4:]) - buf = q.ReadData(IndexLen) + buf = q.ReadData(q.IndexLen) if ip < binary.LittleEndian.Uint32(buf[:4]) { return offset } @@ -144,9 +144,3 @@ func (q *QQwry) searchIndex(ip uint32) uint32 { } } } - -// getMiddleOffset -func (q *QQwry) getMiddleOffset(start uint32, end uint32) uint32 { - records := ((end - start) / IndexLen) >> 1 - return start + records*IndexLen -} diff --git a/pkg/zxipv6wry/update.go b/pkg/zxipv6wry/update.go new file mode 100644 index 0000000..9d8947f --- /dev/null +++ b/pkg/zxipv6wry/update.go @@ -0,0 +1,5 @@ +package zxipv6wry + +func GetOnline() ([]byte, error) { + return nil, nil +} diff --git a/pkg/zxipv6wry/zxipv6wry.go b/pkg/zxipv6wry/zxipv6wry.go index 7fb9d4c..9e25a4a 100644 --- a/pkg/zxipv6wry/zxipv6wry.go +++ b/pkg/zxipv6wry/zxipv6wry.go @@ -3,25 +3,60 @@ package zxipv6wry import ( "encoding/binary" "fmt" + "io/ioutil" + "log" "math/big" "net" + "os" "github.com/zu1k/nali/pkg/common" ) -var ( - header []byte - v6ip uint64 - offset uint32 - start uint32 - end uint32 -) - type ZXwry struct { common.IPDB } -func (q *ZXwry) Find(ip string) (result string) { +func NewZXwry(filePath string) ZXwry { + var tmpData []byte + var fileInfo common.FileInfo + + // 判断文件是否存在 + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + log.Println("文件不存在，尝试从网络获取最新ZX IPv6 库") + tmpData, err = GetOnline() + if err != nil { + panic(err) + } else { + if err := ioutil.WriteFile(filePath, tmpData, 0644); err == nil { + log.Printf("已将最新的ZX IPv6 库保存到本地 %s ", filePath) + } + } + } else { + // 打开文件句柄 + fileInfo.FileBase, err = os.OpenFile(filePath, os.O_RDONLY, 0400) + if err != nil { + panic(err) + } + defer fileInfo.FileBase.Close() + + tmpData, err = ioutil.ReadAll(fileInfo.FileBase) + if err != nil { + panic(err) + } + } + + fileInfo.Data = tmpData + + return ZXwry{ + IPDB: common.IPDB{ + Data: &fileInfo, + IndexLen: 7, + }, + } +} + +func (q ZXwry) Find(ip string) (result string) { q.Offset = 0 tp := big.NewInt(0) @@ -32,8 +67,8 @@ func (q *ZXwry) Find(ip string) (result string) { tp.SetString("FFFFFFFFFFFFFFFF", 16) op.And(op, tp) - v6ip = op.Uint64() - offset = q.searchIndexV6(v6ip) + v6ip := op.Uint64() + offset := q.searchIndex(v6ip) country, area := q.getAddr(offset) @@ -46,7 +81,7 @@ func (q *ZXwry) Find(ip string) (result string) { func (q *ZXwry) getAddr(offset uint32) (string, string) { mode := q.ReadMode(offset) - if mode == 0x01 { + if mode == common.RedirectMode1 { // [IP][0x01][国家和地区信息的绝对偏移地址] offset = q.ReadUInt24() return q.getAddr(offset) @@ -54,7 +89,7 @@ func (q *ZXwry) getAddr(offset uint32) (string, string) { // [IP][0x02][信息的绝对偏移][...] or [IP][国家][...] _offset := q.Offset - 1 c1 := q.ReadArea(_offset) - if mode == 0x02 { + if mode == common.RedirectMode2 { q.Offset = 4 + _offset } else { q.Offset = _offset + uint32(1+len(c1)) @@ -63,51 +98,19 @@ func (q *ZXwry) getAddr(offset uint32) (string, string) { return string(c1), string(c2) } -func (q *ZXwry) searchIndexV4(ip uint32) uint32 { - q.ItemLen = 4 - q.IndexLen = 7 - header = q.Data.Data[0:8] - start = binary.LittleEndian.Uint32(header[:4]) - end = binary.LittleEndian.Uint32(header[4:]) - - buf := make([]byte, q.IndexLen) - - for { - mid := start + q.IndexLen*(((end-start)/q.IndexLen)>>1) - buf = q.Data.Data[mid : mid+q.IndexLen] - _ip := binary.LittleEndian.Uint32(buf[:q.ItemLen]) - - if end-start == q.IndexLen { - if ip >= binary.LittleEndian.Uint32(q.Data.Data[end:end+q.ItemLen]) { - buf = q.Data.Data[end : end+q.IndexLen] - } - return common.ByteToUInt32(buf[q.ItemLen:]) - } - - if _ip > ip { - end = mid - } else if _ip < ip { - start = mid - } else if _ip == ip { - return common.ByteToUInt32(buf[q.ItemLen:]) - } - } -} - -func (q *ZXwry) searchIndexV6(ip uint64) uint32 { - +func (q *ZXwry) searchIndex(ip uint64) uint32 { q.ItemLen = 8 q.IndexLen = 11 - header = q.Data.Data[8:24] - start = binary.LittleEndian.Uint32(header[8:]) + header := q.Data.Data[8:24] + start := binary.LittleEndian.Uint32(header[8:]) counts := binary.LittleEndian.Uint32(header[:8]) - end = start + counts*q.IndexLen + end := start + counts*q.IndexLen buf := make([]byte, q.IndexLen) for { - mid := start + q.IndexLen*(((end-start)/q.IndexLen)>>1) + mid := q.GetMiddleOffset(start, end) buf = q.Data.Data[mid : mid+q.IndexLen] _ip := binary.LittleEndian.Uint64(buf[:q.ItemLen])