From 6bf5d94920093eb4b22c2f8314a98eb067beb90c Mon Sep 17 00:00:00 2001 From: zu1k Date: Wed, 29 Jun 2022 17:00:34 +0800 Subject: [PATCH] fix: CDN support regex Signed-off-by: zu1k --- go.mod | 4 ++-- go.sum | 4 ++++ internal/db/default.go | 2 +- internal/db/type.go | 4 ++-- internal/db/update.go | 2 +- pkg/cdn/cdn.go | 48 ++++++++++++++++++++++++++++++------------ pkg/cdn/update.go | 3 +-- pkg/re/re.go | 9 +++++++- pkg/re/re_test.go | 18 ++++++++++++++++ 9 files changed, 71 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 9d7cfd3..843ee5f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/fatih/color v1.13.0 - github.com/ip2location/ip2location-go/v9 v9.3.0 + github.com/ip2location/ip2location-go/v9 v9.4.0 github.com/ipipdotnet/ipdb-go v1.3.1 github.com/lionsoul2014/ip2region v2.2.0-release+incompatible github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20220624075035-53e6777abd9d @@ -35,7 +35,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect - golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect + golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect golang.org/x/tools v0.1.7 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 9298fbf..575b543 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ip2location/ip2location-go/v9 v9.3.0 h1:qfgr4ppf21ZyrOrmh4/db4SamL9hj+YYEl06dbf1cGM= github.com/ip2location/ip2location-go/v9 v9.3.0/go.mod h1:s5SV6YZL10TpfPpXw//7fEJC65G/yH7Oh+Tjq9JcQEQ= +github.com/ip2location/ip2location-go/v9 v9.4.0 h1:VV+clcfWo+BCi37+1zVOLCoyy01GT18/RTtAeIR5EP0= +github.com/ip2location/ip2location-go/v9 v9.4.0/go.mod h1:s5SV6YZL10TpfPpXw//7fEJC65G/yH7Oh+Tjq9JcQEQ= github.com/ipipdotnet/ipdb-go v1.3.1 h1:iMTt7a4o8r5FmTMzuHLg8XPtz8vb06gpEzJVSZzDZMY= github.com/ipipdotnet/ipdb-go v1.3.1/go.mod h1:yZ+8puwe3R37a/3qRftXo40nZVQbxYDLqls9o5foexs= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -340,6 +342,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810 h1:rHZQSjJdAI4Xf5Qzeh2bBc5YJIkPFVM6oDtMFYmgws0= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/db/default.go b/internal/db/default.go index bee233f..fd6297e 100644 --- a/internal/db/default.go +++ b/internal/db/default.go @@ -78,7 +78,7 @@ func GetDefaultDBList() List { &DB{ Name: "cdn", - Format: FormatCDNSkkYml, + Format: FormatCDNYml, File: "cdn.yml", Languages: LanguagesZH, Types: TypesCDN, diff --git a/internal/db/type.go b/internal/db/type.go index 8756494..5db927b 100644 --- a/internal/db/type.go +++ b/internal/db/type.go @@ -46,7 +46,7 @@ func (d *DB) get() (db dbif.DB) { db, err = ip2region.NewIp2Region(filePath) case FormatIP2Location: db, err = ip2location.NewIP2Location(filePath) - case FormatCDNSkkYml: + case FormatCDNYml: db, err = cdn.NewCDN(filePath) default: panic("DB format not supported!") @@ -70,7 +70,7 @@ const ( FormatIP2Region = "ip2region" FormatIP2Location = "ip2location" - FormatCDNSkkYml = "cdn-skk-yml" + FormatCDNYml = "cdn-yml" ) var ( diff --git a/internal/db/update.go b/internal/db/update.go index 9d31a29..42bbe88 100644 --- a/internal/db/update.go +++ b/internal/db/update.go @@ -84,7 +84,7 @@ func getUpdateFuncByName(name string) (func() error, string) { } return err }, FormatZXIPv6Wry - case FormatCDNSkkYml: + case FormatCDNYml: return func() error { log.Println("正在下载最新 CDN服务提供商数据库...") _, err := cdn.Download(getDbByName("cdn").File) diff --git a/pkg/cdn/cdn.go b/pkg/cdn/cdn.go index eaf2da3..ac78e6c 100644 --- a/pkg/cdn/cdn.go +++ b/pkg/cdn/cdn.go @@ -6,16 +6,23 @@ import ( "io/ioutil" "log" "os" + "regexp" "strings" "gopkg.in/yaml.v2" + + "github.com/zu1k/nali/pkg/re" ) type CDN struct { - Data CDNDist + Map map[string]CDNResult + ReMap []CDNReTuple } -type CDNDist map[string]CDNResult +type CDNReTuple struct { + *regexp.Regexp + CDNResult +} type CDNResult struct { Name string `yaml:"name"` @@ -27,13 +34,11 @@ func (r CDNResult) String() string { } func NewCDN(filePath string) (*CDN, error) { - cdnDist := make(CDNDist) - cdnData := make([]byte, 0) - + fileData := make([]byte, 0) _, err := os.Stat(filePath) if err != nil && os.IsNotExist(err) { log.Println("文件不存在，尝试从网络获取最新CDN数据库") - cdnData, err = Download(filePath) + fileData, err = Download(filePath) if err != nil { return nil, err } @@ -44,33 +49,48 @@ func NewCDN(filePath string) (*CDN, error) { } defer cdnFile.Close() - cdnData, err = ioutil.ReadAll(cdnFile) + fileData, err = ioutil.ReadAll(cdnFile) if err != nil { return nil, err } } - err = yaml.Unmarshal(cdnData, &cdnDist) + cdnMap := make(map[string]CDNResult) + err = yaml.Unmarshal(fileData, &cdnMap) if err != nil { return nil, err } - return &CDN{Data: cdnDist}, nil + cdnReMap := make([]CDNReTuple, 0) + for k, v := range cdnMap { + if re.MaybeRegexp(k) { + rex, err := regexp.Compile(k) + if err != nil { + log.Printf("[CDN Database] entry %s not a valid regexp", k) + } + cdnReMap = append(cdnReMap, CDNReTuple{ + Regexp: rex, + CDNResult: v, + }) + } + } + + return &CDN{Map: cdnMap, ReMap: cdnReMap}, nil } func (db CDN) Find(query string, params ...string) (result fmt.Stringer, err error) { baseCname := parseBaseCname(query) for _, domain := range baseCname { if domain != "" { - cdnResult, found := db.Data[domain] + cdnResult, found := db.Map[domain] if found { return cdnResult, nil } } - if strings.Contains(domain, "kunlun") { - return CDNResult{ - Name: "阿里云 CDN", - }, nil + for _, entry := range db.ReMap { + if entry.Regexp.MatchString(domain) { + return entry.CDNResult, nil + } } } diff --git a/pkg/cdn/update.go b/pkg/cdn/update.go index 3f25089..61a1460 100644 --- a/pkg/cdn/update.go +++ b/pkg/cdn/update.go @@ -7,10 +7,9 @@ import ( ) var DownloadUrls = []string{ - "https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml", - "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml", "https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml", "https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml", + "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml", } // Deprecated: This will be removed from 0.5.0, use package download instead diff --git a/pkg/re/re.go b/pkg/re/re.go index a6bae06..780b159 100644 --- a/pkg/re/re.go +++ b/pkg/re/re.go @@ -1,6 +1,9 @@ package re -import "regexp" +import ( + "regexp" + "strings" +) var ( DomainRe = regexp.MustCompile(`([a-zA-Z0-9][-a-zA-Z0-9]{0,62}\.)+([a-zA-Z][-a-zA-Z]{0,62})`) @@ -8,3 +11,7 @@ var ( IPv4Re = regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) IPv6Re = regexp.MustCompile(`fe80:(:[0-9a-fA-F]{1,4}){0,4}(%\w+)?|([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|::[fF]{4}:(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::(([0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?`) ) + +func MaybeRegexp(s string) bool { + return strings.ContainsAny(s, "[]{}()?") +} diff --git a/pkg/re/re_test.go b/pkg/re/re_test.go index 989d5a0..4760ca1 100644 --- a/pkg/re/re_test.go +++ b/pkg/re/re_test.go @@ -35,3 +35,21 @@ func TestIPv6Re(t *testing.T) { fmt.Println(IPv6Re.FindAllString(ip, -1)) } } + +var maybeRegexList = []string{ + "[a-z]*\\.example.com", + "kunlun[^.]+.com", + "gtm-a[1-7]b[1-9].com", +} + +func TestMaybeRegexp(t *testing.T) { + if MaybeRegexp("abc.com") { + t.Fail() + } + + for _, str := range maybeRegexList { + if !MaybeRegexp(str) { + t.Fail() + } + } +}