feat: Support update db by direct download

Signed-off-by: zu1k <i@zu1k.com>

internal/config/config.go

@@ -35,6 +35,7 @@ func ReadConfig(basePath string) {
 	if err != nil {
 		log.Fatalln("Config invalid:", err)
 	}
+
 	db.NameDBMap.From(dbList)
 	db.TypeDBMap.From(dbList)
 }

internal/db/default.go

@@ -1,5 +1,10 @@
 package db
 
+import (
+	"github.com/zu1k/nali/pkg/cdn"
+	"github.com/zu1k/nali/pkg/ip2region"
+)
+
 func GetDefaultDBList() List {
 	return List{
 		&DB{
@@ -57,10 +62,11 @@ func GetDefaultDBList() List {
 			NameAlias: []string{
 				"i2r",
 			},
-			Format:    FormatIP2Region,
+			Format:       FormatIP2Region,
-			File:      "ip2region.db",
+			File:         "ip2region.db",
-			Languages: LanguagesZH,
+			Languages:    LanguagesZH,
-			Types:     TypesIPv4,
+			Types:        TypesIPv4,
+			DownloadUrls: ip2region.DownloadUrls,
 		},
 		&DB{
 			Name:      "ip2location",
@@ -71,11 +77,12 @@ func GetDefaultDBList() List {
 		},
 
 		&DB{
-			Name:      "cdn",
+			Name:         "cdn",
-			Format:    FormatCDNSkkYml,
+			Format:       FormatCDNSkkYml,
-			File:      "cdn.yml",
+			File:         "cdn.yml",
-			Languages: LanguagesZH,
+			Languages:    LanguagesZH,
-			Types:     TypesCDN,
+			Types:        TypesCDN,
+			DownloadUrls: cdn.DownloadUrls,
 		},
 	}
 }

internal/db/type.go

@@ -15,12 +15,14 @@ import (
 
 type DB struct {
 	Name      string
-	NameAlias []string `yaml:"name-alias,omitempty"`
+	NameAlias []string `yaml:"name-alias,omitempty" mapstructure:"name-alias"`
 	Format    Format
 	File      string
 
 	Languages []string
 	Types     []Type
+
+	DownloadUrls []string `yaml:"download-urls,omitempty" mapstructure:"download-urls"`
 }
 
 func (d *DB) get() (db dbif.DB) {
@@ -43,7 +45,7 @@ func (d *DB) get() (db dbif.DB) {
 	case FormatIP2Region:
 		db, err = ip2region.NewIp2Region(filePath)
 	case FormatIP2Location:
-		db, err = ip2locationdb.NewIP2LocationDB(filePath)
+		db, err = ip2location.NewIP2Location(filePath)
 	case FormatCDNSkkYml:
 		db, err = cdn.NewCDN(filePath)
 	default:

internal/db/update.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/zu1k/nali/pkg/cdn"
+	"github.com/zu1k/nali/pkg/download"
 	"github.com/zu1k/nali/pkg/ip2region"
 	"github.com/zu1k/nali/pkg/qqwry"
 	"github.com/zu1k/nali/pkg/zxipv6wry"
@@ -38,13 +39,30 @@ var DbNameListForUpdate = []string{
 func getUpdateFuncByName(name string) (func() error, string) {
 	name = strings.TrimSpace(name)
 	if db := getDbByName(name); db != nil {
+		// direct download if download-url not null
+		if len(db.DownloadUrls) > 0 {
+			return func() error {
+				log.Printf("正在下载最新 %s 数据库...\n", db.Name)
+				_, err := download.Download(db.File, db.DownloadUrls...)
+				if err != nil {
+					log.Printf("%s 数据库下载失败: %s\n", db.Name, db.File)
+					log.Println("error:", err)
+					return err
+				} else {
+					log.Printf("%s 数据库下载成功: %s\n", db.Name, db.File)
+					return nil
+				}
+			}, string(db.Format)
+		}
+
+		// intenel download func
 		switch db.Format {
 		case FormatQQWry:
 			return func() error {
 				log.Println("正在下载最新 纯真 IPv4数据库...")
 				_, err := qqwry.Download(getDbByName("qqwry").File)
 				if err != nil {
-					log.Fatalln("数据库 QQWry 下载失败:", err)
+					log.Println("数据库 QQWry 下载失败:", err)
 				}
 				return err
 			}, FormatQQWry
@@ -53,7 +71,7 @@ func getUpdateFuncByName(name string) (func() error, string) {
 				log.Println("正在下载最新 ZX IPv6数据库...")
 				_, err := zxipv6wry.Download(getDbByName("zxipv6wry").File)
 				if err != nil {
-					log.Fatalln("数据库 ZXIPv6Wry 下载失败:", err)
+					log.Println("数据库 ZXIPv6Wry 下载失败:", err)
 				}
 				return err
 			}, FormatZXIPv6Wry
@@ -62,7 +80,7 @@ func getUpdateFuncByName(name string) (func() error, string) {
 				log.Println("正在下载最新 Ip2Region 数据库...")
 				_, err := ip2region.Download(getDbByName("ip2region").File)
 				if err != nil {
-					log.Fatalln("数据库 Ip2Region 下载失败:", err)
+					log.Println("数据库 Ip2Region 下载失败:", err)
 				}
 				return err
 			}, FormatZXIPv6Wry
@@ -71,13 +89,14 @@ func getUpdateFuncByName(name string) (func() error, string) {
 				log.Println("正在下载最新 CDN服务提供商数据库...")
 				_, err := cdn.Download(getDbByName("cdn").File)
 				if err != nil {
-					log.Fatalln("数据库 CDN 下载失败:", err)
+					log.Println("数据库 CDN 下载失败:", err)
 				}
 				return err
 			}, FormatZXIPv6Wry
 		default:
 			return func() error {
-				log.Fatalln("不支持该类型数据库的自动更新：", db.Format)
+				log.Println("暂不支持该类型数据库的自动更新")
+				log.Println("可通过指定数据库的 download-urls 从特定链接下载数据库文件")
 				return nil
 			}, time.Now().String()
 		}

pkg/cdn/update.go

@@ -6,19 +6,19 @@ import (
 	"github.com/zu1k/nali/pkg/common"
 )
 
-const (
+var DownloadUrls = []string{
-	githubUrl   = "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml"
+	"https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml",
-	jsdelivrUrl = "https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml"
+	"https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml",
-
+	"https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml",
-	githubUrl2   = "https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml"
+	"https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml",
-	jsdelivrUrl2 = "https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml"
+}
-)
 
+// Deprecated: This will be removed from 0.5.0, use package download instead
 func Download(filePath ...string) (data []byte, err error) {
-	data, err = common.GetHttpClient().Get(jsdelivrUrl, githubUrl, jsdelivrUrl2, githubUrl2)
+	data, err = common.GetHttpClient().Get(DownloadUrls...)
 	if err != nil {
 		log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath)
-		log.Println("下载链接：", githubUrl)
+		log.Println("下载链接：", DownloadUrls)
 		return
 	}
 

pkg/dbif/db.go

@@ -30,6 +30,6 @@ var (
 	_ DB = &ipip.IPIPFree{}
 	_ DB = &geoip.GeoIP{}
 	_ DB = &ip2region.Ip2Region{}
-	_ DB = &ip2locationdb.IP2LocationDB{}
+	_ DB = &ip2location.IP2Location{}
 	_ DB = &cdn.CDN{}
 )

pkg/download/download.go

@@ -0,0 +1,25 @@
+package download
+
+import (
+	"log"
+
+	"github.com/zu1k/nali/pkg/common"
+)
+
+func Download(filePath string, urls ...string) (data []byte, err error) {
+	_ = urls[0]
+
+	data, err = common.GetHttpClient().Get(urls...)
+	if err != nil {
+		log.Printf("文件下载失败，请手动下载解压后保存到本地: %s \n", filePath)
+		log.Println("下载链接：", urls)
+		return
+	}
+
+	if len(filePath) == 1 {
+		if err := common.SaveFile(filePath, data); err == nil {
+			log.Println("文件下载成功:", filePath)
+		}
+	}
+	return
+}

pkg/ip2location/ip2location.go

@@ -1,4 +1,4 @@
-package ip2locationdb
+package ip2location
 
 import (
 	"errors"
@@ -10,13 +10,13 @@ import (
 	"github.com/ip2location/ip2location-go/v9"
 )
 
-// IP2LocationDB
+// IP2Location
-type IP2LocationDB struct {
+type IP2Location struct {
 	db *ip2location.DB
 }
 
 // new IP2Location from database file
-func NewIP2LocationDB(filePath string) (*IP2LocationDB, error) {
+func NewIP2Location(filePath string) (*IP2Location, error) {
 	_, err := os.Stat(filePath)
 	if err != nil && os.IsNotExist(err) {
 		log.Println("文件不存在，请自行下载 IP2Location 库，并保存在", filePath)
@@ -27,11 +27,11 @@ func NewIP2LocationDB(filePath string) (*IP2LocationDB, error) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		return &IP2LocationDB{db: db}, nil
+		return &IP2Location{db: db}, nil
 	}
 }
 
-func (x IP2LocationDB) Find(query string, params ...string) (result fmt.Stringer, err error) {
+func (x IP2Location) Find(query string, params ...string) (result fmt.Stringer, err error) {
 	ip := net.ParseIP(query)
 	if ip == nil {
 		return nil, errors.New("Query should be valid IP")

pkg/ip2region/update.go

@@ -6,16 +6,17 @@ import (
 	"github.com/zu1k/nali/pkg/common"
 )
 
-const (
+var DownloadUrls = []string{
-	githubUrl   = "https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.db"
+	"https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.db",
-	jsdelivrUrl = "https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.db"
+	"https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.db",
-)
+}
 
+// Deprecated: This will be removed from 0.5.0, use package download instead
 func Download(filePath ...string) (data []byte, err error) {
-	data, err = common.GetHttpClient().Get(jsdelivrUrl, githubUrl)
+	data, err = common.GetHttpClient().Get(DownloadUrls...)
 	if err != nil {
 		log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath)
-		log.Println("下载链接：", githubUrl)
+		log.Println("下载链接：", DownloadUrls)
 		return
 	}