Add rule-set releases

2025-02-06 04:52:43 +08:00 · 2023-11-29 14:26:06 +08:00 · 2023-11-29 14:26:06 +08:00 · 4062751566
commit 4062751566
parent b1dc5b3c1d
10 changed files with 141 additions and 175 deletions
--- a/.github/release-rule-set.sh
+++ b/.github/release-rule-set.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+cd rule-set
+git init
+git config --local user.email "github-action@users.noreply.github.com"
+git config --local user.name "GitHub Action"
+git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geoip.git
+git branch -M rule-set
+git add .
+git commit -m "Update rule-set"
+git push -f origin rule-set
--- a/.github/update_dependencies.sh
+++ b/.github/update_dependencies.sh
@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+PROJECTS=$(dirname "$0")/../..
+
+go get -x github.com/sagernet/sing-box@$(git -C $PROJECTS/sing-box rev-parse HEAD)
+go mod tidy
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@ -29,9 +29,4 @@ jobs:
        uses: actions/upload-artifact@v2
        with:
          name: geoip.db
-          path: geoip.db
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: geoip-cn.db
-          path: geoip-cn.db
+          path: geoip.db
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -23,64 +23,25 @@ jobs:
      - name: Build geoip
        id: build
        run: go run -v .
+      - name: Release rule sets
+        if: steps.build.outputs.skip != 'true'
+        run: .github/release-rule-set.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Generate sha256 hash
        if: steps.build.outputs.skip != 'true'
        run: |
          sha256sum geoip.db > geoip.db.sha256sum
-          sha256sum geoip-cn.db > geoip-cn.db.sha256sum
      - uses: dev-drprasad/delete-older-releases@v0.3.2
        with:
-          keep_latest: 3
+          keep_latest: 10
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create a release
+      - name: Release geoip
        if: steps.build.outputs.skip != 'true'
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: softprops/action-gh-release@v1
        with:
          tag_name: ${{ steps.build.outputs.tag }}
-          release_name: ${{ steps.build.outputs.tag }}
-          draft: false
-          prerelease: false
-      - name: Release geoip.db
-        if: steps.build.outputs.skip != 'true'
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./geoip.db
-          asset_name: geoip.db
-          asset_content_type: application/octet-stream
-      - name: Release geoip.db sha256sum
-        if: steps.build.outputs.skip != 'true'
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./geoip.db.sha256sum
-          asset_name: geoip.db.sha256sum
-          asset_content_type: text/plain
-      - name: Release geoip-cn.db
-        if: steps.build.outputs.skip != 'true'
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./geoip-cn.db
-          asset_name: geoip-cn.db
-          asset_content_type: application/octet-stream
-      - name: Release geoip.db sha256sum
-        if: steps.build.outputs.skip != 'true'
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./geoip-cn.db.sha256sum
-          asset_name: geoip-cn.db.sha256sum
-          asset_content_type: text/plain
+          files: |
+            geoip.db
+            geoip.db.sha256sum
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,5 @@
 /.idea/
 /vendor/
 /*.db
-*.mmdb
+*.mmdb
+/rule-set/
--- a/.golangci.yml
+++ b/.golangci.yml
@ -1,54 +1,17 @@
-run:
-  timeout: 5m
-
 linters:
-  enable-all: true
-  disable:
-    - errcheck
-    - wrapcheck
-    - varnamelen
-    - stylecheck
-    - nonamedreturns
-    - nlreturn
-    - ireturn
-    - gomnd
-    - exhaustivestruct
-    - ifshort
-    - goerr113
-    - gochecknoglobals
-    - forcetypeassert
-    - exhaustruct
-    - exhaustive
-    - cyclop
-    - containedctx
-    - wsl
-    - nestif
-    - lll
-    - funlen
-    - goconst
-    - godot
-    - gocognit
-    - golint
-    - goimports
-    - gochecknoinits
-    - maligned
-    - tagliatelle
-    - gocyclo
-    - maintidx
-    - gocritic
-    - nakedret
+  disable-all: true
+  enable:
+    - gofumpt
+    - govet
+    - gci
+    - staticcheck

 linters-settings:
-  revive:
-    rules:
-      - name: var-naming
-        disabled: true
-  govet:
-    enable-all: true
-    disable:
-      - composites
-      - fieldalignment
-      - shadow
-  gosec:
-    excludes:
-      - G404
+  gci:
+    custom-order: true
+    sections:
+      - standard
+      - prefix(github.com/sagernet/)
+      - default
+  staticcheck:
+    go: '1.20'
--- a/17
+++ b/17
@ -0,0 +1,17 @@
+fmt:
+	@gofumpt -l -w .
+	@gofmt -s -w .
+	@gci write --custom-order -s standard -s "prefix(github.com/sagernet/)" -s "default" .
+
+fmt_install:
+	go install -v mvdan.cc/gofumpt@latest
+	go install -v github.com/daixiang0/gci@latest
+
+lint:
+	golangci-lint run ./...
+
+lint_install:
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+
+test:
+	go test -v ./...
--- a/go.mod
+++ b/go.mod
@ -1,4 +1,4 @@
-module sing-geoip
+module github.com/sagernet/sing-geoip

 go 1.18

@ -7,13 +7,20 @@ require (
 	github.com/maxmind/mmdbwriter v1.0.0
 	github.com/oschwald/geoip2-golang v1.9.0
 	github.com/oschwald/maxminddb-golang v1.12.0
-	github.com/sagernet/sing v0.2.17
-	github.com/sirupsen/logrus v1.9.3
+	github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be
+	github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544
 )

 require (
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
+	github.com/miekg/dns v1.1.57 // indirect
+	github.com/sagernet/sing-dns v0.1.11 // indirect
+	go4.org/netipx v0.0.0-20230824141953-6213f710f925 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -1,35 +1,41 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
 github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
 github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
+github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
+github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
 github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
 github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
 github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
 github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sagernet/sing v0.2.17 h1:vMPKb3MV0Aa5ws4dCJkRI8XEjrsUcDn810czd0FwmzI=
-github.com/sagernet/sing v0.2.17/go.mod h1:OL6k2F0vHmEzXz2KW19qQzu172FDgSbUSODylighuVo=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be h1:FigAM9kq7RRXmHvgn8w2a8tqCY5CMV5GIk0id84dI0o=
+github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be/go.mod h1:OL6k2F0vHmEzXz2KW19qQzu172FDgSbUSODylighuVo=
+github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544 h1:kHe9kQpKMEmGJbTdDntXA0rl1BdQWQZtkDaxmZkAPC4=
+github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544/go.mod h1:C5Gcyr9BsHs1Iq7eAY53hAlh/j+5fjJVsLA18z/7ZL4=
+github.com/sagernet/sing-dns v0.1.11 h1:PPrMCVVrAeR3f5X23I+cmvacXJ+kzuyAsBiWyUKhGSE=
+github.com/sagernet/sing-dns v0.1.11/go.mod h1:zJ/YjnYB61SYE+ubMcMqVdpaSvsyQ2iShQGO3vuLvvE=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw=
-go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ=
+go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
+golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
--- a/main.go
+++ b/main.go
@ -6,20 +6,22 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"

-	"github.com/sagernet/sing/common"
-	E "github.com/sagernet/sing/common/exceptions"
-	"github.com/sagernet/sing/common/rw"
-
 	"github.com/google/go-github/v45/github"
 	"github.com/maxmind/mmdbwriter"
 	"github.com/maxmind/mmdbwriter/inserter"
 	"github.com/maxmind/mmdbwriter/mmdbtype"
 	"github.com/oschwald/geoip2-golang"
 	"github.com/oschwald/maxminddb-golang"
-	"github.com/sirupsen/logrus"
+	"github.com/sagernet/sing-box/common/srs"
+	C "github.com/sagernet/sing-box/constant"
+	"github.com/sagernet/sing-box/log"
+	"github.com/sagernet/sing-box/option"
+	"github.com/sagernet/sing/common"
+	E "github.com/sagernet/sing/common/exceptions"
 )

 var githubClient *github.Client
@ -46,7 +48,7 @@ func fetch(from string) (*github.RepositoryRelease, error) {
 }

 func get(downloadURL *string) ([]byte, error) {
-	logrus.Info("download ", *downloadURL)
+	log.Info("download ", *downloadURL)
 	response, err := http.Get(*downloadURL)
 	if err != nil {
 		return nil, err
@ -160,38 +162,17 @@ func write(writer *mmdbwriter.Tree, dataMap map[string][]*net.IPNet, output stri
 	return err
 }

-func local(input string, output string, codes []string) error {
-	binary, err := os.ReadFile(input)
-	if err != nil {
-		return err
-	}
-	metadata, countryMap, err := parse(binary)
-	if err != nil {
-		return err
-	}
-	var writer *mmdbwriter.Tree
-	if rw.FileExists(output) {
-		writer, err = open(output, codes)
-	} else {
-		writer, err = newWriter(metadata, codes)
-	}
-	if err != nil {
-		return err
-	}
-	return write(writer, countryMap, output, codes)
-}
-
-func release(source string, destination string) error {
+func release(source string, destination string, output string, ruleSetOutput string) error {
 	sourceRelease, err := fetch(source)
 	if err != nil {
 		return err
 	}
 	destinationRelease, err := fetch(destination)
 	if err != nil {
-		logrus.Warn("missing destination latest release")
+		log.Warn("missing destination latest release")
 	} else {
 		if os.Getenv("NO_SKIP") != "true" && strings.Contains(*destinationRelease.Name, *sourceRelease.Name) {
-			logrus.Info("already latest")
+			log.Info("already latest")
 			setActionOutput("skip", "true")
 			return nil
 		}
@ -212,20 +193,41 @@ func release(source string, destination string) error {
 	if err != nil {
 		return err
 	}
-	err = write(writer, countryMap, "geoip.db", nil)
+	err = write(writer, countryMap, output, nil)
 	if err != nil {
 		return err
 	}
-	writer, err = newWriter(metadata, []string{"cn"})
+
+	os.RemoveAll(ruleSetOutput)
+	err = os.MkdirAll(ruleSetOutput, 0o755)
 	if err != nil {
 		return err
 	}
-	err = write(writer, countryMap, "geoip-cn.db", []string{"cn"})
-	if err != nil {
-		return err
-	}
-	if err != nil {
-		return err
+	for countryCode, ipNets := range countryMap {
+		var headlessRule option.DefaultHeadlessRule
+		headlessRule.IPCIDR = make([]string, 0, len(ipNets))
+		for _, cidr := range ipNets {
+			headlessRule.IPCIDR = append(headlessRule.IPCIDR, cidr.String())
+		}
+		var plainRuleSet option.PlainRuleSet
+		plainRuleSet.Rules = []option.HeadlessRule{
+			{
+				Type:           C.RuleTypeDefault,
+				DefaultOptions: headlessRule,
+			},
+		}
+		srsPath, _ := filepath.Abs(filepath.Join(ruleSetOutput, "geoip-"+countryCode+".srs"))
+		os.Stderr.WriteString("write " + srsPath + "\n")
+		outputRuleSet, err := os.Create(srsPath)
+		if err != nil {
+			return err
+		}
+		err = srs.Write(outputRuleSet, plainRuleSet)
+		if err != nil {
+			outputRuleSet.Close()
+			return err
+		}
+		outputRuleSet.Close()
 	}
 	setActionOutput("tag", *sourceRelease.Name)
 	return nil
@ -236,13 +238,8 @@ func setActionOutput(name string, content string) {
 }

 func main() {
-	var err error
-	if len(os.Args) >= 3 {
-		err = local(os.Args[1], os.Args[2], os.Args[3:])
-	} else {
-		err = release("Dreamacro/maxmind-geoip", "sagernet/sing-geoip")
-	}
+	err := release("Dreamacro/maxmind-geoip", "sagernet/sing-geoip", "geoip.db", "rule-set")
 	if err != nil {
-		logrus.Fatal(err)
+		log.Fatal(err)
 	}
 }