commit 782ba35a59910fb14191a53d7b7cc198d82139d8 Author: 世界 Date: Mon Jul 4 22:45:11 2022 +0800 Init commit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..de233b3 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,65 @@ +name: Release +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 1" +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Get latest go version + id: version + run: | + echo ::set-output name=go_version::$(curl -s https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json | grep -oE '"version": "[0-9]{1}.[0-9]{1,}(.[0-9]{1,})?"' | head -1 | cut -d':' -f2 | sed 's/ //g; s/"//g') + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ steps.version.outputs.go_version }} + - name: Build geosite + id: build + env: + GOPRIVATE: github.com/sagernet + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + run: | + git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github.com".insteadOf "https://github.com" + go run -v . + - name: Generate sha256 hash + if: steps.build.outputs.skip != 'true' + run: | + sha256sum geosite.db > geosite.db.sha256sum + - name: Create a release + if: steps.build.outputs.skip != 'true' + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.build.outputs.tag }} + release_name: ${{ steps.build.outputs.tag }} + draft: false + prerelease: false + - name: Release geosite.db + if: steps.build.outputs.skip != 'true' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./geosite.db + asset_name: geosite.db + asset_content_type: application/octet-stream + - name: Release geosite.db sha256sum + if: steps.build.outputs.skip != 'true' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./geosite.db.sha256sum + asset_name: geosite.db.sha256sum + asset_content_type: text/plain \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e417e3f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/.idea/ +/vendor/ +/geosite.db \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..948fcc4 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,54 @@ +run: + timeout: 5m + +linters: + enable-all: true + disable: + - errcheck + - wrapcheck + - varnamelen + - stylecheck + - nonamedreturns + - nlreturn + - ireturn + - gomnd + - exhaustivestruct + - ifshort + - goerr113 + - gochecknoglobals + - forcetypeassert + - exhaustruct + - exhaustive + - cyclop + - containedctx + - wsl + - nestif + - lll + - funlen + - goconst + - godot + - gocognit + - golint + - goimports + - gochecknoinits + - maligned + - tagliatelle + - gocyclo + - maintidx + - gocritic + - nakedret + +linters-settings: + revive: + rules: + - name: var-naming + disabled: true + govet: + enable-all: true + disable: + - composites + - fieldalignment + - shadow + gosec: + excludes: + - G404 \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3e3e29e --- /dev/null +++ b/LICENSE @@ -0,0 +1,14 @@ +Copyright (C) 2022 by nekohasekai + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . \ No newline at end of file diff --git a/format.go b/format.go new file mode 100644 index 0000000..37faa45 --- /dev/null +++ b/format.go @@ -0,0 +1,7 @@ +package main + +//go:generate go install -v mvdan.cc/gofumpt@latest +//go:generate go install -v github.com/daixiang0/gci@latest +//go:generate gofumpt -l -w . +//go:generate gofmt -s -w . +//go:generate gci write . diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3e3310f --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module sing-geosite + +go 1.18 + +require ( + github.com/golang/protobuf v1.5.2 + github.com/google/go-github/v45 v45.2.0 + github.com/sagernet/sing v0.0.0-20220704113227-8b990551511a + github.com/sagernet/sing-box v0.0.0-20220704113958-f76102dab512 + github.com/sirupsen/logrus v1.8.1 + github.com/v2fly/v2ray-core/v5 v5.0.7 +) + +require ( + github.com/adrg/xdg v0.4.0 // indirect + github.com/goccy/go-json v0.9.8 // indirect + github.com/google/go-querystring v1.1.0 // indirect + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect + golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d1a2730 --- /dev/null +++ b/go.sum @@ -0,0 +1,45 @@ +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.9.8 h1:DxXB6MLd6yyel7CLph8EwNIonUtVZd3Ue5iRcL4DQCE= +github.com/goccy/go-json v0.9.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= +github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sagernet/sing v0.0.0-20220704113227-8b990551511a h1:IvYjuvuPNmZzQfBbCxE/uQqGkNWUa5/KrEMIecRMjZk= +github.com/sagernet/sing v0.0.0-20220704113227-8b990551511a/go.mod h1:3ZmoGNg/nNJTyHAZFNRSPaXpNIwpDvyIiAUd0KIWV5c= +github.com/sagernet/sing-box v0.0.0-20220704113958-f76102dab512 h1:hZ/oWGx6Ff1HMqhCqmahvaW4W3gb83/jBhmWu1Cuf8Y= +github.com/sagernet/sing-box v0.0.0-20220704113958-f76102dab512/go.mod h1:Yf1jgOxozYlIYCK2HFUzVf06U2wJP3RcTHl0XACFX/c= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/v2fly/v2ray-core/v5 v5.0.7 h1:wR8x5KyYpe0W35tcJz/dlkpCClDhc/xe+36BQjVV3EM= +github.com/v2fly/v2ray-core/v5 v5.0.7/go.mod h1:whgevEWmA6LrAfnPoM97IGMYhUF8837sAZ4U6MNJfzk= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/main.go b/main.go new file mode 100644 index 0000000..bb1cc38 --- /dev/null +++ b/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/google/go-github/v45/github" + "github.com/sagernet/sing-box/common/geosite" + "github.com/sagernet/sing/common" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sirupsen/logrus" + "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "google.golang.org/protobuf/proto" +) + +var githubClient *github.Client + +func init() { + accessToken, loaded := os.LookupEnv("ACCESS_TOKEN") + if !loaded { + githubClient = github.NewClient(nil) + return + } + transport := &github.BasicAuthTransport{ + Username: accessToken, + } + githubClient = github.NewClient(transport.Client()) +} + +func fetch(from string) (*github.RepositoryRelease, error) { + names := strings.SplitN(from, "/", 2) + latestRelease, _, err := githubClient.Repositories.GetLatestRelease(context.Background(), names[0], names[1]) + if err != nil { + return nil, err + } + return latestRelease, err +} + +func get(downloadURL *string) ([]byte, error) { + logrus.Info("download ", *downloadURL) + response, err := http.Get(*downloadURL) + if err != nil { + return nil, err + } + defer response.Body.Close() + return io.ReadAll(response.Body) +} + +func download(release *github.RepositoryRelease) ([]byte, error) { + geositeAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool { + return *it.Name == "dlc.dat" + }) + geositeChecksumAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool { + return *it.Name == "dlc.dat.sha256sum" + }) + if geositeAsset == nil { + return nil, E.New("geosite asset not found in upstream release ", release.Name) + } + if geositeChecksumAsset == nil { + return nil, E.New("geosite asset not found in upstream release ", release.Name) + } + data, err := get(geositeAsset.BrowserDownloadURL) + if err != nil { + return nil, err + } + remoteChecksum, err := get(geositeChecksumAsset.BrowserDownloadURL) + if err != nil { + return nil, err + } + checksum := sha256.Sum256(data) + if hex.EncodeToString(checksum[:]) != string(remoteChecksum[:64]) { + return nil, E.New("checksum mismatch") + } + return data, nil +} + +func parse(vGeositeData []byte) (map[string][]geosite.Item, error) { + vGeositeList := routercommon.GeoSiteList{} + err := proto.Unmarshal(vGeositeData, &vGeositeList) + if err != nil { + return nil, err + } + domainMap := make(map[string][]geosite.Item) + for _, vGeositeEntry := range vGeositeList.Entry { + domains := make([]geosite.Item, 0, len(vGeositeEntry.Domain)*2) + for _, domain := range vGeositeEntry.Domain { + switch domain.Type { + case routercommon.Domain_Plain: + domains = append(domains, geosite.Item{ + Type: geosite.RuleTypeDomainKeyword, + Value: domain.Value, + }) + case routercommon.Domain_Regex: + domains = append(domains, geosite.Item{ + Type: geosite.RuleTypeDomainRegex, + Value: domain.Value, + }) + case routercommon.Domain_RootDomain: + domains = append(domains, geosite.Item{ + Type: geosite.RuleTypeDomain, + Value: domain.Value, + }) + domains = append(domains, geosite.Item{ + Type: geosite.RuleTypeDomainSuffix, + Value: "." + domain.Value, + }) + case routercommon.Domain_Full: + domains = append(domains, geosite.Item{ + Type: geosite.RuleTypeDomain, + Value: domain.Value, + }) + } + } + domainMap[strings.ToLower(vGeositeEntry.CountryCode)] = common.Uniq(domains) + } + return domainMap, nil +} + +func generate(release *github.RepositoryRelease, output string) error { + outputFile, err := os.Create(output) + if err != nil { + return err + } + defer outputFile.Close() + vData, err := download(release) + if err != nil { + return err + } + domainMap, err := parse(vData) + if err != nil { + return err + } + outputPath, _ := filepath.Abs(output) + os.Stderr.WriteString("write " + outputPath + "\n") + return geosite.Write(outputFile, domainMap) +} + +func setActionOutput(name string, content string) { + os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n") +} + +func release(source string, destination string, output string) error { + sourceRelease, err := fetch(source) + if err != nil { + return err + } + destinationRelease, err := fetch(destination) + if err != nil { + logrus.Warn("missing destination latest release") + } else { + if strings.Contains(*destinationRelease.Name, *sourceRelease.Name) { + logrus.Info("already latest") + setActionOutput("skip", "true") + return nil + } + } + err = generate(sourceRelease, output) + if err != nil { + return err + } + setActionOutput("tag", *sourceRelease.Name) + return nil +} + +func main() { + err := release("v2fly/domain-list-community", "sagernet/sing-geosite", "geosite.db") + if err != nil { + logrus.Fatal(err) + } +}