检查域名有效性逻辑

.gitignore

@@ -1,4 +1,5 @@
 
 optimize-dns.conf
 .gitignore
-.idea/
+.idea/
+origin-files/split-tmp-list_*

README.md

@@ -1,7 +1,14 @@
-# anti-AD v3.0
+# anti-AD v4.0
 
 ### 基于dnsmasq服务的广告封杀、恶意网站屏蔽、隐私保护工具
 
+#### 4.0 changelog
+
+- 开始支持检查无效域名，进一步降低最终生成文件的体积
+- 分离国内域名和海外域名，可以选择不同屏蔽等级
+- 代码重构，工程化，分离class，分离工具，逻辑更清晰
+
+
 #### 3.0 changlog
 
 - 严格匹配域名，增强生成列表的有效性

make-addr.php

@@ -11,7 +11,7 @@
 
 set_time_limit(600);
 
-error_reporting(7);
+error_reporting(0);
 
 if(PHP_SAPI != 'cli'){
 	die('nothing.');

tools/start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source /etc/profile
+cd $(cd "$(dirname "$0")";pwd)
+
+cd ../origin-files
+
+rm -f split-tmp-list_*
+
+split -l 5000 ../adblock-for-dnsmasq.conf split-tmp-list_
+
+cd ../tools
+
+# shellcheck disable=SC2045
+for f in $(ls ../origin-files/split-tmp-list_*)
+do
+  nohup php valid-addr.php $(basename "$f") >> ./std-$(basename "$f").out &
+done

tools/valid-addr.php

@@ -1,50 +1,110 @@
 <?php
+/**
+ * address dns checker
+ *
+ * @file valid-addr.php
+ * @date 2019-12-14
+ * @author gently
+ *
+ */
+set_time_limit(0);
+
+error_reporting(0);
+
+if (PHP_SAPI != 'cli') {
+    die('nothing.');
+}
+
+$src_file = '';
+$file_num = 'unknown';
+try {
+    $file = $argv[1];
+    $src_file = dirname(__DIR__) . '/origin-files/' . $file;
+    list(,$file_num) = explode('_', $file);
+} catch (Exception $e) {
+    echo "get args failed.", $e->getMessage(), "\n";
+    die(0);
+}
+
+if (empty($src_file) || !is_file($src_file)) {
+    echo 'src_file:', $src_file, ' is not found.';
+    die(0);
+}
+
+define('START_TIME', microtime(true));
+define('CHINA_LIST', dirname(__DIR__) . '/origin-files/china-list_' . $file_num . '.php');
+define('DEAD_HORSE', dirname(__DIR__) . '/origin-files/dead-horse_' . $file_num . '.php');
 
 //http://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.BIN.ZIP
-
 require_once './IP2Location.php';
 require_once './Net/DNS2.php';
 
+$china_list = is_file(CHINA_LIST) ? require CHINA_LIST : array();
+$dead_horse = is_file(DEAD_HORSE) ? require DEAD_HORSE : array();
 
-$db = new \IP2Location\Database('./databases/IP2LOCATION-LITE-DB1.BIN', \IP2Location\Database::FILE_IO);
-$r = new Net_DNS2_Resolver(array('nameservers' => array('223.5.5.5', '223.6.6.6')));
+//大中华区
+$CHINA_AREA = array('CN' => true, 'HK' => true, 'MO' => true, 'TW' => true);
 
-$src_fp = fopen('../adblock-for-dnsmasq.conf', 'r');
 
-while(!feof($src_fp)){
-  $row = fgets($src_fp, 512);
-  if(empty($row)){
-    continue;
-  }
+$db = new \IP2Location\Database('./databases/IP2LOCATION-LITE-DB1.BIN', \IP2Location\Database::MEMORY_CACHE);
+$r = new Net_DNS2_Resolver(array('nameservers' => array('223.5.5.5', '223.6.6.6', '119.29.29.29')));
 
-  if(preg_match('/^address=\/(.+)?\/$/', $row, $matchs)){
-    try{
-      $result = $r->query($matchs[1], 'A');
-      $result = $result->answer;
+$src_fp = fopen($src_file, 'r');
 
-      if(is_array($result) && count($result) > 0){
-
-        //find the A record
-        $a_record = null;
-        foreach($result as $res){
-          if($res->type == 'A'){
-            $a_record = $res;
-            break;
-          }
-        }
-
-        if(!$a_record){
-          echo '未找到A记录: ', $matchs[1], "\n";
-          continue;
-        }
-        $records = $db->lookup($a_record->address, \IP2Location\Database::ALL);
-        echo $a_record->address,"\t", $matchs[1],"\t",$records['countryCode'], "\t", $records['countryName'] ,"\n";
-      }else{
-        echo '记录为空: ', $matchs[1], "\n";
-      }
-    }catch(Net_DNS2_Exception $e){
-      //@TODO code=3的时候再试一次
-      echo "failed: ", $matchs[1],"\t", $e->getMessage(),"\t", $e->getCode(), "\n";
+while (!feof($src_fp)) {
+    $row = fgets($src_fp, 512);
+    if (empty($row)) {
+        continue;
+    }
+
+    echo $row;
+
+    if (preg_match('/^address=\/(.+)?\/$/', $row, $matches)) {
+        try {
+            $result = $r->query($matches[1], 'A');
+            $result = $result->answer;
+
+            if (is_array($result) && count($result) > 0) {
+
+                //find the A record
+                $a_record = null;
+                foreach ($result as $res) {
+                    if ($res->type == 'A') {
+                        $a_record = $res;
+                        break;
+                    }
+                }
+
+                if (!$a_record) {
+                    $dead_horse[$matches[1]]['empty']++;
+                    continue;
+                }
+                $records = $db->lookup($a_record->address, \IP2Location\Database::ALL);
+
+                if (array_key_exists($records['countryCode'], $CHINA_AREA) && !array_key_exists($matches[1], $china_list)) {
+                    $china_list[$matches[1]] = $records['countryCode'];//$records['countryName'];
+                }
+            } else {
+                $dead_horse[$matches[1]]['empty']++;
+            }
+        } catch (Net_DNS2_Exception $e) {
+            //3=dns记录不存在
+            if ($e->getCode() == 3) {
+                $dead_horse[$matches[1]]['dead']++;
+            } else {
+                echo date('m-d H:i:s'), "[", $matches[1], "]", $e->getMessage(), ",code:", $e->getCode(), "\n";
+            }
+        }
     }
-  }
 }
+
+try {
+    $dead_horse = "<?php \nreturn " . var_export($dead_horse, true) . ';';
+    $china_list = "<?php \nreturn " . var_export($china_list, true) . ';';
+    file_put_contents(DEAD_HORSE, $dead_horse);
+    file_put_contents(CHINA_LIST, $china_list);
+} catch (Exception $e) {
+    echo date('m-d H:i:s'), "write file failed:", $e->getMessage(), "\t", $e->getCode(), "\n";
+}
+
+echo 'Time cost:', microtime(true) - START_TIME, "s\n";