检查域名有效性逻辑

.gitignore

@@ -2,3 +2,4 @@
 optimize-dns.conf
 .gitignore
 .idea/
+origin-files/split-tmp-list_*

README.md

@@ -1,7 +1,14 @@
-# anti-AD v3.0
+# anti-AD v4.0
 
 ### 基于dnsmasq服务的广告封杀、恶意网站屏蔽、隐私保护工具
 
+#### 4.0 changelog
+
+- 开始支持检查无效域名，进一步降低最终生成文件的体积
+- 分离国内域名和海外域名，可以选择不同屏蔽等级
+- 代码重构，工程化，分离class，分离工具，逻辑更清晰
+
+
 #### 3.0 changlog
 
 - 严格匹配域名，增强生成列表的有效性

make-addr.php

@@ -11,7 +11,7 @@
 
 set_time_limit(600);
 
-error_reporting(7);
+error_reporting(0);
 
 if(PHP_SAPI != 'cli'){
 	die('nothing.');

tools/start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source /etc/profile
+cd $(cd "$(dirname "$0")";pwd)
+
+cd ../origin-files
+
+rm -f split-tmp-list_*
+
+split -l 5000 ../adblock-for-dnsmasq.conf split-tmp-list_
+
+cd ../tools
+
+# shellcheck disable=SC2045
+for f in $(ls ../origin-files/split-tmp-list_*)
+do
+  nohup php valid-addr.php $(basename "$f") >> ./std-$(basename "$f").out &
+done

tools/valid-addr.php

@@ -1,15 +1,55 @@
 <?php
+/**
+ * address dns checker
+ *
+ * @file valid-addr.php
+ * @date 2019-12-14
+ * @author gently
+ *
+ */
+set_time_limit(0);
+
+error_reporting(0);
+
+if (PHP_SAPI != 'cli') {
+    die('nothing.');
+}
+
+$src_file = '';
+$file_num = 'unknown';
+try {
+    $file = $argv[1];
+    $src_file = dirname(__DIR__) . '/origin-files/' . $file;
+    list(,$file_num) = explode('_', $file);
+} catch (Exception $e) {
+    echo "get args failed.", $e->getMessage(), "\n";
+    die(0);
+}
+
+if (empty($src_file) || !is_file($src_file)) {
+    echo 'src_file:', $src_file, ' is not found.';
+    die(0);
+}
+
+define('START_TIME', microtime(true));
+define('CHINA_LIST', dirname(__DIR__) . '/origin-files/china-list_' . $file_num . '.php');
+define('DEAD_HORSE', dirname(__DIR__) . '/origin-files/dead-horse_' . $file_num . '.php');
 
 //http://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.BIN.ZIP
-
 require_once './IP2Location.php';
 require_once './Net/DNS2.php';
 
+$china_list = is_file(CHINA_LIST) ? require CHINA_LIST : array();
+$dead_horse = is_file(DEAD_HORSE) ? require DEAD_HORSE : array();
 
-$db = new \IP2Location\Database('./databases/IP2LOCATION-LITE-DB1.BIN', \IP2Location\Database::FILE_IO);
+//大中华区
-$r = new Net_DNS2_Resolver(array('nameservers' => array('223.5.5.5', '223.6.6.6')));
+$CHINA_AREA = array('CN' => true, 'HK' => true, 'MO' => true, 'TW' => true);
 
-$src_fp = fopen('../adblock-for-dnsmasq.conf', 'r');
+
+$db = new \IP2Location\Database('./databases/IP2LOCATION-LITE-DB1.BIN', \IP2Location\Database::MEMORY_CACHE);
+$r = new Net_DNS2_Resolver(array('nameservers' => array('223.5.5.5', '223.6.6.6', '119.29.29.29')));
+
+$src_fp = fopen($src_file, 'r');
 
 while (!feof($src_fp)) {
     $row = fgets($src_fp, 512);
@@ -17,9 +57,11 @@ while(!feof($src_fp)){
         continue;
     }
 
-  if(preg_match('/^address=\/(.+)?\/$/', $row, $matchs)){
+    echo $row;
+
+    if (preg_match('/^address=\/(.+)?\/$/', $row, $matches)) {
         try {
-      $result = $r->query($matchs[1], 'A');
+            $result = $r->query($matches[1], 'A');
             $result = $result->answer;
 
             if (is_array($result) && count($result) > 0) {
@@ -34,17 +76,35 @@ while(!feof($src_fp)){
                 }
 
                 if (!$a_record) {
-          echo '未找到A记录: ', $matchs[1], "\n";
+                    $dead_horse[$matches[1]]['empty']++;
                     continue;
                 }
                 $records = $db->lookup($a_record->address, \IP2Location\Database::ALL);
-        echo $a_record->address,"\t", $matchs[1],"\t",$records['countryCode'], "\t", $records['countryName'] ,"\n";
+
+                if (array_key_exists($records['countryCode'], $CHINA_AREA) && !array_key_exists($matches[1], $china_list)) {
+                    $china_list[$matches[1]] = $records['countryCode'];//$records['countryName'];
+                }
             } else {
-        echo '记录为空: ', $matchs[1], "\n";
+                $dead_horse[$matches[1]]['empty']++;
             }
         } catch (Net_DNS2_Exception $e) {
-      //@TODO code=3的时候再试一次
+            //3=dns记录不存在
-      echo "failed: ", $matchs[1],"\t", $e->getMessage(),"\t", $e->getCode(), "\n";
+            if ($e->getCode() == 3) {
+                $dead_horse[$matches[1]]['dead']++;
+            } else {
+                echo date('m-d H:i:s'), "[", $matches[1], "]", $e->getMessage(), ",code:", $e->getCode(), "\n";
             }
         }
     }
+}
+
+try {
+    $dead_horse = "<?php \nreturn " . var_export($dead_horse, true) . ';';
+    $china_list = "<?php \nreturn " . var_export($china_list, true) . ';';
+    file_put_contents(DEAD_HORSE, $dead_horse);
+    file_put_contents(CHINA_LIST, $china_list);
+} catch (Exception $e) {
+    echo date('m-d H:i:s'), "write file failed:", $e->getMessage(), "\t", $e->getCode(), "\n";
+}
+
+echo 'Time cost:', microtime(true) - START_TIME, "s\n";