From c89bea7202eb9bb3d98df4b86530611d4f92b321 Mon Sep 17 00:00:00 2001
From: gently <1243610+gentlyxu@users.noreply.github.com>
Date: Fri, 17 Jan 2020 14:31:06 +0800
Subject: [PATCH] resolve duplicate white list

---
 adblock-for-dnsmasq.conf  |  2 +-
 anti-ad-easylist.txt      | 80 +++++----------------------------------
 anti-ad-surge.txt         |  2 +-
 tools/easylist-extend.php |  7 +++-
 4 files changed, 18 insertions(+), 73 deletions(-)

diff --git a/adblock-for-dnsmasq.conf b/adblock-for-dnsmasq.conf
index b32ef6b8..9f76d77f 100644
--- a/adblock-for-dnsmasq.conf
+++ b/adblock-for-dnsmasq.conf
@@ -1,4 +1,4 @@
-#TIME=20200117051510
+#TIME=20200117062832
 #URL=https://github.com/privacy-protection-tools/anti-AD
 address=/0024aaaa.com/
 address=/0026645142c89aeb1.com/
diff --git a/anti-ad-easylist.txt b/anti-ad-easylist.txt
index 94e76b10..989a8cc6 100644
--- a/anti-ad-easylist.txt
+++ b/anti-ad-easylist.txt
@@ -1,4 +1,4 @@
-!TIME=20200117051510
+!TIME=20200117062832
 !URL=https://github.com/privacy-protection-tools/anti-AD
 ||0024aaaa.com^
 ||0026645142c89aeb1.com^
@@ -297,7 +297,7 @@
 ||r.mail.ru^
 ||rs.mail.ru^
 ||top-fwz1.mail.ru^
-/^ad([0-9]|m)?\.[0-9a-z\.\-]+\.([a-z]+)?$/
+/^ad([0-9]|m|s)?\.[0-9a-z\.\-]+\.([a-z]+)?$/
 ||1lib.cn^
 ||1lo0.net^
 ||1loop.com^
@@ -1664,7 +1664,6 @@
 ||adacts.com^
 ||adadmin.house365.com^
 ||stat.house365.com^
-||ads.house365.com^
 ||adadvisor.net^
 ||adagora.com^
 ||adakgpoi.com^
@@ -2021,7 +2020,6 @@
 ||admddc.com^
 ||admd.housefun.com.tw^
 ||admd.yam.com^
-||ads.yam.com^
 ||admedia.com^
 ||admedit.net^
 ||admedo.com^
@@ -2240,7 +2238,6 @@
 ||ad-score.com^
 ||adscpm.net^
 ||adsc.wasu.tv^
-||ads.wasu.tv^
 ||adsystem.wasu.tv^
 ||adwasu.wasu.tv^
 ||countatm.cp12.wasu.tv^
@@ -2312,7 +2309,6 @@
 ||adslala.com^
 ||adslidango.com^
 ||adslingers.com^
-||ads.linkedin.com^
 ||platform.linkedin.com^
 ||adslivecorp.com^
 ||adslot.com^
@@ -2356,8 +2352,6 @@
 ||adss.dotdo.net^
 ||adssend.net^
 ||servebom.com^
-||ads.sexier.com^
-||ads.sohu.com^
 ||assp.sohu.com^
 ||count.vrs.sohu.com^
 ||fpb.sohu.com^
@@ -2393,7 +2387,6 @@
 ||vg.aty.sohu.com^
 ||x1.go.sohu.com^
 ||adss.yahoo.com^
-||ads.yahoo.com^
 ||bats.video.yahoo.com^
 ||beap-bc.yahoo.com^
 ||gemini.yahoo.com^
@@ -2432,7 +2425,6 @@
 ||adstuna.com^
 ||ads-twitter.com^
 ||ad.style^
-||ads.uc.cn^
 ||bench.uc.cn^
 ||track.uc.cn^
 ||alissl.ucdl.pp.uc.cn^
@@ -2447,7 +2439,6 @@
 ||ads-union.jd.com^
 ||jcm.jd.com^
 ||wl.jd.com^
-||ads.union.jd.com^
 ||x.jd.com^
 ||cm.jd.com^
 ||du.jd.com^
@@ -2464,7 +2455,6 @@
 ||ad-survey.com^
 ||adsvcs.com^
 ||adsvert.com^
-||ads.videosz.com^
 ||banners.videosz.com^
 ||feeds.videosz.com^
 ||galleries.videosz.com^
@@ -2475,7 +2465,6 @@
 ||adsymptotic.com^
 ||adsyst.biz^
 ||adszom.com^
-||ads.zynga.com^
 ||adt100.com^
 ||adtag.cc^
 ||adtaily.com^
@@ -4611,7 +4600,6 @@
 ||bavftcgoapga.com^
 ||b-aws.aol.com^
 ||adinfo.aol.com^
-||ads.web.aol.com^
 ||advertising.aol.com^
 ||cdn.komentary.aol.com^
 ||dynamic.aol.com^
@@ -5753,8 +5741,6 @@
 ||ads1.msn.com^
 ||ads2.msn.com^
 ||ac3.msn.com^
-||ads.eu.msn.com^
-||ads.msn.com^
 ||adsyndication.msn.com^
 ||catalog.video.msn.com^
 ||c.msn.com^
@@ -6425,7 +6411,6 @@
 ||cl.expedia.com^
 ||collector.prod.expedia.com^
 ||oms.expedia.com^
-||ads.expedia.com^
 ||clgserv.pro^
 ||clgyhwaoh.com^
 ||clic2pub.com^
@@ -8154,7 +8139,6 @@
 ||dealswarm.com^
 ||dealzone.co.za^
 ||de.as.pptv.com^
-||ads.aplus.pptv.com^
 ||app.aplus.pptv.com^
 ||as.aplus.pptv.com^
 ||img1.g.pptv.com^
@@ -10929,7 +10913,6 @@
 ||geobeacon.ign.com^
 ||g.ign.com^
 ||wrapper.ign.com^
-||ads.ign.com^
 ||geobytes.com^
 ||geocompteur.com^
 ||geo.connexionsecure.com^
@@ -11086,9 +11069,7 @@
 ||ggijrjktcld.com^
 ||gg.jkmeishi.com^
 ||gg.kugou.com^
-||ads.bssdl.kugou.com^
 ||adserviceretry.kugou.com^
-||ads.service.kugou.com^
 ||fanxing.kugou.com^
 ||d.kugou.com^
 ||gamebox.kugou.com^
@@ -11373,7 +11354,6 @@
 ||plusone.google.com^
 ||video-stats.video.google.com^
 ||adservice.google.com^
-||ads.google.com^
 ||afd.l.google.com^
 ||mobileads.google.com^
 ||pagead.google.com^
@@ -17421,7 +17401,6 @@
 ||nonstoppartner.de^
 ||nonxt1.c.youtube.com^
 ||s.youtube.com^
-||ads.youtube.com^
 ||noolablkcuyu.com^
 ||noonshdnkt.bid^
 ||noosaphy.net^
@@ -20147,7 +20126,6 @@
 ||puzzlingfall.com^
 ||pvclouds.com^
 ||pv.csdn.net^
-||ads.csdn.net^
 ||dc2.csdn.net^
 ||dc.csdn.net^
 ||kunpeng.csdn.net^
@@ -20692,7 +20670,6 @@
 ||qorvoxejr.com^
 ||qosrridifvsr.com^
 ||qos.video.yimg.com^
-||ads.yimg.com^
 ||qotuhsvlqlpnfd.com^
 ||qotwtnckqrke.com^
 ||qouiruhpxoa.com^
@@ -31799,7 +31776,6 @@
 ||img1.nend.net^
 ||js1.nend.net^
 ||output.nend.net^
-||ads.sina.com^
 ||oascentral.sina.com^
 ||pfpip.sina.com^
 ||ad-android.51wnl.com^
@@ -31926,7 +31902,6 @@
 ||admin.louxia.org^
 ||admin.ninebox.cn^
 ||admin.vserv.mobi^
-||ads.vserv.mobi^
 ||a.vserv.mobi^
 ||c.vserv.mobi^
 ||sf.vserv.mobi^
@@ -31953,42 +31928,26 @@
 ||ads1.lfengmobile.com^
 ||ads2.opensubtitles.org^
 ||ads8.com^
-||ads.979799777.com^
-||ads.adadapted.com^
-||ads.adinall.com^
 ||app-tj.adinall.com^
 ||cdn.adinall.com^
 ||pmp.adinall.com^
-||ads.admarvel.com^
-||ads.admoda.com^
-||ads.adxadserv.com^
 ||static.adxadserv.com^
 ||adsame1.cnr.cn^
 ||d1.cnr.cn^
 ||ggdata1.cnr.cn^
-||ads.aol.co.uk^
 ||adsapi.manhuaren.com^
 ||videojj.com^
 ||starwave.com^
-||ads.b10f.jp^
-||ads.ccstock.cn^
-||ads.cdn.rovio.com^
-||ads.ce.cn^
-||ads.celtra.com^
 ||cache.celtra.com^
 ||cache-ssl.celtra.com^
 ||cdn.celtra.com^
 ||track.celtra.com^
 ||adscript.gmarket.co.kr^
-||ads.cyberctm.com^
-||ads.data.pplive.com^
 ||gas.data.pplive.com^
 ||plt.data.pplive.com^
 ||ppvabs.pplive.com^
 ||web.data.pplive.com^
-||ads.dixcom.com^
 ||adsdk.9imobi.com^
-||ads.dvworld.com.tw^
 ||adserve2.tom.com^
 ||js.pub.tom.com^
 ||slides.discovery.tom.com^
@@ -32003,8 +31962,6 @@
 ||adserviceretry.kglink.cn^
 ||adservice.sigmob.cn^
 ||adse.ximalaya.com^
-||ads.feelingtouch.com^
-||ads.fetnet.net^
 ||adsfs.oppomobile.com^
 ||adx.ads.oppomobile.com^
 ||bdapi.ads.oppomobile.com^
@@ -32012,11 +31969,7 @@
 ||httpdns.push.oppomobile.com^
 ||i.flow.browser.oppomobile.com^
 ||imagzine.oppomobile.com^
-||ads.gionee.com^
 ||pdl.gionee.com^
-||ads.globo.com^
-||ads.gmodules.com^
-||ads.heyzap.com^
 ||adshow.58.com^
 ||brandshow.58.com^
 ||jing.58.com^
@@ -32029,47 +31982,26 @@
 ||adsmart.yicha.cn^
 ||tongji.yicha.cn^
 ||tuiguang.yicha.cn^
-||ads.matomymobile.com^
-||ads.mdotm.com^
 ||cdn.mdotm.com^
 ||miidi.net^
-||ads.mobclix.com^
 ||data.mobclix.com^
 ||s.mobclix.com^
-||ads.mobilityware.com^
 ||apps.mobilityware.com^
 ||adsmogo.com^
 ||mopub.com^
-||ads.ninemsn.com.au^
-||ads.n-ws.org^
-||ads.oneway.mobi^
-||ads.ookla.com^
-||ads.passiontimes.hk^
 ||ad-specs.guoshipartners.com^
-||ads.playboy.com^
 ||pei-ads.playboy.com^
-||ads.supfast.net^
 ||apk.supfast.net^
 ||apps.supfast.net^
 ||click.supfast.net^
 ||img.supfast.net^
 ||l.supfast.net^
-||ads.tripod.lycos.co.uk^
 ||fe.lea.lycos.co.uk^
 ||hit.webcentre.lycos.co.uk^
-||ads.tripod.lycos.de^
-||ads.tripod.lycos.es^
-||ads.tripod.lycos.nl^
-||ads.ttv.com.tw^
 ||adsunflower.com^
-||ads.unister-gmbh.de^
-||ads.vipbaihe.com^
 ||voiceads.cn^
-||ads.waps.cn^
 ||app.waps.cn^
 ||wapx.cn^
-||ads.wepiao.com^
-||ads.xxxad.net^
 ||adsys.chinacloudapp.cn^
 ||xdrig.com^
 ||adui.tg.meitu.com^
@@ -35757,3 +35689,11 @@
 @@||ad.filmweb.pl^
 @@||ad.kazakinfo.com^
 @@||ad.ourgame.com^
+@@||ads.askgamblers.com^
+@@||ads.fuckingmachines.com^
+@@||ads.microsoft.com^
+@@||ads.nipr.ac.jp^
+@@||ads.pinterest.com^
+@@||ads.tiktok.com^
+@@||ads.twitter.com^
+@@||ads.ultimatesurrender.com^
diff --git a/anti-ad-surge.txt b/anti-ad-surge.txt
index 6042708b..bfab5c96 100644
--- a/anti-ad-surge.txt
+++ b/anti-ad-surge.txt
@@ -1,4 +1,4 @@
-#TIME=20200117051511
+#TIME=20200117062832
 #URL=https://github.com/privacy-protection-tools/anti-AD
 [RULE]
 DOMAIN-SUFFIX,0024aaaa.com,REJECT
diff --git a/tools/easylist-extend.php b/tools/easylist-extend.php
index 56fa7b5e..b8cebde8 100644
--- a/tools/easylist-extend.php
+++ b/tools/easylist-extend.php
@@ -63,7 +63,7 @@ $ARR_REGEX_LIST = array(
     '/^9377[a-z]{2}\.com$/' => null,
 //    '/^[1-3]\.[0-9a-z\.\-]+\.(com|cn|net|org)$/' => null,
 //    '/^a1\.[0-9a-z\.]+\.(com|cn|org|net|me)$/' => null,
-    '/^ad([0-9]|m)?\.[0-9a-z\.\-]+\.([a-z]+)?$/' => null,
+    '/^ad([0-9]|m|s)?\.[0-9a-z\.\-]+\.([a-z]+)?$/' => null,
     '/^affiliat(es|ion|e)\..+$/' => null,
     '/^afgr[0-9]{1,2}\.com$/' => null,
     '/^analytics(\-|\.).+$/' => null,
@@ -164,6 +164,7 @@ while(!feof($src_fp)){
 }
 
 //按需写入白名单规则
+$wrote_whitelist = array();
 $whiterule = file(WHITERULE_SRC, FILE_SKIP_EMPTY_LINES);
 $ARR_WHITE_RULE_LIST = array_merge($ARR_WHITE_RULE_LIST, $whiterule);
 foreach ($ARR_WHITE_RULE_LIST as $row){
@@ -188,6 +189,10 @@ foreach ($ARR_WHITE_RULE_LIST as $row){
             ){
                 continue;
             }
+            if(array_key_exists($matches[1], $wrote_whitelist)){
+                continue;
+            }
+            $wrote_whitelist[$matches[1]] = null;
             fwrite($new_fp, "@@||${matches[1]}^\n");
         }
     }