From 165c361c8e1ded0d13f8020e81c72bb6e581885d Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 8 Jun 2026 21:27:29 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E6=BA=90daili66,=20=E7=A7=BB=E9=99=A4=E5=A4=B1=E6=95=88?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E6=BA=90ip66?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 fetcher/sources/daili66.py (66代理 JSON API) - 删除 fetcher/sources/ip66.py (已失效) - 新增 TestDaiLi66Fetcher 测试类 - 更新 README.md、docs/project-structure.md、docs/changelog.md --- README.md | 26 +++++++++--------- docs/changelog.md | 2 ++ docs/project-structure.md | 1 - fetcher/sources/daili66.py | 42 ++++++++++++++++++++++++++++++ fetcher/sources/freevpnnode.py | 4 +-- fetcher/sources/ip66.py | 37 -------------------------- tests/unit/test_fetcher_sources.py | 41 +++++++++++++++++------------ 7 files changed, 85 insertions(+), 68 deletions(-) create mode 100644 fetcher/sources/daili66.py delete mode 100644 fetcher/sources/ip66.py diff --git a/README.md b/README.md index 6f68b7cf..f8869590 100644 --- a/README.md +++ b/README.md @@ -194,18 +194,20 @@ class MyProxyFetcher(BaseFetcher): 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): - | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | - |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| - | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | - | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | - | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | - | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | - | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | + |-------------| ---- | -------- |----|----------------------------------|-----------------------------------------------------| + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | + | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | 66代理 | ✔ | ★★ | * | [地址](https://www.66daili.com) | [`daili66.py`](/fetcher/sources/daili66.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | + | FreeVPNNode | ✔ | ★★ | * | [地址](https://cn.freevpnnode.com) | [`freevpnnode.py`](/fetcher/sources/freevpnnode.py) | + | Geonode | ✔ | ★★ | ** | [地址](https://geonode.com) | [`geonode.py`](/fetcher/sources/geonode.py) | 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。 diff --git a/docs/changelog.md b/docs/changelog.md index 2867c455..453e515d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,8 @@ - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 8. 新增代理源 **Proxifly**; (2026-06-01) 9. 移除失效代理源 **FreeProxyList**; (2026-06-02) +10. 新增代理源 **66代理 (daili66)**; (2026-06-08) +11. 移除失效代理源 **66代理 (ip66)**; (2026-06-08) ## 2.4.2 (2024-01-18) diff --git a/docs/project-structure.md b/docs/project-structure.md index 795650b8..30077d8b 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -14,7 +14,6 @@ proxy_pool/ │ ├── baseFetcher.py # BaseFetcher 基类（共享解析方法） │ └── sources/ # 各代理源独立文件 │ ├── zdaye.py # 站大爷 -│ ├── ip66.py # 代理66 │ ├── kxdaili.py # 开心代理 │ ├── kuaidaili.py # 快代理 │ ├── geonode.py # Geonode diff --git a/fetcher/sources/daili66.py b/fetcher/sources/daili66.py new file mode 100644 index 00000000..6e6dd005 --- /dev/null +++ b/fetcher/sources/daili66.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： daili66.py + Description : 66代理 + Author : JHao + date： 2026/06/08 +------------------------------------------------- + Change Activity: + 2026/06/08: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + +class DaiLi66Fetcher(BaseFetcher): + """66代理 https://www.66daili.com""" + + name = "daili66" + url = "https://www.66daili.com" + + enabled = True + + def fetch(self): + url = "http://api.66daili.com/?format=json" + r = WebRequest().get(url, timeout=10) + try: + for each in r.json.get("data", []): + yield "%s:%s" % (each["ip"], each["port"]) + except Exception as e: + logger.error("ProxyFetch - daili66: %s" % e) + + + +if __name__ == '__main__': + for proxy in DaiLi66Fetcher().fetch(): + print(proxy) diff --git a/fetcher/sources/freevpnnode.py b/fetcher/sources/freevpnnode.py index 45e4e461..27e871e0 100644 --- a/fetcher/sources/freevpnnode.py +++ b/fetcher/sources/freevpnnode.py @@ -19,10 +19,10 @@ class FreeVPNNodeFetcher(BaseFetcher): - """FreeVPNNode https://cn.freevpnnode.com/free-proxy/""" + """FreeVPNNode https://cn.freevpnnode.com""" name = "freevpnnode" - url = "https://cn.freevpnnode.com/free-proxy/" + url = "https://cn.freevpnnode.com" def fetch(self): url = "https://cn.freevpnnode.com/free-proxy/" diff --git a/fetcher/sources/ip66.py b/fetcher/sources/ip66.py deleted file mode 100644 index 3cec44f7..00000000 --- a/fetcher/sources/ip66.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： ip66.py - Description : 代理66代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class Ip66Fetcher(BaseFetcher): - """代理66 http://www.66ip.cn/""" - - name = "ip66" - url = "http://www.66ip.cn/" - - def fetch(self): - url = "http://www.66ip.cn/" - resp = WebRequest().get(url, timeout=10).tree - for i, tr in enumerate(resp.xpath("(//table)[3]//tr")): - if i > 0: - ip = "".join(tr.xpath("./td[1]/text()")).strip() - port = "".join(tr.xpath("./td[2]/text()")).strip() - yield "%s:%s" % (ip, port) - - -if __name__ == '__main__': - for proxy in Ip66Fetcher().fetch(): - print(proxy) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index abed28b2..eda33456 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -55,7 +55,6 @@ class TestFetcherInterface(object): """所有 fetcher 的接口约定""" FETCHER_CLASSES = [ - ("fetcher.sources.ip66", "Ip66Fetcher"), ("fetcher.sources.kxdaili", "KxdailiFetcher"), ("fetcher.sources.ip3366", "Ip3366Fetcher"), ("fetcher.sources.jiangxianli", "JiangxianliFetcher"), @@ -70,6 +69,7 @@ class TestFetcherInterface(object): ("fetcher.sources.zdaye", "ZdayeFetcher"), ("fetcher.sources.ihuan", "IhuanFetcher"), ("fetcher.sources.proxifly", "ProxiFlyFetcher"), + ("fetcher.sources.daili66", "DaiLi66Fetcher"), ] def test_all_fetchers_have_name_url_enabled(self): @@ -95,21 +95,6 @@ def test_all_fetchers_have_fetch_method(self): # --------------- 各 fetcher 逻辑测试 --------------- -class TestIp66Fetcher(object): - - @patch("fetcher.sources.ip66.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.ip66 import Ip66Fetcher - # ip66 使用 (//table)[3] 取第3个table，if i > 0 跳过第一行 - html = ("
" - + _html_table([("IP", "Port"), ("1.2.3.4", "8080"), ("5.6.7.8", "3128")])) - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(Ip66Fetcher().fetch()) - assert "1.2.3.4:8080" in result - assert "5.6.7.8:3128" in result - - class TestKxdailiFetcher(object): @patch("fetcher.sources.kxdaili.WebRequest") @@ -386,3 +371,27 @@ def test_fetch_filters_non_http(self, mock_wr): result = list(ProxiFlyFetcher().fetch()) assert "1.2.3.4:8080" in result assert "9.9.9.9:8080" not in result + + +class TestDaiLi66Fetcher(object): + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + json_data = { + "data": [ + {"ip": "1.2.3.4", "port": "8080"}, + {"ip": "5.6.7.8", "port": "3128"}, + ] + } + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(DaiLi66Fetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.daili66.WebRequest") + def test_fetch_empty_data_returns_empty(self, mock_wr): + from fetcher.sources.daili66 import DaiLi66Fetcher + mock_wr.return_value.get.return_value = _make_response(json_data={}) + result = list(DaiLi66Fetcher().fetch()) + assert result == []