From 5bd662bc9e35517e1ca888f3c1e8d720d40aacc9 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 21:38:12 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[update]=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=E6=BA=90=20FreeProxyList?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/changelog.md | 1 + fetcher/sources/freeproxylist.py | 47 ------------------------------ tests/unit/test_fetcher_sources.py | 18 +----------- 4 files changed, 3 insertions(+), 65 deletions(-) delete mode 100644 fetcher/sources/freeproxylist.py diff --git a/README.md b/README.md index 19b61358..78ae7a3f 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ class MyProxyFetcher(BaseFetcher): |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeproxylist.py`](/fetcher/sources/freeproxylist.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | diff --git a/docs/changelog.md b/docs/changelog.md index 791d12f1..2867c455 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 8. 新增代理源 **Proxifly**; (2026-06-01) +9. 移除失效代理源 **FreeProxyList**; (2026-06-02) ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/freeproxylist.py b/fetcher/sources/freeproxylist.py deleted file mode 100644 index e02b6e17..00000000 --- a/fetcher/sources/freeproxylist.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： freeproxylist.py - Description : FreeProxyList代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -import re -from urllib import parse - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class FreeProxyListFetcher(BaseFetcher): - """FreeProxyList https://www.freeproxylists.net/zh/""" - - name = "freeproxylist" - url = "https://www.freeproxylists.net/zh/" - - @staticmethod - def _parse_ip(input_str): - html_str = parse.unquote(input_str) - ips = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', html_str) - return ips[0] if ips else None - - def fetch(self): - url = ("https://www.freeproxylists.net/zh/" - "?c=CN&pt=&pr=&a%5B%5D=0&a%5B%5D=1&a%5B%5D=2&u=50") - tree = WebRequest().get(url, verify=False).tree - for tr in tree.xpath("//tr[@class='Odd']") + tree.xpath("//tr[@class='Even']"): - ip = self._parse_ip("".join(tr.xpath('./td[1]/script/text()')).strip()) - port = "".join(tr.xpath('./td[2]/text()')).strip() - if ip: - yield "%s:%s" % (ip, port) - - -if __name__ == '__main__': - for proxy in FreeProxyListFetcher().fetch(): - print(proxy) diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index 88ea4d99..d074a7f5 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -63,7 +63,7 @@ class TestFetcherInterface(object): ("fetcher.sources.docip", "DocipFetcher"), ("fetcher.sources.goodips", "GoodipsFetcher"), ("fetcher.sources.geonode", "GeonodeFetcher"), - ("fetcher.sources.freeproxylist", "FreeProxyListFetcher"), + ("fetcher.sources.kuaidaili", "KuaidailiFetcher"), ("fetcher.sources.freevpnnode", "FreeVPNNodeFetcher"), ("fetcher.sources.scdn", "ScdnFetcher"), @@ -201,22 +201,6 @@ def test_fetch_text_fallback(self, mock_wr): assert "1.2.3.4:8080" in result -class TestFreeProxyListFetcher(object): - - @patch("fetcher.sources.freeproxylist.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.freeproxylist import FreeProxyListFetcher - # FreeProxyList 使用 JS 混淆 IP，这里模拟 script 标签中包含编码后的 IP - script_content = "document.write('%31%2E%32%2E%33%2E%34')" - html = '
8080
' % script_content - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(FreeProxyListFetcher().fetch()) - # 注意：实际 JS 解码逻辑可能需要更复杂的 mock - # 这里主要验证 fetch() 不报错且返回列表 - assert isinstance(result, list) - - class TestKuaidailiFetcher(object): @patch("fetcher.sources.kuaidaili.WebRequest") From e7488e216557a972699b64fe63e516db23992688 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 22:05:42 +0800 Subject: [PATCH 2/3] fix: pin werkzeug<2.2 to fix url_quote ImportError with Flask 2.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f648f70b..c658ae71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ APScheduler==3.10.0;python_version>="3.10" APScheduler==3.2.0;python_version<"3.10" click==8.0.1 Flask==2.1.1 -werkzeug==3.1.6 +werkzeug>=2.0,<2.2 From 72b755f335b0c458cdf57ca6b637720ff172487d Mon Sep 17 00:00:00 2001 From: jhao104 Date: Tue, 2 Jun 2026 22:23:58 +0800 Subject: [PATCH 3/3] docs: update badges - tests badge point to default branch, add codecov badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78ae7a3f..c30e013d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ ProxyPool 爬虫代理IP池 ======= -[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![codecov](https://codecov.io/gh/jhao104/proxy_pool/graph/badge.svg?token=8WHGkrQA6E)](https://codecov.io/gh/jhao104/proxy_pool) [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/) [![Packagist](https://img.shields.io/packagist/l/doctrine/orm.svg)](https://github.com/jhao104/proxy_pool/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/jhao104/proxy_pool.svg)](https://github.com/jhao104/proxy_pool/graphs/contributors)