diff --git a/README.md b/README.md index 19b61358..c30e013d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ ProxyPool 爬虫代理IP池 ======= -[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![Tests](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml/badge.svg)](https://github.com/jhao104/proxy_pool/actions/workflows/test.yml) +[![codecov](https://codecov.io/gh/jhao104/proxy_pool/graph/badge.svg?token=8WHGkrQA6E)](https://codecov.io/gh/jhao104/proxy_pool) [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/) [![Packagist](https://img.shields.io/packagist/l/doctrine/orm.svg)](https://github.com/jhao104/proxy_pool/blob/master/LICENSE) [![GitHub contributors](https://img.shields.io/github/contributors/jhao104/proxy_pool.svg)](https://github.com/jhao104/proxy_pool/graphs/contributors) @@ -197,7 +198,7 @@ class MyProxyFetcher(BaseFetcher): |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | - | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeproxylist.py`](/fetcher/sources/freeproxylist.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | diff --git a/docs/changelog.md b/docs/changelog.md index 791d12f1..2867c455 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 8. 新增代理源 **Proxifly**; (2026-06-01) +9. 移除失效代理源 **FreeProxyList**; (2026-06-02) ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/freeproxylist.py b/fetcher/sources/freeproxylist.py deleted file mode 100644 index e02b6e17..00000000 --- a/fetcher/sources/freeproxylist.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name: freeproxylist.py - Description : FreeProxyList代理源 - Author : JHao - date: 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -import re -from urllib import parse - -from fetcher.baseFetcher import BaseFetcher -from util.webRequest import WebRequest - - -class FreeProxyListFetcher(BaseFetcher): - """FreeProxyList https://www.freeproxylists.net/zh/""" - - name = "freeproxylist" - url = "https://www.freeproxylists.net/zh/" - - @staticmethod - def _parse_ip(input_str): - html_str = parse.unquote(input_str) - ips = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', html_str) - return ips[0] if ips else None - - def fetch(self): - url = ("https://www.freeproxylists.net/zh/" - "?c=CN&pt=&pr=&a%5B%5D=0&a%5B%5D=1&a%5B%5D=2&u=50") - tree = WebRequest().get(url, verify=False).tree - for tr in tree.xpath("//tr[@class='Odd']") + tree.xpath("//tr[@class='Even']"): - ip = self._parse_ip("".join(tr.xpath('./td[1]/script/text()')).strip()) - port = "".join(tr.xpath('./td[2]/text()')).strip() - if ip: - yield "%s:%s" % (ip, port) - - -if __name__ == '__main__': - for proxy in FreeProxyListFetcher().fetch(): - print(proxy) diff --git a/requirements.txt b/requirements.txt index f648f70b..c658ae71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ APScheduler==3.10.0;python_version>="3.10" APScheduler==3.2.0;python_version<"3.10" click==8.0.1 Flask==2.1.1 -werkzeug==3.1.6 +werkzeug>=2.0,<2.2 diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index 88ea4d99..d074a7f5 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -63,7 +63,7 @@ class TestFetcherInterface(object): ("fetcher.sources.docip", "DocipFetcher"), ("fetcher.sources.goodips", "GoodipsFetcher"), ("fetcher.sources.geonode", "GeonodeFetcher"), - ("fetcher.sources.freeproxylist", "FreeProxyListFetcher"), + ("fetcher.sources.kuaidaili", "KuaidailiFetcher"), ("fetcher.sources.freevpnnode", "FreeVPNNodeFetcher"), ("fetcher.sources.scdn", "ScdnFetcher"), @@ -201,22 +201,6 @@ def test_fetch_text_fallback(self, mock_wr): assert "1.2.3.4:8080" in result -class TestFreeProxyListFetcher(object): - - @patch("fetcher.sources.freeproxylist.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.freeproxylist import FreeProxyListFetcher - # FreeProxyList 使用 JS 混淆 IP,这里模拟 script 标签中包含编码后的 IP - script_content = "document.write('%31%2E%32%2E%33%2E%34')" - html = '
8080
' % script_content - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(FreeProxyListFetcher().fetch()) - # 注意:实际 JS 解码逻辑可能需要更复杂的 mock - # 这里主要验证 fetch() 不报错且返回列表 - assert isinstance(result, list) - - class TestKuaidailiFetcher(object): @patch("fetcher.sources.kuaidaili.WebRequest")