From 5ba5d29887d4b10ef9bcab442e25939f8bdb6400 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 14:22:01 +0000 Subject: [PATCH 1/4] Bump werkzeug from 2.1.0 to 3.1.6 Bumps [werkzeug](https://github.com/pallets/werkzeug) from 2.1.0 to 3.1.6. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/2.1.0...3.1.6) --- updated-dependencies: - dependency-name: werkzeug dependency-version: 3.1.6 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad68b2c0..7e3276e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ APScheduler==3.10.0;python_version>="3.10" APScheduler==3.2.0;python_version<"3.10" click==8.0.1 Flask==2.1.1 -werkzeug==2.1.0 +werkzeug==3.1.6 From b393014ce8dfc26c50e3c8b75b3cb23999024b79 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 21:17:36 +0800 Subject: [PATCH 2/4] =?UTF-8?q?[update]=20=E7=A7=BB=E9=99=A4=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E4=BB=A3=E7=90=86=20=20=E5=86=B0=E5=87=8C=E4=BB=A3?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fetcher/sources/binglx.py | 41 ------------------------------ tests/unit/test_fetcher_sources.py | 14 ---------- 2 files changed, 55 deletions(-) delete mode 100644 fetcher/sources/binglx.py diff --git a/fetcher/sources/binglx.py b/fetcher/sources/binglx.py deleted file mode 100644 index 7458bfd5..00000000 --- a/fetcher/sources/binglx.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -""" -------------------------------------------------- - File Name： binglx.py - Description : 冰凌代理代理源 - Author : JHao - date： 2026/5/31 -------------------------------------------------- - Change Activity: - 2026/05/31: -------------------------------------------------- -""" -__author__ = 'JHao' - -from fetcher.baseFetcher import BaseFetcher -from handler.logHandler import LogHandler -from util.webRequest import WebRequest - -logger = LogHandler("fetcher") - - -class BinglxFetcher(BaseFetcher): - """冰凌代理 https://www.binglx.cn""" - - name = "binglx" - url = "https://www.binglx.cn/" - - def fetch(self): - url = "https://www.binglx.cn/?page=1" - try: - tree = WebRequest().get(url).tree - proxy_list = tree.xpath('.//table//tr') - for tr in proxy_list[1:]: - yield ':'.join(tr.xpath('./td/text()')[0:2]) - except Exception as e: - logger.error("ProxyFetch - binglx: %s" % e) - - -if __name__ == '__main__': - for proxy in BinglxFetcher().fetch(): - print(proxy) \ No newline at end of file diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index 453ca7af..b641fec8 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -57,7 +57,6 @@ class TestFetcherInterface(object): FETCHER_CLASSES = [ ("fetcher.sources.ip66", "Ip66Fetcher"), ("fetcher.sources.kxdaili", "KxdailiFetcher"), - ("fetcher.sources.binglx", "BinglxFetcher"), ("fetcher.sources.ip3366", "Ip3366Fetcher"), ("fetcher.sources.jiangxianli", "JiangxianliFetcher"), ("fetcher.sources.ip89", "Ip89Fetcher"), @@ -123,19 +122,6 @@ def test_fetch(self, mock_wr): assert "1.2.3.4:8080" in result -class TestBinglxFetcher(object): - - @patch("fetcher.sources.binglx.WebRequest") - def test_fetch(self, mock_wr): - from fetcher.sources.binglx import BinglxFetcher - # binglx 使用 proxy_list[1:] 跳过第一行，需要 header 行 - html = _html_table([("IP", "Port"), ("1.2.3.4", "8080")]) - tree = etree.HTML(html) - mock_wr.return_value.get.return_value = _make_response(tree=tree) - result = list(BinglxFetcher().fetch()) - assert "1.2.3.4:8080" in result - - class TestIp3366Fetcher(object): @patch("fetcher.sources.ip3366.WebRequest") From 1731f55221f38ef6b1e290356cf5c002d78b3896 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 22:29:00 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20Proxifly=20?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E6=BA=90=E5=8F=8A=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 fetcher/sources/proxifly.py 代理源 - 新增 TestProxiFlyFetcher 测试类（正常解析、非CN过滤、非http过滤） - TestFetcherInterface 注册 proxifly - 修复 url 拼写错误 hhttps -> https - 更新 changelog --- docs/changelog.md | 1 + fetcher/sources/proxifly.py | 42 ++++++++++++++++++++++++++++++ tests/unit/test_fetcher_sources.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 fetcher/sources/proxifly.py diff --git a/docs/changelog.md b/docs/changelog.md index 44ca09e2..791d12f1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,7 @@ - 自动扫描启用的代理源，无需在配置中列举 - 新增 `PROXY_FETCHER_EXCLUDE` 黑名单配置 - 新增 `proxyPool.py fetcher` 命令查看启用的代理源 +8. 新增代理源 **Proxifly**; (2026-06-01) ## 2.4.2 (2024-01-18) diff --git a/fetcher/sources/proxifly.py b/fetcher/sources/proxifly.py new file mode 100644 index 00000000..4f8780ba --- /dev/null +++ b/fetcher/sources/proxifly.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： proxifly.py + Description : Proxifly代理源 + Author : JHao + date： 2026/06/01 +------------------------------------------------- + Change Activity: + 2026/06/01: +------------------------------------------------- +""" +__author__ = 'JHao' + +from fetcher.baseFetcher import BaseFetcher +from handler.logHandler import LogHandler +from util.webRequest import WebRequest + +logger = LogHandler("fetcher") + + +class ProxiFlyFetcher(BaseFetcher): + """Proxifly https://proxifly.dev""" + + name = "proxifly" + url = "https://proxifly.dev/" + + enabled = True # 是否启用 + + def fetch(self): + r = WebRequest().get("https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json", timeout=10) + try: + for each in r.json: + if each.get("geolocation", {}).get("country", "") == "CN" and each.get("protocol") == "http": + yield self.parseProxiesFromText(each.get('proxy', "")).pop() + except Exception as e: + logger.error("ProxyFetch - proxifly: %s" % e) + + +if __name__ == '__main__': + for proxy in ProxiFlyFetcher().fetch(): + print(proxy) \ No newline at end of file diff --git a/tests/unit/test_fetcher_sources.py b/tests/unit/test_fetcher_sources.py index b641fec8..88ea4d99 100644 --- a/tests/unit/test_fetcher_sources.py +++ b/tests/unit/test_fetcher_sources.py @@ -69,6 +69,7 @@ class TestFetcherInterface(object): ("fetcher.sources.scdn", "ScdnFetcher"), ("fetcher.sources.zdaye", "ZdayeFetcher"), ("fetcher.sources.ihuan", "IhuanFetcher"), + ("fetcher.sources.proxifly", "ProxiFlyFetcher"), ] def test_all_fetchers_have_name_url_enabled(self): @@ -355,3 +356,42 @@ def test_fetch_no_key_returns_empty(self, mock_wr): mock_wr.return_value.get.return_value = ti_resp result = list(IhuanFetcher().fetch()) assert result == [] + + +class TestProxiFlyFetcher(object): + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "5.6.7.8:3128", "protocol": "http", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "5.6.7.8:3128" in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_cn(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "http", "geolocation": {"country": "US"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result + + @patch("fetcher.sources.proxifly.WebRequest") + def test_fetch_filters_non_http(self, mock_wr): + from fetcher.sources.proxifly import ProxiFlyFetcher + json_data = [ + {"proxy": "1.2.3.4:8080", "protocol": "http", "geolocation": {"country": "CN"}}, + {"proxy": "9.9.9.9:8080", "protocol": "https", "geolocation": {"country": "CN"}}, + ] + mock_wr.return_value.get.return_value = _make_response(json_data=json_data) + result = list(ProxiFlyFetcher().fetch()) + assert "1.2.3.4:8080" in result + assert "9.9.9.9:8080" not in result From c1aa529f3c85f0a52e4ffcdbd4c359f695a7cc80 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 1 Jun 2026 22:36:23 +0800 Subject: [PATCH 4/4] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=20README=20?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E4=BB=A3=E7=90=86=E6=BA=90=E5=92=8C=E5=85=8D?= =?UTF-8?q?=E8=B4=B9=E4=BB=A3=E7=90=86=E6=BA=90=E8=A1=A8=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 扩展代理部分更新为 BaseFetcher 插件架构说明 - 免费代理源表格代码链接更新为 fetcher/sources/*.py - 新增 Proxifly 代理源到表格 - 配置部分更新为 PROXY_FETCHER_EXCLUDE 黑名单说明 --- README.md | 76 +++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0d080bf2..19b61358 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,10 @@ PORT = 5000 # 监听端口 DB_CONN = 'redis://:pwd@127.0.0.1:8888/0' -# 配置 ProxyFetcher - -PROXY_FETCHER = [ - "freeProxy01", # 这里是启用的代理抓取方法名，所有fetch方法位于fetcher/proxyFetcher.py - "freeProxy02", - # .... -] +# 配置代理源（可选） +# 默认自动扫描 fetcher/sources/ 目录下所有 enabled=True 的代理源 +# 如需禁用某些代理源，在黑名单中添加其 name 即可 +# PROXY_FETCHER_EXCLUDE = ["freevpnnode"] ``` #### 启动项目: @@ -167,59 +164,48 @@ def getHtml(): 　　添加一个新的代理源方法如下: -* 1、首先在[ProxyFetcher](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/fetcher/proxyFetcher.py#L21)类中添加自定义的获取代理的静态方法， -该方法需要以生成器(yield)形式返回`host:ip`格式的代理，例如: +* 1、在 `fetcher/sources/` 目录下新建 `.py` 文件，继承 `BaseFetcher` 基类，声明 `name`/`url`/`enabled` 属性，实现 `fetch()` 方法以生成器(yield)形式返回`host:port`格式的代理，例如: ```python +from fetcher.baseFetcher import BaseFetcher +from util.webRequest import WebRequest -class ProxyFetcher(object): - # .... +class MyProxyFetcher(BaseFetcher): + """我的代理源""" - # 自定义代理源获取方法 - @staticmethod - def freeProxyCustom1(): # 命名不和已有重复即可 + name = "myproxy" + url = "https://www.example.com/" + enabled = True - # 通过某网站或者某接口或某数据库获取代理 - # 假设你已经拿到了一个代理列表 - proxies = ["x.x.x.x:3128", "x.x.x.x:80"] - for proxy in proxies: - yield proxy - # 确保每个proxy都是 host:ip正确的格式返回 + def fetch(self): + r = WebRequest().get("https://www.example.com/api/proxies") + for item in r.json: + yield item["ip"] + ":" + item["port"] ``` -* 2、添加好方法后，修改[setting.py](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/setting.py#L47)文件中的`PROXY_FETCHER`项： - -　　在`PROXY_FETCHER`下添加自定义方法的名字: - -```python -PROXY_FETCHER = [ - "freeProxy01", - "freeProxy02", - # .... - "freeProxyCustom1" # # 确保名字和你添加方法名字一致 -] -``` +* 2、添加好后，`schedule` 进程下次抓取时会自动扫描 `fetcher/sources/` 目录并启用新代理源，无需修改配置。 +　　可用 `python proxyPool.py fetcher` 命令查看当前启用的代理源列表。 -　　`schedule` 进程会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。 +　　如需临时禁用某个代理源，在 [setting.py](setting.py) 的 `PROXY_FETCHER_EXCLUDE` 黑名单中添加其 `name` 即可。 ### 免费代理源 - 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): + 目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): | 代理名称 | 状态 | 更新速度 | 可用率 | 地址 | 代码 | |---------------| ---- | -------- | ------ | ----- |------------------------------------------------| - | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`freeProxy02`](/fetcher/proxyFetcher.py#L50) | - | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`freeProxy03`](/fetcher/proxyFetcher.py#L63) | - | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeProxy04`](/fetcher/proxyFetcher.py#L74) | - | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`freeProxy05`](/fetcher/proxyFetcher.py#L92) | - | 冰凌代理 | ✔ | ★★★ | * | [地址](https://www.binglx.cn/) | [`freeProxy06`](/fetcher/proxyFetcher.py#L111) | - | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`freeProxy07`](/fetcher/proxyFetcher.py#L123) | - | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`freeProxy08`](/fetcher/proxyFetcher.py#L133) | - | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`freeProxy09`](/fetcher/proxyFetcher.py#L143) | - | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`freeProxy10`](/fetcher/proxyFetcher.py#L154) | - | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`freeProxy11`](/fetcher/proxyFetcher.py#L164) | - | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`freeProxy12`](/fetcher/proxyFetcher.py#L174) | + | 66代理 | ✔ | ★ | * | [地址](http://www.66ip.cn/) | [`ip66.py`](/fetcher/sources/ip66.py) | + | 开心代理 | ✔ | ★ | * | [地址](http://www.kxdaili.com/) | [`kxdaili.py`](/fetcher/sources/kxdaili.py) | + | FreeProxyList | ✔ | ★ | * | [地址](https://www.freeproxylists.net/zh/) | [`freeproxylist.py`](/fetcher/sources/freeproxylist.py) | + | 快代理 | ✔ | ★ | * | [地址](https://www.kuaidaili.com/) | [`kuaidaili.py`](/fetcher/sources/kuaidaili.py) | + | 云代理 | ✔ | ★ | * | [地址](http://www.ip3366.net/) | [`ip3366.py`](/fetcher/sources/ip3366.py) | + | 小幻代理 | ✔ | ★★ | * | [地址](https://ip.ihuan.me/) | [`ihuan.py`](/fetcher/sources/ihuan.py) | + | 免费代理库 | ✔ | ☆ | * | [地址](http://ip.jiangxianli.com/) | [`jiangxianli.py`](/fetcher/sources/jiangxianli.py) | + | 89代理 | ✔ | ☆ | * | [地址](https://www.89ip.cn/) | [`ip89.py`](/fetcher/sources/ip89.py) | + | 稻壳代理 | ✔ | ★★ | *** | [地址](https://www.docip.ne) | [`docip.py`](/fetcher/sources/docip.py) | + | 谷德代理 | ✔ | ★★ | *** | [地址](https://www.goodips.com) | [`goodips.py`](/fetcher/sources/goodips.py) | + | Proxifly | ✔ | ★★ | ** | [地址](https://proxifly.dev) | [`proxifly.py`](/fetcher/sources/proxifly.py) | 如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。