From 02456d5692683034c5e9b9a3fd1cc4cffabcc144 Mon Sep 17 00:00:00 2001 From: KYTG-Dev Date: Mon, 15 Dec 2025 00:00:03 -0500 Subject: [PATCH 1/4] Add support for custom http headers for multiple URLS --- web_retriever/version.py | 2 +- web_retriever/web_retriever/fetch.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/web_retriever/version.py b/web_retriever/version.py index 779733f..5140fa1 100644 --- a/web_retriever/version.py +++ b/web_retriever/version.py @@ -1 +1 @@ -version = "0.3.0" +version = "0.4.0" diff --git a/web_retriever/web_retriever/fetch.py b/web_retriever/web_retriever/fetch.py index 178f81b..5eda7e5 100644 --- a/web_retriever/web_retriever/fetch.py +++ b/web_retriever/web_retriever/fetch.py @@ -3,6 +3,7 @@ from typing import Any from typing import Dict from typing import List from typing import Union +from multidict import CIMultiDict from aiohttp.web import Request # type: ignore @@ -107,6 +108,9 @@ async def get_multiple( hdrs = hub.web_retriever.ops.mods(request.headers, mods) + custom_headers = url_params.get("headers", {}) + hdrs = CIMultiDict({**hdrs, **custom_headers}) + url_data, status = await hub.web_retriever.ops.get(url, hdrs, method="get", **url_params) ret["data"].append(url_data) # type: ignore -- GitLab From 964a2c7b085b2856ec2d92ce195d4d5c61b1c8e8 Mon Sep 17 00:00:00 2001 From: KYTG-Dev Date: Mon, 15 Dec 2025 00:24:45 -0500 Subject: [PATCH 2/4] precommit changes --- requirements/base.txt | 1 + requirements/py3.10/tests.txt | 45 +++++++++++++------------- requirements/py3.11/tests.txt | 41 +++++++++++++----------- requirements/py3.8/tests.txt | 47 +++++++++++++--------------- requirements/py3.9/tests.txt | 47 +++++++++++++--------------- web_retriever/web_retriever/fetch.py | 2 +- 6 files changed, 90 insertions(+), 93 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 11c59d3..61036b3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,3 +3,4 @@ pop>=17.0 pop-aiohttp>=1.2.0 rule-engine>=4.1.0 PyYAML>=6.0.1 +multidict>=6.4.3 diff --git a/requirements/py3.10/tests.txt b/requirements/py3.10/tests.txt index 929b21b..946b21a 100644 --- a/requirements/py3.10/tests.txt +++ b/requirements/py3.10/tests.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.10/tests.txt requirements/tests.in # +--trusted-host pypi.org + acct==8.5.2 # via pop-aiohttp aiodns==3.0.0 @@ -43,8 +45,6 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend -exceptiongroup==1.1.1 - # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.3.3 @@ -69,9 +69,10 @@ msgpack==1.0.5 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via # aiohttp + # web-retriever # yarl nest-asyncio==1.5.6 # via @@ -83,6 +84,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -97,20 +108,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -122,12 +129,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -151,8 +152,6 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend -tomli==2.0.1 - # via pytest file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.11/tests.txt b/requirements/py3.11/tests.txt index 441a401..92b2faf 100644 --- a/requirements/py3.11/tests.txt +++ b/requirements/py3.11/tests.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.11/tests.txt requirements/tests.in # +--trusted-host pypi.org + acct==8.6.1 # via pop-aiohttp aiodns==3.0.0 @@ -67,9 +69,10 @@ msgpack==1.0.6 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via # aiohttp + # web-retriever # yarl nest-asyncio==1.5.8 # via @@ -81,6 +84,16 @@ pluggy==1.3.0 # via pytest ply==3.11 # via rule-engine +pop==27.0.1 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.4 @@ -95,20 +108,16 @@ pop-loop==1.1.0 # pop-aiohttp pop-serial==2.0.0 # via acct -pop==27.0.1 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.4.2 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.5 # via -r requirements/tests.in pytest-async==0.1.1 @@ -120,12 +129,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.4.2 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 diff --git a/requirements/py3.8/tests.txt b/requirements/py3.8/tests.txt index 34be441..4816097 100644 --- a/requirements/py3.8/tests.txt +++ b/requirements/py3.8/tests.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.8/tests.txt requirements/tests.in # +--trusted-host pypi.org + acct==8.5.2 # via pop-aiohttp aiodns==3.0.0 @@ -26,8 +28,6 @@ attrs==23.1.0 # via aiohttp brotli==1.0.9 # via aiohttp -cchardet==2.1.7 - # via aiohttp cffi==1.15.1 # via # cryptography @@ -45,8 +45,6 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend -exceptiongroup==1.1.1 - # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.3.3 @@ -71,9 +69,10 @@ msgpack==1.0.5 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via # aiohttp + # web-retriever # yarl nest-asyncio==1.5.6 # via @@ -85,6 +84,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -99,20 +108,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -124,12 +129,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -153,8 +152,6 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend -tomli==2.0.1 - # via pytest file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.9/tests.txt b/requirements/py3.9/tests.txt index 93a5aab..7b68789 100644 --- a/requirements/py3.9/tests.txt +++ b/requirements/py3.9/tests.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.9/tests.txt requirements/tests.in # +--trusted-host pypi.org + acct==8.5.2 # via pop-aiohttp aiodns==3.0.0 @@ -26,8 +28,6 @@ attrs==23.1.0 # via aiohttp brotli==1.0.9 # via aiohttp -cchardet==2.1.7 - # via aiohttp cffi==1.15.1 # via # cryptography @@ -45,8 +45,6 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend -exceptiongroup==1.1.1 - # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.3.3 @@ -71,9 +69,10 @@ msgpack==1.0.5 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via # aiohttp + # web-retriever # yarl nest-asyncio==1.5.6 # via @@ -85,6 +84,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -99,20 +108,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -124,12 +129,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -153,8 +152,6 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend -tomli==2.0.1 - # via pytest file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/web_retriever/web_retriever/fetch.py b/web_retriever/web_retriever/fetch.py index 5eda7e5..913d323 100644 --- a/web_retriever/web_retriever/fetch.py +++ b/web_retriever/web_retriever/fetch.py @@ -3,9 +3,9 @@ from typing import Any from typing import Dict from typing import List from typing import Union -from multidict import CIMultiDict from aiohttp.web import Request # type: ignore +from multidict import CIMultiDict def __init__(hub: Any) -> None: -- GitLab From fe4f5b04cda5f3f751caad7fe1b3ec8c331db21e Mon Sep 17 00:00:00 2001 From: KYTG-Dev Date: Mon, 15 Dec 2025 10:34:32 -0500 Subject: [PATCH 3/4] pre-commit changes and remove python3.8 --- .gitlab-ci.yml | 4 - .pre-commit-config.yaml | 15 +-- requirements/base.txt | 2 +- requirements/py3.10/tests.txt | 47 ++++---- requirements/py3.11/tests.txt | 39 +++---- requirements/py3.8/tests.txt | 158 --------------------------- requirements/py3.9/tests.txt | 49 +++++---- requirements/tests.in | 1 + web_retriever/web_retriever/fetch.py | 2 +- web_retriever/web_retriever/ops.py | 2 +- 10 files changed, 85 insertions(+), 234 deletions(-) delete mode 100644 requirements/py3.8/tests.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da3ac28..042f390 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,10 +41,6 @@ pre-commit-run-all: junit: artifacts/junit-report.xml expire_in: 30 days -tests-3.8: - extends: .test-suite - image: python:3.8 - tests-3.9: extends: .test-suite image: python:3.9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2aad1f2..7447680 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,6 +86,7 @@ repos: exclude: .*/contracts/.* args: [--disallow-untyped-defs, --disallow-incomplete-defs, --disallow-untyped-calls] additional_dependencies: + - multidict>6.0 - types-attrs - types-setuptools - pydantic>=2.0,<3.0 @@ -98,18 +99,6 @@ repos: - repo: https://github.com/saltstack/pip-tools-compile-impersonate rev: '4.8' hooks: - - id: pip-tools-compile - alias: compile-3.8-test-requirements - name: Py3.8 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - --no-emit-index-url - - requirements/tests.in - - id: pip-tools-compile alias: compile-3.9-test-requirements name: Py3.9 Test Requirements @@ -176,5 +165,7 @@ repos: (?x)^( tests/.* )$ + additional_dependencies: + - pbr # <---- Security --------------------------------------------------------------------------------------------------- diff --git a/requirements/base.txt b/requirements/base.txt index 61036b3..966e320 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,4 +3,4 @@ pop>=17.0 pop-aiohttp>=1.2.0 rule-engine>=4.1.0 PyYAML>=6.0.1 -multidict>=6.4.3 +multidict>=6.0 diff --git a/requirements/py3.10/tests.txt b/requirements/py3.10/tests.txt index 946b21a..6e2fb1f 100644 --- a/requirements/py3.10/tests.txt +++ b/requirements/py3.10/tests.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # -# pip-compile --no-emit-index-url --output-file=requirements/py3.10/tests.txt requirements/tests.in +# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.10/tests.txt --pip-args='' requirements/tests.in # --trusted-host pypi.org @@ -45,6 +45,8 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend +exceptiongroup==1.3.1 + # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.3.3 @@ -71,6 +73,7 @@ msgpack==1.0.5 # pop-serial multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp # web-retriever # yarl @@ -84,16 +87,6 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -108,16 +101,20 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -129,6 +126,12 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -152,6 +155,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend +tomli==2.3.0 + # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.11/tests.txt b/requirements/py3.11/tests.txt index 92b2faf..7c4485d 100644 --- a/requirements/py3.11/tests.txt +++ b/requirements/py3.11/tests.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # -# pip-compile --no-emit-index-url --output-file=requirements/py3.11/tests.txt requirements/tests.in +# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.11/tests.txt --pip-args='' requirements/tests.in # --trusted-host pypi.org @@ -71,6 +71,7 @@ msgpack==1.0.6 # pop-serial multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp # web-retriever # yarl @@ -84,16 +85,6 @@ pluggy==1.3.0 # via pytest ply==3.11 # via rule-engine -pop==27.0.1 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.4 @@ -108,16 +99,20 @@ pop-loop==1.1.0 # pop-aiohttp pop-serial==2.0.0 # via acct +pop==27.0.1 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi -pytest==7.4.2 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop pytest-aiohttp==1.0.5 # via -r requirements/tests.in pytest-async==0.1.1 @@ -129,6 +124,12 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in +pytest==7.4.2 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 diff --git a/requirements/py3.8/tests.txt b/requirements/py3.8/tests.txt deleted file mode 100644 index 4816097..0000000 --- a/requirements/py3.8/tests.txt +++ /dev/null @@ -1,158 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --no-emit-index-url --output-file=requirements/py3.8/tests.txt requirements/tests.in -# ---trusted-host pypi.org - -acct==8.5.2 - # via pop-aiohttp -aiodns==3.0.0 - # via aiohttp -aiofiles==23.1.0 - # via - # acct - # dict-toolbox -aiohttp[speedups]==3.8.4 - # via - # pop-aiohttp - # pytest-aiohttp -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -asynctest==0.13.0 - # via -r requirements/tests.in -attrs==23.1.0 - # via aiohttp -brotli==1.0.9 - # via aiohttp -cffi==1.15.1 - # via - # cryptography - # pycares -charset-normalizer==3.1.0 - # via aiohttp -colorama==0.4.6 - # via rend -cryptography==40.0.2 - # via acct -dict-toolbox==3.1.1 - # via - # acct - # pop - # pop-config - # pytest-pop - # rend -feedparser==6.0.10 - # via web-retriever -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -idna==3.4 - # via yarl -iniconfig==2.0.0 - # via pytest -jinja2==3.1.2 - # via rend -lazy-object-proxy==1.9.0 - # via pop -markupsafe==2.1.2 - # via jinja2 -mock==5.0.2 - # via - # -r requirements/tests.in - # pytest-pop -msgpack==1.0.5 - # via - # dict-toolbox - # pop-serial -multidict==6.7.0 - # via - # aiohttp - # web-retriever - # yarl -nest-asyncio==1.5.6 - # via - # pop-loop - # pytest-pop -packaging==23.1 - # via pytest -pluggy==1.0.0 - # via pytest -ply==3.11 - # via rule-engine -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever -pop-aiohttp==1.2.0 - # via web-retriever -pop-config==12.0.2 - # via - # acct - # pop - # pop-aiohttp - # pytest-pop -pop-loop==1.0.6 - # via - # pop - # pop-aiohttp -pop-serial==1.1.1 - # via acct -pycares==4.3.0 - # via aiodns -pycparser==2.21 - # via cffi -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop -pytest-aiohttp==1.0.4 - # via -r requirements/tests.in -pytest-async==0.1.1 - # via pytest-pop -pytest-asyncio==0.18.3 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-pop -pytest-pop==12.0.0 - # via -r requirements/tests.in -python-dateutil==2.8.2 - # via rule-engine -pyyaml==6.0.1 - # via - # acct - # dict-toolbox - # pop - # rend - # web-retriever -rend==6.5.2 - # via - # acct - # pop-config -rule-engine==4.1.0 - # via web-retriever -sgmllib3k==1.0.0 - # via feedparser -six==1.16.0 - # via python-dateutil -sniffio==1.3.0 - # via pop-loop -toml==0.10.2 - # via rend -file:. - # via -r requirements/tests.in -yarl==1.9.2 - # via aiohttp diff --git a/requirements/py3.9/tests.txt b/requirements/py3.9/tests.txt index 7b68789..4a11a71 100644 --- a/requirements/py3.9/tests.txt +++ b/requirements/py3.9/tests.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by pip-compile +# To update, run: # -# pip-compile --no-emit-index-url --output-file=requirements/py3.9/tests.txt requirements/tests.in +# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.9/tests.txt --pip-args='' requirements/tests.in # --trusted-host pypi.org @@ -28,6 +28,8 @@ attrs==23.1.0 # via aiohttp brotli==1.0.9 # via aiohttp +cchardet==2.1.7 + # via aiohttp cffi==1.15.1 # via # cryptography @@ -45,6 +47,8 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend +exceptiongroup==1.3.1 + # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.3.3 @@ -71,6 +75,7 @@ msgpack==1.0.5 # pop-serial multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp # web-retriever # yarl @@ -84,16 +89,6 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -108,16 +103,20 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -129,6 +128,12 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -152,6 +157,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend +tomli==2.3.0 + # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/tests.in b/requirements/tests.in index 287b590..edad43e 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -5,3 +5,4 @@ pytest>=7.1.2 pytest-aiohttp pytest-asyncio pytest-pop>=8.0 +multidict>=6.0 diff --git a/web_retriever/web_retriever/fetch.py b/web_retriever/web_retriever/fetch.py index 913d323..119c752 100644 --- a/web_retriever/web_retriever/fetch.py +++ b/web_retriever/web_retriever/fetch.py @@ -5,7 +5,7 @@ from typing import List from typing import Union from aiohttp.web import Request # type: ignore -from multidict import CIMultiDict +from multidict import CIMultiDict # type: ignore def __init__(hub: Any) -> None: diff --git a/web_retriever/web_retriever/ops.py b/web_retriever/web_retriever/ops.py index f6083f0..7678e89 100644 --- a/web_retriever/web_retriever/ops.py +++ b/web_retriever/web_retriever/ops.py @@ -177,7 +177,7 @@ def mods(hub: Any, headers: Dict[str, Any], mods: List[Dict[str, Any]]) -> Any: """ Function to modify request headers """ - hdrs: Dict[str, str] = CIMultiDict() + hdrs: CIMultiDict[str] = CIMultiDict() for mod in mods: if mod.get("pass_request_headers"): hdrs.update(headers) -- GitLab From b4f5763f78de20113951d346193706ef85ca7153 Mon Sep 17 00:00:00 2001 From: KYTG-Dev Date: Mon, 15 Dec 2025 10:50:55 -0500 Subject: [PATCH 4/4] precommit changes --- requirements/py3.10/tests.txt | 40 ++++++++++++++--------------- requirements/py3.11/tests.txt | 48 ++++++++++++++++++++--------------- requirements/py3.9/tests.txt | 42 ++++++++++++++---------------- 3 files changed, 65 insertions(+), 65 deletions(-) diff --git a/requirements/py3.10/tests.txt b/requirements/py3.10/tests.txt index 6e2fb1f..be0b6dd 100644 --- a/requirements/py3.10/tests.txt +++ b/requirements/py3.10/tests.txt @@ -1,11 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.10/tests.txt --pip-args='' requirements/tests.in +# pip-compile --no-emit-index-url --output-file=requirements/py3.10/tests.txt requirements/tests.in # ---trusted-host pypi.org - acct==8.5.2 # via pop-aiohttp aiodns==3.0.0 @@ -87,6 +85,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -101,20 +109,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -126,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 diff --git a/requirements/py3.11/tests.txt b/requirements/py3.11/tests.txt index 7c4485d..f74c383 100644 --- a/requirements/py3.11/tests.txt +++ b/requirements/py3.11/tests.txt @@ -1,11 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.11/tests.txt --pip-args='' requirements/tests.in +# pip-compile --no-emit-index-url --output-file=requirements/py3.11/tests.txt requirements/tests.in # ---trusted-host pypi.org - acct==8.6.1 # via pop-aiohttp aiodns==3.0.0 @@ -45,6 +43,8 @@ dict-toolbox==4.1.1 # pop-config # pytest-pop # rend +exceptiongroup==1.3.1 + # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.4.0 @@ -85,6 +85,16 @@ pluggy==1.3.0 # via pytest ply==3.11 # via rule-engine +pop==27.0.1 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.4 @@ -99,20 +109,16 @@ pop-loop==1.1.0 # pop-aiohttp pop-serial==2.0.0 # via acct -pop==27.0.1 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.4.2 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.5 # via -r requirements/tests.in pytest-async==0.1.1 @@ -124,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.4.2 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -153,6 +153,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend +tomli==2.3.0 + # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.9/tests.txt b/requirements/py3.9/tests.txt index 4a11a71..9e177ff 100644 --- a/requirements/py3.9/tests.txt +++ b/requirements/py3.9/tests.txt @@ -1,11 +1,9 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --cert='' --client-cert='' --index-url='' --no-emit-index-url --output-file=requirements/py3.9/tests.txt --pip-args='' requirements/tests.in +# pip-compile --no-emit-index-url --output-file=requirements/py3.9/tests.txt requirements/tests.in # ---trusted-host pypi.org - acct==8.5.2 # via pop-aiohttp aiodns==3.0.0 @@ -28,8 +26,6 @@ attrs==23.1.0 # via aiohttp brotli==1.0.9 # via aiohttp -cchardet==2.1.7 - # via aiohttp cffi==1.15.1 # via # cryptography @@ -89,6 +85,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -103,20 +109,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -128,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 -- GitLab