diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da3ac28aa0913ed8517a84b29a61a715a8165fe1..042f3900d10e8d07c6519aecb1c6139beed2578c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,10 +41,6 @@ pre-commit-run-all: junit: artifacts/junit-report.xml expire_in: 30 days -tests-3.8: - extends: .test-suite - image: python:3.8 - tests-3.9: extends: .test-suite image: python:3.9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2aad1f27a8c2fbfa15bde99aa011dc61b7abdd4d..7447680093c1d447d17363a9a3f263a4841d3b78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,6 +86,7 @@ repos: exclude: .*/contracts/.* args: [--disallow-untyped-defs, --disallow-incomplete-defs, --disallow-untyped-calls] additional_dependencies: + - multidict>6.0 - types-attrs - types-setuptools - pydantic>=2.0,<3.0 @@ -98,18 +99,6 @@ repos: - repo: https://github.com/saltstack/pip-tools-compile-impersonate rev: '4.8' hooks: - - id: pip-tools-compile - alias: compile-3.8-test-requirements - name: Py3.8 Test Requirements - files: ^requirements/tests.in$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - --no-emit-index-url - - requirements/tests.in - - id: pip-tools-compile alias: compile-3.9-test-requirements name: Py3.9 Test Requirements @@ -176,5 +165,7 @@ repos: (?x)^( tests/.* )$ + additional_dependencies: + - pbr # <---- Security --------------------------------------------------------------------------------------------------- diff --git a/requirements/base.txt b/requirements/base.txt index 11c59d37f3f87af0847a50a57312f6e5c6a86b9e..966e32092be40cb2d216024f4e033a4ec46e6e75 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,3 +3,4 @@ pop>=17.0 pop-aiohttp>=1.2.0 rule-engine>=4.1.0 PyYAML>=6.0.1 +multidict>=6.0 diff --git a/requirements/py3.10/tests.txt b/requirements/py3.10/tests.txt index 929b21b74448051313d5e82c6c2e1e0dd1501b44..be0b6ddf7c842323684eb26ab3f5e2e285f40c7c 100644 --- a/requirements/py3.10/tests.txt +++ b/requirements/py3.10/tests.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.10/tests.txt requirements/tests.in # @@ -43,7 +43,7 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend -exceptiongroup==1.1.1 +exceptiongroup==1.3.1 # via pytest feedparser==6.0.10 # via web-retriever @@ -69,9 +69,11 @@ msgpack==1.0.5 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp + # web-retriever # yarl nest-asyncio==1.5.6 # via @@ -83,6 +85,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -97,20 +109,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -122,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -151,8 +153,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend -tomli==2.0.1 +tomli==2.3.0 # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.11/tests.txt b/requirements/py3.11/tests.txt index 441a4011dce275e6b1b6d0de92b4a2746f276a7f..f74c383260acacd1059199c3ed4ca091069918af 100644 --- a/requirements/py3.11/tests.txt +++ b/requirements/py3.11/tests.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.11/tests.txt requirements/tests.in # @@ -43,6 +43,8 @@ dict-toolbox==4.1.1 # pop-config # pytest-pop # rend +exceptiongroup==1.3.1 + # via pytest feedparser==6.0.10 # via web-retriever frozenlist==1.4.0 @@ -67,9 +69,11 @@ msgpack==1.0.6 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp + # web-retriever # yarl nest-asyncio==1.5.8 # via @@ -81,6 +85,16 @@ pluggy==1.3.0 # via pytest ply==3.11 # via rule-engine +pop==27.0.1 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.4 @@ -95,20 +109,16 @@ pop-loop==1.1.0 # pop-aiohttp pop-serial==2.0.0 # via acct -pop==27.0.1 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.4.2 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.5 # via -r requirements/tests.in pytest-async==0.1.1 @@ -120,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.4.2 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -149,6 +153,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend +tomli==2.3.0 + # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/py3.8/tests.txt b/requirements/py3.8/tests.txt deleted file mode 100644 index 34be4413e13bb6275a66be0970141515bd20ccb5..0000000000000000000000000000000000000000 --- a/requirements/py3.8/tests.txt +++ /dev/null @@ -1,161 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --no-emit-index-url --output-file=requirements/py3.8/tests.txt requirements/tests.in -# -acct==8.5.2 - # via pop-aiohttp -aiodns==3.0.0 - # via aiohttp -aiofiles==23.1.0 - # via - # acct - # dict-toolbox -aiohttp[speedups]==3.8.4 - # via - # pop-aiohttp - # pytest-aiohttp -aiosignal==1.3.1 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -asynctest==0.13.0 - # via -r requirements/tests.in -attrs==23.1.0 - # via aiohttp -brotli==1.0.9 - # via aiohttp -cchardet==2.1.7 - # via aiohttp -cffi==1.15.1 - # via - # cryptography - # pycares -charset-normalizer==3.1.0 - # via aiohttp -colorama==0.4.6 - # via rend -cryptography==40.0.2 - # via acct -dict-toolbox==3.1.1 - # via - # acct - # pop - # pop-config - # pytest-pop - # rend -exceptiongroup==1.1.1 - # via pytest -feedparser==6.0.10 - # via web-retriever -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -idna==3.4 - # via yarl -iniconfig==2.0.0 - # via pytest -jinja2==3.1.2 - # via rend -lazy-object-proxy==1.9.0 - # via pop -markupsafe==2.1.2 - # via jinja2 -mock==5.0.2 - # via - # -r requirements/tests.in - # pytest-pop -msgpack==1.0.5 - # via - # dict-toolbox - # pop-serial -multidict==6.0.4 - # via - # aiohttp - # yarl -nest-asyncio==1.5.6 - # via - # pop-loop - # pytest-pop -packaging==23.1 - # via pytest -pluggy==1.0.0 - # via pytest -ply==3.11 - # via rule-engine -pop-aiohttp==1.2.0 - # via web-retriever -pop-config==12.0.2 - # via - # acct - # pop - # pop-aiohttp - # pytest-pop -pop-loop==1.0.6 - # via - # pop - # pop-aiohttp -pop-serial==1.1.1 - # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever -pycares==4.3.0 - # via aiodns -pycparser==2.21 - # via cffi -pytest-aiohttp==1.0.4 - # via -r requirements/tests.in -pytest-async==0.1.1 - # via pytest-pop -pytest-asyncio==0.18.3 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-pop -pytest-pop==12.0.0 - # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop -python-dateutil==2.8.2 - # via rule-engine -pyyaml==6.0.1 - # via - # acct - # dict-toolbox - # pop - # rend - # web-retriever -rend==6.5.2 - # via - # acct - # pop-config -rule-engine==4.1.0 - # via web-retriever -sgmllib3k==1.0.0 - # via feedparser -six==1.16.0 - # via python-dateutil -sniffio==1.3.0 - # via pop-loop -toml==0.10.2 - # via rend -tomli==2.0.1 - # via pytest -file:. - # via -r requirements/tests.in -yarl==1.9.2 - # via aiohttp diff --git a/requirements/py3.9/tests.txt b/requirements/py3.9/tests.txt index 93a5aabf649fe20a1f451ee36f4a44cc3f00d942..9e177ff68772297acae77be6db906f0fa361ebc6 100644 --- a/requirements/py3.9/tests.txt +++ b/requirements/py3.9/tests.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile --no-emit-index-url --output-file=requirements/py3.9/tests.txt requirements/tests.in # @@ -26,8 +26,6 @@ attrs==23.1.0 # via aiohttp brotli==1.0.9 # via aiohttp -cchardet==2.1.7 - # via aiohttp cffi==1.15.1 # via # cryptography @@ -45,7 +43,7 @@ dict-toolbox==3.1.1 # pop-config # pytest-pop # rend -exceptiongroup==1.1.1 +exceptiongroup==1.3.1 # via pytest feedparser==6.0.10 # via web-retriever @@ -71,9 +69,11 @@ msgpack==1.0.5 # via # dict-toolbox # pop-serial -multidict==6.0.4 +multidict==6.7.0 # via + # -r requirements/tests.in # aiohttp + # web-retriever # yarl nest-asyncio==1.5.6 # via @@ -85,6 +85,16 @@ pluggy==1.0.0 # via pytest ply==3.11 # via rule-engine +pop==23.1.0 + # via + # acct + # pop-aiohttp + # pop-config + # pop-loop + # pop-serial + # pytest-pop + # rend + # web-retriever pop-aiohttp==1.2.0 # via web-retriever pop-config==12.0.2 @@ -99,20 +109,16 @@ pop-loop==1.0.6 # pop-aiohttp pop-serial==1.1.1 # via acct -pop==23.1.0 - # via - # acct - # pop-aiohttp - # pop-config - # pop-loop - # pop-serial - # pytest-pop - # rend - # web-retriever pycares==4.3.0 # via aiodns pycparser==2.21 # via cffi +pytest==7.3.1 + # via + # -r requirements/tests.in + # pytest-aiohttp + # pytest-asyncio + # pytest-pop pytest-aiohttp==1.0.4 # via -r requirements/tests.in pytest-async==0.1.1 @@ -124,12 +130,6 @@ pytest-asyncio==0.18.3 # pytest-pop pytest-pop==12.0.0 # via -r requirements/tests.in -pytest==7.3.1 - # via - # -r requirements/tests.in - # pytest-aiohttp - # pytest-asyncio - # pytest-pop python-dateutil==2.8.2 # via rule-engine pyyaml==6.0.1 @@ -153,8 +153,12 @@ sniffio==1.3.0 # via pop-loop toml==0.10.2 # via rend -tomli==2.0.1 +tomli==2.3.0 # via pytest +typing-extensions==4.15.0 + # via + # exceptiongroup + # multidict file:. # via -r requirements/tests.in yarl==1.9.2 diff --git a/requirements/tests.in b/requirements/tests.in index 287b59080307717c8be53fe16fceba4349ba43d4..edad43ebf4155f3e9b429357258c8df11e8de3b2 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -5,3 +5,4 @@ pytest>=7.1.2 pytest-aiohttp pytest-asyncio pytest-pop>=8.0 +multidict>=6.0 diff --git a/web_retriever/version.py b/web_retriever/version.py index 779733fc2635efc598194abd4a77279fef0a808b..5140fa1d6ed3fbff9db5dbf07ddd9b9e8dd0f3f0 100644 --- a/web_retriever/version.py +++ b/web_retriever/version.py @@ -1 +1 @@ -version = "0.3.0" +version = "0.4.0" diff --git a/web_retriever/web_retriever/fetch.py b/web_retriever/web_retriever/fetch.py index 178f81b8ae0fea0db38357176aa09e1f263a516c..119c7521775bdc9601eeb3331043edb162cb3b93 100644 --- a/web_retriever/web_retriever/fetch.py +++ b/web_retriever/web_retriever/fetch.py @@ -5,6 +5,7 @@ from typing import List from typing import Union from aiohttp.web import Request # type: ignore +from multidict import CIMultiDict # type: ignore def __init__(hub: Any) -> None: @@ -107,6 +108,9 @@ async def get_multiple( hdrs = hub.web_retriever.ops.mods(request.headers, mods) + custom_headers = url_params.get("headers", {}) + hdrs = CIMultiDict({**hdrs, **custom_headers}) + url_data, status = await hub.web_retriever.ops.get(url, hdrs, method="get", **url_params) ret["data"].append(url_data) # type: ignore diff --git a/web_retriever/web_retriever/ops.py b/web_retriever/web_retriever/ops.py index f6083f0c3458a1fd5dfc8109774fb0fc5d1d3bfa..7678e890c90d00030a1cea102359c9172fea9c15 100644 --- a/web_retriever/web_retriever/ops.py +++ b/web_retriever/web_retriever/ops.py @@ -177,7 +177,7 @@ def mods(hub: Any, headers: Dict[str, Any], mods: List[Dict[str, Any]]) -> Any: """ Function to modify request headers """ - hdrs: Dict[str, str] = CIMultiDict() + hdrs: CIMultiDict[str] = CIMultiDict() for mod in mods: if mod.get("pass_request_headers"): hdrs.update(headers)