From 5ecea09b8420cfacd6607f4b86aa7dcc16ba0b57 Mon Sep 17 00:00:00 2001 From: Julian Carrivick Date: Mon, 10 Jul 2023 16:16:58 +0800 Subject: [PATCH] Correctly resolve nexus_url from environment variables - Check CAR_TMDATA_REPOSITORY_URL before CAR_RAW_REPOSITORY_URL - If CAR_RAW_REPOSITORY_URL is used, replace `internal` with `telmodel` like what is done in ska-cicd-makefile `.make-tmdata-support`. - Add explicit test of nexus returning 404 to get 100% coverage as that was implicitly tested previously by the CI CAR_RAW_REPOSITORY_URL not being correctly tweaked to replace `internal` with `telmodel` --- src/ska_telmodel/data/backend.py | 4 ++ tests/test_data.py | 107 ++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 22 deletions(-) diff --git a/src/ska_telmodel/data/backend.py b/src/ska_telmodel/data/backend.py index 23d22525..043a9520 100644 --- a/src/ska_telmodel/data/backend.py +++ b/src/ska_telmodel/data/backend.py @@ -741,8 +741,12 @@ class GitlabBackend(TMDataBackend): # Determine Nexus URL to try nexus_url = os.getenv("SKA_TELMODEL_NEXUS") + if nexus_url is None: + nexus_url = os.getenv("CAR_TMDATA_REPOSITORY_URL") if nexus_url is None: nexus_url = os.getenv("CAR_RAW_REPOSITORY_URL") + if nexus_url: + nexus_url = nexus_url.replace("internal", "telmodel") if nexus_url is None: # pragma: no cover nexus_url = "https://artefact.skao.int/repository/raw-telmodel" diff --git a/tests/test_data.py b/tests/test_data.py index ca3e28fd..87fac1ff 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -5,7 +5,10 @@ import os import pathlib import tarfile import tempfile +import threading from hashlib import sha1 +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Literal import gitlab import pytest @@ -38,6 +41,26 @@ def tmdata_file(): yield TMData([f"file://{dirname}"]) +@pytest.fixture +def http_server(): + class TestHttpServerHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_error(404) + + # Choose a random port + httpd = HTTPServer(("localhost", 0), TestHttpServerHandler) + + def serve(): + with httpd: + httpd.serve_forever() + + thread = threading.Thread(target=serve) + thread.start() + yield httpd + httpd.shutdown() + thread.join() + + def _add_file_to_tar(tar, name, contents): bio = io.BytesIO(contents.encode("utf8")) info = tarfile.TarInfo(name) @@ -505,7 +528,20 @@ def test_tmdata_get_sources(tmdata_all, backend): assert sources_p == sources -def test_tmdata_nexus(monkeypatch): +@pytest.mark.parametrize( + "nexus_env_name, custom_suffix, env_value_replacement", + [ + ("SKA_TELMODEL_NEXUS", None, None), + ("CAR_TMDATA_REPOSITORY_URL", None, None), + ("CAR_RAW_REPOSITORY_URL", "telmodel", "internal"), + ], +) +def test_tmdata_nexus( + monkeypatch: pytest.MonkeyPatch, + nexus_env_name: str, + custom_suffix: str | None, + env_value_replacement: str | None, +): """Check that we check Nexus for refs""" # Use a Gitlab that doesn't have anything @@ -548,10 +584,23 @@ def test_tmdata_nexus(monkeypatch): }, ] + # Clear any env variables that might be set in CI + monkeypatch.delenv("SKA_TELMODEL_NEXUS", raising=False) + monkeypatch.delenv("CAR_TMDATA_REPOSITORY_URL", raising=False) + monkeypatch.delenv("CAR_RAW_REPOSITORY_URL", raising=False) + # Set up a temporary directory as source for "Nexus" data - with tempfile.TemporaryDirectory("test-telmodel-nexus") as dirname: + telmodel_dir = "test-telmodel-nexus" + if custom_suffix: + telmodel_dir += "-" + custom_suffix + with tempfile.TemporaryDirectory(telmodel_dir) as dirname: with tempfile.TemporaryDirectory("test-telmodel-cache") as cachedir: - monkeypatch.setenv("SKA_TELMODEL_NEXUS", f"file://{dirname}") + nexus_env_value = ( + dirname.replace(custom_suffix, env_value_replacement) + if env_value_replacement + else dirname + ) + monkeypatch.setenv(nexus_env_name, f"file://{nexus_env_value}") monkeypatch.setenv("SKA_TELMODEL_CACHE", cachedir) path = pathlib.Path( dirname, "gitlab.com/ska-telescope/ska-telmodel-nexus/tmdata" @@ -605,7 +654,12 @@ def test_tmdata_nexus(monkeypatch): assert f.read() == expected -def test_tmdata_nexus_project_doesnt_exist(monkeypatch): +@pytest.mark.parametrize("source", ("fs", "http")) +def test_tmdata_nexus_project_doesnt_exist( + monkeypatch: pytest.MonkeyPatch, + source: Literal["fs", "http"], + http_server: HTTPServer, +): """Check that we check Nexus for refs""" # Use a Gitlab that doesn't have anything @@ -618,27 +672,36 @@ def test_tmdata_nexus_project_doesnt_exist(monkeypatch): def get(self, project_name): raise gitlab.exceptions.GitlabGetError() - # Set up a temporary directory as source for "Nexus" data - with tempfile.TemporaryDirectory("test-telmodel-nexus") as dirname: - with tempfile.TemporaryDirectory("test-telmodel-cache") as cachedir: + import contextlib + + with contextlib.ExitStack() as stack: + # Set up a temporary source for "Nexus" data + if source == "http": + (host, port) = http_server.server_address + monkeypatch.setenv("SKA_TELMODEL_NEXUS", f"http://{host}:{port}") + elif source == "fs": + dirname = stack.enter_context( + tempfile.TemporaryDirectory("test-telmodel-nexus") + ) monkeypatch.setenv("SKA_TELMODEL_NEXUS", f"file://{dirname}") - monkeypatch.setenv("SKA_TELMODEL_CACHE", cachedir) + else: + raise ValueError("Unknown telmodel source") - # Create TMData object - sources = [ - "gitlab://gitlab.com/ska-telescope/" - "ska-telmodel-nexus2?main#tmdata" - ] - pars = {"gitlab": dict(gl=PseudoGitlab(), try_nexus=True)} + cachedir = tempfile.TemporaryDirectory("test-telmodel-cache") + monkeypatch.setenv("SKA_TELMODEL_CACHE", cachedir) + + # Create TMData object + sources = [ + "gitlab://gitlab.com/ska-telescope/" + "ska-telmodel-nexus2?main#tmdata" + ] + pars = {"gitlab": dict(gl=PseudoGitlab(), try_nexus=True)} - # This should try to access the GitLab project, which fails - with pytest.raises( - RuntimeError, match=r"Could not access gitlab.com project.*" - ): - with pytest.warns( - UserWarning, match=r".*not cached in SKA CAR.*" - ): - TMData(sources, backend_pars=pars) + # This should try to access the GitLab project, which fails + with pytest.raises( + RuntimeError, match=r"Could not access gitlab.com project.*" + ), pytest.warns(UserWarning, match=r".*not cached in SKA CAR.*"): + TMData(sources, backend_pars=pars) @pytest.mark.parametrize("path", ["", "tmdata", "nested/tmdata"]) -- GitLab