diff --git a/codemeta.json b/codemeta.json index 90cc6ce1e884539843921314e064c7752947428b..9bdf66b981112120cf9b96382617f04a6901e1ae 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev22+g2a70d3e", - "softwareVersion": "v0.7.dev22+g2a70d3e", + "version": "v0.7.dev55+g3ba6238", + "softwareVersion": "v0.7.dev55+g3ba6238", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,11 +43,11 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev22+g2a70d3e/eossr-v0.7.dev22+g2a70d3e.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev55+g3ba6238/eossr-v0.7.dev55+g3ba6238.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", - "datePublished": "2022-06-10", - "dateModified": "2022-06-10", + "datePublished": "2022-06-14", + "dateModified": "2022-06-14", "operatingSystem": "", "maintainer": { "@type": "Person", diff --git a/eossr/api/__init__.py b/eossr/api/__init__.py deleted file mode 100644 index ec9b13dbcff091f5038507d071fc1ec386f60389..0000000000000000000000000000000000000000 --- a/eossr/api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .ossr import get_ossr_records # noqa -from .zenodo import Record, get_zenodo_records # noqa diff --git a/eossr/api/ossr.py b/eossr/api/ossr.py deleted file mode 100644 index 212d6694dd1c75c57f4fabf1ecfceed89ada84f2..0000000000000000000000000000000000000000 --- a/eossr/api/ossr.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -import requests - -from .zenodo import ZenodoAPI, get_zenodo_records, zenodo_api_url - -__all__ = [ - 'get_ossr_records', - 'get_ossr_pending_requests', -] - -escape_community = 'escape2020' - - -def get_ossr_records(search='', sandbox=False, **kwargs): - """ - Search the OSSR for records whose names or descriptions include the provided string `search`. - The default record type is 'software' or 'record'. - Function rewritten from pyzenodo3 (https://github.com/space-physics/pyzenodo3) - - :param search: string - A string to refine the search in the OSSR. The default will search for all records in the OSSR. - :param sandbox: bool - Indicates the use of sandbox zenodo or not. - :param kwargs: Zenodo query arguments. - For an exhaustive list, see the query arguments at https://developers.zenodo.org/#list36 - Common arguments are: - - size: int - Number of results to return. Default = 100 - - all_versions: int - Show (1) or hide (0) all versions of records - - type: string or list[string] - Default: ['software', 'dataset'] - Records of the specified type (Publication, Poster, Presentation, Software, ...). - A logical OR is applied in case of a list - - keywords: string or list[string] - Records with the specified keywords. A logical OR is applied in case of a list - - file_type: string or list[string] - Records from the specified keywords. A logical OR is applied in case of a list - - :return: [Record] - """ - - # make sure we find all OSSR records without limit on the number - params = kwargs - params['communities'] = escape_community - r = requests.get(zenodo_api_url + '/records', params=params) - number_of_ossr_entries = r.json()['aggregations']['access_right']['buckets'][0]['doc_count'] - kwargs['size'] = number_of_ossr_entries - - # if another community is specified, a logical OR is applied by zenodo API, - # thus potentially finding entries that are not part of escape2020 - # ruling out that possibility at the moment - if 'communities' in kwargs and kwargs['communities'] != escape_community: - raise NotImplementedError( - "Searching in another community will search outside of the OSSR" - "Use `eossr.api.zenodo.get_zenodo_records` to do so" - ) - kwargs['communities'] = escape_community - - # OSSR is limited to software and datasets - kwargs.setdefault('type', ['software', 'dataset']) - - return get_zenodo_records(search, sandbox=sandbox, **kwargs) - - -def get_ossr_pending_requests(**params): - """ - Get a list of records that have been requested to be added to the OSSR. - - :param params: dict - Parameters for the request. Override the class parameters. - :return: - """ - zen = ZenodoAPI() - return zen.get_community_pending_requests(escape_community, **params) diff --git a/eossr/api/tests/test_api.py b/eossr/api/tests/test_api.py deleted file mode 100644 index b331451f861adf7b4539663851af018accc0f81d..0000000000000000000000000000000000000000 --- a/eossr/api/tests/test_api.py +++ /dev/null @@ -1,8 +0,0 @@ -from eossr import api - - -def test_get_ossr_records(): - ossr_records = api.get_ossr_records(all_versions=True) - assert len(ossr_records) >= 12 # number of records October 01, 2021 - all_ids = [rec.data['id'] for rec in ossr_records] - assert 5524913 in all_ids # id of the version v0.2 of the eossr diff --git a/eossr/api/zenodo/__init__.py b/eossr/api/zenodo/__init__.py deleted file mode 100644 index 065ccdf6f80106851bab65db98e4078a491a62c0..0000000000000000000000000000000000000000 --- a/eossr/api/zenodo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .zenodo import * # noqa diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index 4e30c04071a4dfca97aebcc880ed1a9d61f8a549..1ad2dae21fbc0b52228fcb15707b73df6f5ec6b4 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -287,7 +287,7 @@ class CodeMeta2ZenodoController(object): def converter(codemeta_dict, add_escape2020=True): """ - Convert codemeta metadata into zendoo metadata + Convert an already loaded codemeta metadata json file into a zenodo metadata dict :param codemeta_dict: dict :param add_escape2020: bool @@ -315,7 +315,7 @@ def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add add_escape2020: bool adds escape2020 metadata in zenodo metadata file overwrite: bool - overwrite existing `.zendoo.json` file in `outdir` + overwrite existing `.zenodo.json` file in `outdir` """ meta_converter = CodeMeta2ZenodoController.from_file(codemeta_filename) meta_converter.convert() @@ -325,5 +325,6 @@ def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add outfile = Path(outdir).joinpath('.zenodo.json') if not outfile.exists() or overwrite: meta_converter.write_zenodo(outfile.name) + return outfile else: raise FileExistsError(f"The file {outfile} exists. Use overwrite.") diff --git a/eossr/ossr.py b/eossr/ossr.py new file mode 100644 index 0000000000000000000000000000000000000000..86f5305edf4d66b1b314637919f4e8beae0fb9fb --- /dev/null +++ b/eossr/ossr.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +import json + +import requests + +from .metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file +from .zenodo import ZenodoAPI, get_zenodo_records, zenodo_api_url + +__all__ = ["escape_community", "get_ossr_records", "get_ossr_pending_requests", "OssrAPI"] + +escape_community = 'escape2020' + + +class OssrAPI(ZenodoAPI): + def __init__(self, access_token=None, sandbox=False): + super().__init__(access_token, sandbox) + + def load_metadata(self, directory): + """ + Performs the codemeta to zenodo conversion and loads the metadata into a json dict + + :param directory: Path or str + path to the directory containing the metadata file + + :return: dict + metadata loaded dictionary + """ + codemeta_metadata_file = self.path_codemeta_file(directory) + if codemeta_metadata_file.exists(): + print(f" - Record metadata based on codemeta file {codemeta_metadata_file}") + with open(codemeta_metadata_file) as file: + codemeta = json.load(file) + metadata = converter(codemeta) + else: + raise FileNotFoundError(" ! No `codemeta.json` metadata file provided") + + return metadata + + def check_upload_to_zenodo(self, directory): + """ + `Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every + stage is the correct one. + + The upload of the directory (passed by argument) to zenodo is done starting from a `codemeta.json` file + and doing the codemeta2zenodo conversion. + + Checks: + - The existence of a `codemeta.json` file in the ROOT dir of the project and + - If it exists it performs the codemeta2zenodo conversion + - If not, it exits the program + + - The communication with Zenodo through its API to verify that: + - You can fetch an user entries + - You can create a new entry + - The provided zenodo metadata can be digested, and not errors appear + - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED ! + + :param directory: Path + Path to directory whose content will be uploaded to Zenodo + """ + self._raise_token_status() + path_codemeta_file = self.path_codemeta_file(directory) + + if not path_codemeta_file.exists(): + raise FileNotFoundError(f"No codemeta {path_codemeta_file} file found.") + + print("\n * Performing the codemeta to zenodo conversion... ") + zenodo_file = parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, directory) + + self._test_zenodo_connection(zenodo_file) + + +def get_ossr_records(search='', sandbox=False, **kwargs): + """ + Search the OSSR for records whose names or descriptions include the provided string `search`. + The default record type is 'software' or 'record'. + Function rewritten from pyzenodo3 (https://github.com/space-physics/pyzenodo3) + + :param search: string + A string to refine the search in the OSSR. The default will search for all records in the OSSR. + :param sandbox: bool + Indicates the use of sandbox zenodo or not. + :param kwargs: Zenodo query arguments. + For an exhaustive list, see the query arguments at https://developers.zenodo.org/#list36 + Common arguments are: + - size: int + Number of results to return. Default = 100 + - all_versions: int + Show (1) or hide (0) all versions of records + - type: string or list[string] + Default: ['software', 'dataset'] + Records of the specified type (Publication, Poster, Presentation, Software, ...). + A logical OR is applied in case of a list + - keywords: string or list[string] + Records with the specified keywords. A logical OR is applied in case of a list + - file_type: string or list[string] + Records from the specified keywords. A logical OR is applied in case of a list + + :return: [Record] + """ + + # make sure we find all OSSR records without limit on the number + params = kwargs + params['communities'] = escape_community + r = requests.get(zenodo_api_url + '/records', params=params) + number_of_ossr_entries = r.json()['aggregations']['access_right']['buckets'][0]['doc_count'] + kwargs['size'] = number_of_ossr_entries + + # if another community is specified, a logical OR is applied by zenodo API, + # thus potentially finding entries that are not part of escape2020 + # ruling out that possibility at the moment + if 'communities' in kwargs and kwargs['communities'] != escape_community: + raise NotImplementedError( + "Searching in another community will search outside of the OSSR" + "Use `eossr.api.zenodo.get_zenodo_records` to do so" + ) + kwargs['communities'] = escape_community + + # OSSR is limited to software and datasets + kwargs.setdefault('type', ['software', 'dataset']) + + return get_zenodo_records(search, sandbox=sandbox, **kwargs) + + +def get_ossr_pending_requests(community=escape_community, **params): + """ + Get a list of records that have been requested to be added to the OSSR. + + :param community: str + Name of the community. DEFAULT: `escape2020` + :param params: dict + Parameters for the request. Override the class parameters. + :return: + """ + ossr = OssrAPI() + return ossr.get_community_pending_requests(community, **params) diff --git a/eossr/scripts/check_connection_zenodo.py b/eossr/scripts/check_connection_zenodo.py index f11b9a48fd25e3d1f351dc1058af3adb713a36b8..c40c6e1597233388b04ae59a140dee8bcb40e200 100644 --- a/eossr/scripts/check_connection_zenodo.py +++ b/eossr/scripts/check_connection_zenodo.py @@ -2,7 +2,7 @@ import argparse -from eossr.api.zenodo import ZenodoAPI +from eossr.zenodo import ZenodoAPI def build_argparser(): diff --git a/eossr/scripts/eossr_upload_repository.py b/eossr/scripts/eossr_upload_repository.py index 271e079e169ad501f2d3ba09c826257c018d5f5e..6e6279de6efee0ec6a318079cdbe3262e0ab4191 100644 --- a/eossr/scripts/eossr_upload_repository.py +++ b/eossr/scripts/eossr_upload_repository.py @@ -8,9 +8,9 @@ import tempfile from copy import deepcopy from pathlib import Path -from eossr.api.zenodo import Record, SimilarRecordError, ZenodoAPI from eossr.metadata.codemeta2zenodo import converter from eossr.utils import zip_repository +from eossr.zenodo import Record, SimilarRecordError, ZenodoAPI def upload( diff --git a/eossr/scripts/zenodo_user_entries_cleanup.py b/eossr/scripts/zenodo_user_entries_cleanup.py index 4177dc10360b56646c3251f988838e34b3bdf221..7a67cafe091917913daa9426b4b33c2256c81926 100644 --- a/eossr/scripts/zenodo_user_entries_cleanup.py +++ b/eossr/scripts/zenodo_user_entries_cleanup.py @@ -4,7 +4,7 @@ Simple code to delete all user entries that have not been published import argparse import os -from eossr.api.zenodo import ZenodoAPI +from eossr.zenodo import ZenodoAPI def zenodo_cleanup(token, sandbox=True): diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py new file mode 100644 index 0000000000000000000000000000000000000000..6f9f959b7679102cdc75d7f07b0603ab250e3bb6 --- /dev/null +++ b/eossr/tests/test_ossr.py @@ -0,0 +1,83 @@ +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +import pytest + +from eossr import ROOT_DIR +from eossr.ossr import OssrAPI, get_ossr_pending_requests, get_ossr_records + +zenodo_key_fields = [ + 'access_right', + 'communities', + 'creators', + 'description', + 'grants', + 'keywords', + 'language', + 'license', + 'notes', + 'publication_date', + 'related_identifiers', + 'title', + 'upload_type', + 'version', +] + + +def test_get_ossr_records(): + ossr_records = get_ossr_records(all_versions=True) + assert len(ossr_records) >= 12 # number of records October 01, 2021 + all_ids = [rec.data['id'] for rec in ossr_records] + assert 5524913 in all_ids # id of the version v0.2 of the eossr + + +class TestOssrAPINoToken(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(TestOssrAPINoToken, self).__init__(*args, **kwargs) + self.token = '' + self.ossr = OssrAPI(access_token=self.token, sandbox=False) + + def test_initialization(self): + assert isinstance(self.ossr, OssrAPI) + assert self.ossr.api_url == 'https://zenodo.org/api' + assert self.ossr.access_token == self.token + + @pytest.mark.xfail(raises=ValueError) + def test_raise_token_status(self): + # A value error should be raised as no valid token was provided + self.ossr._raise_token_status() + + def test_load_metadata(self): + metadata = self.ossr.load_metadata(ROOT_DIR) + assert isinstance(metadata, dict) + assert all(item in zenodo_key_fields for item in list(metadata.keys())) + + @pytest.mark.xfail(raises=FileNotFoundError) + def test_fail_load_metadata(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + self.ossr.load_metadata(tmp_dirname) + + +@pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") +class TestOssrAPI(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(TestOssrAPI, self).__init__(*args, **kwargs) + self.token = os.getenv('SANDBOX_ZENODO_TOKEN') + self.ossr = OssrAPI(access_token=self.token, sandbox=True) + + def test_get_ossr_pending_requests(self): + records = get_ossr_pending_requests(**{"access_token": self.token}) + records_ids = [rec.id for rec in records] + assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 + + def test_check_upload_to_zenodo(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) + _, filename = tempfile.mkstemp(dir=tmp_dirname) + Path(filename).write_text('Hello from test_eossr.py unit tests') + self.ossr.check_upload_to_zenodo(tmp_dirname) diff --git a/eossr/zenodo/__init__.py b/eossr/zenodo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..129bd1b91279bccbbe14190eb1f14dec622571b1 --- /dev/null +++ b/eossr/zenodo/__init__.py @@ -0,0 +1,41 @@ +""" + isort:skip_file +""" +from .http_status import HTTPStatusError, ZenodoHTTPStatus +from .zenodo import ( + zenodo_api_url, + zenodo_sandbox_api_url, + ZenodoAPI, + SimilarRecordError, + Record, + query_zenodo_records, + get_zenodo_records, + query_record, + get_record, + search_records, + search_funders, + search_grants, + search_communities, + search_licenses, + is_live, +) + +__all__ = [ + "HTTPStatusError", + "ZenodoHTTPStatus", + "zenodo_api_url", + "zenodo_sandbox_api_url", + "ZenodoAPI", + "SimilarRecordError", + "Record", + "query_zenodo_records", + "get_zenodo_records", + "query_record", + "get_record", + "search_records", + "search_funders", + "search_grants", + "search_communities", + "search_licenses", + "is_live", +] diff --git a/eossr/api/zenodo/http_status.py b/eossr/zenodo/http_status.py similarity index 97% rename from eossr/api/zenodo/http_status.py rename to eossr/zenodo/http_status.py index 4db9ec9a7efa2e9fe92bc3eae3daf3d0fe1dbe49..84b7871b6b2a84da34f02da9568549df4bac19b1 100644 --- a/eossr/api/zenodo/http_status.py +++ b/eossr/zenodo/http_status.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -__all__ = ['ZenodoHTTPStatus'] +__all__ = ["ZenodoHTTPStatus", "HTTPStatusError"] class HTTPStatusError(Exception): @@ -73,7 +73,7 @@ class ZenodoHTTPStatus: def __init__(self, code: int, json=None): if code not in self.status_codes.keys(): - raise ValueError("Unkown status code") + raise ValueError("Unknown status code") self.code = code self.json = json if self.is_error(): diff --git a/eossr/api/zenodo/tests/test_http_status.py b/eossr/zenodo/tests/test_http_status.py similarity index 93% rename from eossr/api/zenodo/tests/test_http_status.py rename to eossr/zenodo/tests/test_http_status.py index 3a8ec5380b63882bf3b83f9f927875e94d50c14f..0f7bbd67072c510b52a7e460ce884750984b9380 100644 --- a/eossr/api/zenodo/tests/test_http_status.py +++ b/eossr/zenodo/tests/test_http_status.py @@ -1,6 +1,6 @@ import pytest -from eossr.api.zenodo import http_status +from eossr.zenodo import http_status def test_ZenodoHTTPStatus(): diff --git a/eossr/api/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py similarity index 84% rename from eossr/api/zenodo/tests/test_zenodo.py rename to eossr/zenodo/tests/test_zenodo.py index 8cf3134fd59a81fb48c55e42e6359b9138464729..d9593a87d4934951fa05b313d2614ff826ce483d 100644 --- a/eossr/api/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -9,11 +9,10 @@ from pathlib import Path import pytest import requests -from eossr import ROOT_DIR -from eossr.api import zenodo -from eossr.api.zenodo import Record, ZenodoAPI, get_record, get_zenodo_records -from eossr.api.zenodo.http_status import HTTPStatusError -from eossr.api.zenodo.zenodo import is_live, query_record +from eossr import ROOT_DIR, zenodo +from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController, parse_codemeta_and_write_zenodo_metadata_file +from eossr.tests.test_ossr import zenodo_key_fields +from eossr.zenodo import HTTPStatusError, Record, ZenodoAPI, get_record, get_zenodo_records, is_live, query_record eossr_test_lib_id = 930570 # test library in sandbox (owner: T. Vuillaume) @@ -61,6 +60,25 @@ class TestZenodoAPINoToken(unittest.TestCase): # A value error should be raised as no valid token was provided self.zenodo._raise_token_status() + def test_load_metadata(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) + + # parse_codemeta_and_write_zenodo_metadata_file does not work fine with temp dirs, + # thus writing explicitly the .zenodo.json file + converter = CodeMeta2ZenodoController.from_file(Path(tmp_dirname).joinpath('codemeta.json')) + converter.convert() + converter.add_escape2020_community() + converter.add_escape2020_grant() + converter.write_zenodo(Path(tmp_dirname).joinpath('.zenodo.json')) + + assert Path(tmp_dirname).joinpath('codemeta.json').exists() + assert Path(tmp_dirname).joinpath('.zenodo.json').exists() + metadata = self.zenodo.load_metadata(tmp_dirname) + assert isinstance(metadata, dict) + assert all(item in zenodo_key_fields for item in list(metadata.keys())) + @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") class TestZenodoAPIToken(unittest.TestCase): @@ -94,6 +112,9 @@ class TestZenodoAPIToken(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdirname: print(f"tmpdir {tmpdirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmpdirname) + parse_codemeta_and_write_zenodo_metadata_file( + Path(tmpdirname).joinpath('codemeta.json'), Path(tmpdirname).joinpath('.zenodo.json') + ) _, filename = tempfile.mkstemp(dir=tmpdirname) Path(filename).write_text('Hello from eossr unit tests') @@ -120,7 +141,7 @@ class TestZenodoAPIToken(unittest.TestCase): def test_pending_request(self): zk = ZenodoAPI(os.getenv('SANDBOX_ZENODO_TOKEN_GARCIA'), sandbox=True) record_id = 970583 - from eossr.api.ossr import escape_community + from eossr.ossr import escape_community # Add to escape2020 community and test request meta = {'communities': [{'identifier': escape_community}]} @@ -163,7 +184,7 @@ def test_get_record_42(): def test_query_record_941144(): - # unsubmitted record in T. Vuillaume's sandbox - for test purposes + # non-submitted record in T. Vuillaume's sandbox - for test purposes answer = query_record(941144, sandbox=True) assert answer.status_code == 404 diff --git a/eossr/api/zenodo/zenodo.py b/eossr/zenodo/zenodo.py similarity index 95% rename from eossr/api/zenodo/zenodo.py rename to eossr/zenodo/zenodo.py index 380a473708777cde36d2cdf23614edcf2ccd51ae..9d0326f83845169aef907f8635c39f58ab4e5b32 100644 --- a/eossr/api/zenodo/zenodo.py +++ b/eossr/zenodo/zenodo.py @@ -12,15 +12,14 @@ from urllib.request import urlopen import requests from bs4 import BeautifulSoup -from ...metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file -from ...utils import get_codemeta_from_zipurl -from . import http_status +from ..utils import get_codemeta_from_zipurl +from ..zenodo import http_status __all__ = [ 'zenodo_api_url', 'zenodo_sandbox_api_url', 'ZenodoAPI', - 'SimilarRecordError', # noqa + 'SimilarRecordError', 'Record', 'query_zenodo_records', 'get_zenodo_records', @@ -310,7 +309,7 @@ class ZenodoAPI: If a record_id is provided, a new version of the record will be created. :param metadata: dict or None dictionary of zenodo metadata - if None, the metadata will be read from a `.zenodo.json` file or a `codemeta.json` file in `self.root_dir` + if None, the metadata will be read from a `.zenodo.json` file in `self.root_dir` :param erase_previous_files: bool In case of making a new version of an existing record (`record_id` not None), erase files from the previous version @@ -342,25 +341,13 @@ class ZenodoAPI: print(f" * New record id: {new_record_id}") # get metadata - path_codemeta_file = self.path_codemeta_file(directory) - path_zenodo_file = self.path_zenodo_file(directory) - if metadata is not None: + if metadata: print(f" * Record metadata based on provided metadata: {metadata}") - elif path_zenodo_file.exists(): - print(f" - Record metadata based on zenodo file {path_zenodo_file}") - with open(path_zenodo_file) as file: - metadata = json.load(file) - elif path_codemeta_file.exists(): - print(f" - Record metadata based on codemeta file {path_codemeta_file}") - with open(path_codemeta_file) as file: - codemeta = json.load(file) - metadata = converter(codemeta) else: - raise FileNotFoundError(" ! No metadata file provided") + metadata = self.load_metadata(directory) # upload files - dir_to_upload = Path(directory) - for file in dir_to_upload.iterdir(): + for file in Path(directory).iterdir(): self.upload_file_entry(entry_id=new_record_id, name_file=file.name, path_file=file) print(f" * {file.name} uploaded") @@ -381,34 +368,62 @@ class ZenodoAPI: return new_record_id + def load_metadata(self, directory): + """ + Loads the metadata from a `.zenodo.json` file into a json dict + + :param directory: Path or str + path to the directory containing the metadata file + + :return: Loaded metadata dictionary + """ + zenodo_metadata_file = self.path_zenodo_file(directory) + if zenodo_metadata_file.exists(): + print(f" - Record metadata based on zenodo file {zenodo_metadata_file}") + with open(zenodo_metadata_file) as file: + metadata = json.load(file) + else: + raise FileNotFoundError(" ! No `.zenodo.json` metadata file provided") + + return metadata + def check_upload_to_zenodo(self, directory): """ `Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every stage is the correct one. + The upload of the directory (passed by argument) to zenodo is done starting from a `.zenodo.json` file + and doing the codemeta2zenodo conversion. + Checks: - The existence of a `.zenodo.json` file in the ROOT dir of the project - - If not, it checks if it exists a `codemeta.json` file - - If it exists it performs the codemeta2zenodo conversion - - If not, it exits the program + - If it exists, it continues with the test + - If not, it exits the program - The communication with Zenodo through its API to verify that: - - You can fetch a user entries + - You can fetch an user entries - You can create a new entry - The provided zenodo metadata can be digested, and not errors appear - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED ! + + :param directory: Path + Path to directory whose content will be uploaded to Zenodo """ self._raise_token_status() path_zenodo_file = self.path_zenodo_file(directory) - path_codemeta_file = self.path_codemeta_file(directory) if not path_zenodo_file.exists(): - if not path_codemeta_file.exists(): - raise FileNotFoundError(f"No codemeta {path_codemeta_file} nor zenodo {path_zenodo_file} files.") + raise FileNotFoundError(f"No zenodo {path_zenodo_file} file found.") + + self._test_zenodo_connection(path_zenodo_file) - print("\n * Creating a .zenodo.json file from your codemeta.json file...") + def _test_zenodo_connection(self, zenodo_metadata_file): + """ + Performs the checks described in the `check_upload_to_zenodo` method - parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, path_zenodo_file) - print(f"\n * Using {path_zenodo_file} file to simulate a new upload to Zenodo... \n") + :param zenodo_metadata_file: Path + Path to `.zenodo.json` file + """ + print(f"\n * Using {zenodo_metadata_file} file to simulate a new upload to Zenodo... \n") # 1 - Test connection print("1 --> Testing communication with Zenodo...") @@ -430,7 +445,7 @@ class ZenodoAPI: print("3 --> Testing the ingestion of the Zenodo metadata...") test_entry_id = new_entry.json()['id'] - with open(path_zenodo_file) as file: + with open(zenodo_metadata_file) as file: metadata_entry = json.load(file) updated_metadata = self.set_deposit_metadata(test_entry_id, json_metadata=metadata_entry)