From 8372fc1837b23228268ff91f701dce9a396b1b91 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Wed, 4 May 2022 18:52:58 +0200 Subject: [PATCH 01/26] move submodules and fix imports due to refactoring --- eossr/api/__init__.py | 2 - eossr/api/zenodo/__init__.py | 1 - eossr/{api => }/ossr.py | 44 +++++++++++++++++-- eossr/scripts/check_connection_zenodo.py | 2 +- eossr/scripts/eossr_upload_repository.py | 2 +- eossr/scripts/zenodo_user_entries_cleanup.py | 2 +- .../tests/test_api.py => tests/test_ossr.py} | 4 +- eossr/zenodo/__init__.py | 41 +++++++++++++++++ eossr/{api => }/zenodo/http_status.py | 2 +- .../zenodo/tests/test_http_status.py | 2 +- eossr/{api => }/zenodo/tests/test_zenodo.py | 9 ++-- eossr/{api => }/zenodo/zenodo.py | 10 ++--- 12 files changed, 96 insertions(+), 25 deletions(-) delete mode 100644 eossr/api/__init__.py delete mode 100644 eossr/api/zenodo/__init__.py rename eossr/{api => }/ossr.py (67%) rename eossr/{api/tests/test_api.py => tests/test_ossr.py} (74%) create mode 100644 eossr/zenodo/__init__.py rename eossr/{api => }/zenodo/http_status.py (98%) rename eossr/{api => }/zenodo/tests/test_http_status.py (93%) rename eossr/{api => }/zenodo/tests/test_zenodo.py (96%) rename eossr/{api => }/zenodo/zenodo.py (99%) diff --git a/eossr/api/__init__.py b/eossr/api/__init__.py deleted file mode 100644 index ec9b13d..0000000 --- a/eossr/api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .ossr import get_ossr_records # noqa -from .zenodo import Record, get_zenodo_records # noqa diff --git a/eossr/api/zenodo/__init__.py b/eossr/api/zenodo/__init__.py deleted file mode 100644 index 065ccdf..0000000 --- a/eossr/api/zenodo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .zenodo import * # noqa diff --git a/eossr/api/ossr.py b/eossr/ossr.py similarity index 67% rename from eossr/api/ossr.py rename to eossr/ossr.py index b519107..7d43025 100644 --- a/eossr/api/ossr.py +++ b/eossr/ossr.py @@ -4,14 +4,50 @@ import requests from .zenodo import ZenodoAPI, get_zenodo_records, zenodo_api_url -__all__ = [ - 'get_ossr_records', - 'get_ossr_pending_requests', -] +__all__ = ["escape_community", "get_ossr_records", "get_ossr_pending_requests", "OssrAPI"] escape_community = 'escape2020' +class OssrAPI(ZenodoAPI): + def upload_dir_content(self, directory, record_id=None, metadata=None, erase_previous_files=True, publish=True): + """ + Up + + :param directory: + :param record_id: + :param metadata: + :param erase_previous_files: + :param publish: + :return: + """ + pass + + def check_upload_to_zenodo(self, directory): + """ + `Tests` the different stages of the GitLab-Zenodo connection. + + The upload to zenodo is done starting from a `codemeta.json` file and doing the codemeta2zenodo + conversion. + + Checks: + - The existence of a `codemeta.json` file in the ROOT dir of the project and + - If not, it checks if it exists a `codemeta.json` file + - If it exists it performs the codemeta2zenodo conversion + - If not, it exits the program + + - The communication with Zenodo through its API to verify that: + - You can fetch a user entries + - You can create a new entry + - The provided zenodo metadata can be digested, and not errors appear + - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED ! + + :param directory: + :return: + """ + pass + + def get_ossr_records(search='', sandbox=False, **kwargs): """ Search the OSSR for records whose names or descriptions include the provided string `search`. diff --git a/eossr/scripts/check_connection_zenodo.py b/eossr/scripts/check_connection_zenodo.py index f11b9a4..c40c6e1 100644 --- a/eossr/scripts/check_connection_zenodo.py +++ b/eossr/scripts/check_connection_zenodo.py @@ -2,7 +2,7 @@ import argparse -from eossr.api.zenodo import ZenodoAPI +from eossr.zenodo import ZenodoAPI def build_argparser(): diff --git a/eossr/scripts/eossr_upload_repository.py b/eossr/scripts/eossr_upload_repository.py index 271e079..6e6279d 100644 --- a/eossr/scripts/eossr_upload_repository.py +++ b/eossr/scripts/eossr_upload_repository.py @@ -8,9 +8,9 @@ import tempfile from copy import deepcopy from pathlib import Path -from eossr.api.zenodo import Record, SimilarRecordError, ZenodoAPI from eossr.metadata.codemeta2zenodo import converter from eossr.utils import zip_repository +from eossr.zenodo import Record, SimilarRecordError, ZenodoAPI def upload( diff --git a/eossr/scripts/zenodo_user_entries_cleanup.py b/eossr/scripts/zenodo_user_entries_cleanup.py index 4177dc1..7a67caf 100644 --- a/eossr/scripts/zenodo_user_entries_cleanup.py +++ b/eossr/scripts/zenodo_user_entries_cleanup.py @@ -4,7 +4,7 @@ Simple code to delete all user entries that have not been published import argparse import os -from eossr.api.zenodo import ZenodoAPI +from eossr.zenodo import ZenodoAPI def zenodo_cleanup(token, sandbox=True): diff --git a/eossr/api/tests/test_api.py b/eossr/tests/test_ossr.py similarity index 74% rename from eossr/api/tests/test_api.py rename to eossr/tests/test_ossr.py index 60cf73d..20bd228 100644 --- a/eossr/api/tests/test_api.py +++ b/eossr/tests/test_ossr.py @@ -1,8 +1,8 @@ -from eossr import api +from eossr import ossr def test_get_ossr_records(): - ossr_records = api.get_ossr_records(all_versions=1) + ossr_records = ossr.get_ossr_records(all_versions=1) assert len(ossr_records) >= 12 # number of records October 01, 2021 all_ids = [rec.data['id'] for rec in ossr_records] assert 5524913 in all_ids # id of the version v0.2 of the eossr diff --git a/eossr/zenodo/__init__.py b/eossr/zenodo/__init__.py new file mode 100644 index 0000000..129bd1b --- /dev/null +++ b/eossr/zenodo/__init__.py @@ -0,0 +1,41 @@ +""" + isort:skip_file +""" +from .http_status import HTTPStatusError, ZenodoHTTPStatus +from .zenodo import ( + zenodo_api_url, + zenodo_sandbox_api_url, + ZenodoAPI, + SimilarRecordError, + Record, + query_zenodo_records, + get_zenodo_records, + query_record, + get_record, + search_records, + search_funders, + search_grants, + search_communities, + search_licenses, + is_live, +) + +__all__ = [ + "HTTPStatusError", + "ZenodoHTTPStatus", + "zenodo_api_url", + "zenodo_sandbox_api_url", + "ZenodoAPI", + "SimilarRecordError", + "Record", + "query_zenodo_records", + "get_zenodo_records", + "query_record", + "get_record", + "search_records", + "search_funders", + "search_grants", + "search_communities", + "search_licenses", + "is_live", +] diff --git a/eossr/api/zenodo/http_status.py b/eossr/zenodo/http_status.py similarity index 98% rename from eossr/api/zenodo/http_status.py rename to eossr/zenodo/http_status.py index 4db9ec9..d4e87bf 100644 --- a/eossr/api/zenodo/http_status.py +++ b/eossr/zenodo/http_status.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -__all__ = ['ZenodoHTTPStatus'] +__all__ = ["ZenodoHTTPStatus", "HTTPStatusError"] class HTTPStatusError(Exception): diff --git a/eossr/api/zenodo/tests/test_http_status.py b/eossr/zenodo/tests/test_http_status.py similarity index 93% rename from eossr/api/zenodo/tests/test_http_status.py rename to eossr/zenodo/tests/test_http_status.py index 3a8ec53..0f7bbd6 100644 --- a/eossr/api/zenodo/tests/test_http_status.py +++ b/eossr/zenodo/tests/test_http_status.py @@ -1,6 +1,6 @@ import pytest -from eossr.api.zenodo import http_status +from eossr.zenodo import http_status def test_ZenodoHTTPStatus(): diff --git a/eossr/api/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py similarity index 96% rename from eossr/api/zenodo/tests/test_zenodo.py rename to eossr/zenodo/tests/test_zenodo.py index d56661f..169b5ec 100644 --- a/eossr/api/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -9,11 +9,8 @@ from pathlib import Path import pytest import requests -from eossr import ROOT_DIR -from eossr.api import zenodo -from eossr.api.zenodo import Record, ZenodoAPI, get_record, get_zenodo_records -from eossr.api.zenodo.http_status import HTTPStatusError -from eossr.api.zenodo.zenodo import is_live, query_record +from eossr import ROOT_DIR, zenodo +from eossr.zenodo import HTTPStatusError, Record, ZenodoAPI, get_record, get_zenodo_records, is_live, query_record eossr_test_lib_id = 930570 # test library in sandbox (owner: T. Vuillaume) @@ -120,7 +117,7 @@ class TestZenodoAPIToken(unittest.TestCase): def test_pending_request(self): zk = ZenodoAPI(os.getenv('SANDBOX_ZENODO_TOKEN_GARCIA'), sandbox=True) record_id = 970583 - from eossr.api.ossr import escape_community + from eossr.ossr import escape_community # Add to escape2020 community and test request meta = {'communities': [{'identifier': escape_community}]} diff --git a/eossr/api/zenodo/zenodo.py b/eossr/zenodo/zenodo.py similarity index 99% rename from eossr/api/zenodo/zenodo.py rename to eossr/zenodo/zenodo.py index 0790aca..2cccf92 100644 --- a/eossr/api/zenodo/zenodo.py +++ b/eossr/zenodo/zenodo.py @@ -12,15 +12,15 @@ from urllib.request import urlopen import requests from bs4 import BeautifulSoup -from ...metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file -from ...utils import get_codemeta_from_zipurl -from . import http_status +from ..metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file +from ..utils import get_codemeta_from_zipurl +from ..zenodo import http_status __all__ = [ 'zenodo_api_url', 'zenodo_sandbox_api_url', 'ZenodoAPI', - 'SimilarRecordError', # noqa + 'SimilarRecordError', 'Record', 'query_zenodo_records', 'get_zenodo_records', @@ -679,7 +679,7 @@ class Record: if 'version' in record.metadata: versions[record.id] = record.metadata['version'] else: - version[record.id] = None + versions[record.id] = None # nqa return versions def print_info(self, linebreak='\n', file=sys.stdout): -- GitLab From 1626720b77309951b8fd3f1d42c86ddef48750b5 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Thu, 9 Jun 2022 19:16:40 +0200 Subject: [PATCH 02/26] typo in docstring --- eossr/zenodo/http_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eossr/zenodo/http_status.py b/eossr/zenodo/http_status.py index d4e87bf..84b7871 100644 --- a/eossr/zenodo/http_status.py +++ b/eossr/zenodo/http_status.py @@ -73,7 +73,7 @@ class ZenodoHTTPStatus: def __init__(self, code: int, json=None): if code not in self.status_codes.keys(): - raise ValueError("Unkown status code") + raise ValueError("Unknown status code") self.code = code self.json = json if self.is_error(): -- GitLab From 52ab1d8b23228561947dba32f79c21fec23413fd Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 01:24:44 +0200 Subject: [PATCH 03/26] create OssrAPI class and refactor circular imports --- eossr/metadata/codemeta2zenodo.py | 2 +- eossr/ossr.py | 63 ++++++++++++++++++--------- eossr/zenodo/zenodo.py | 71 +++++++++++++++++++------------ 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index 4e30c04..4a25d8f 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -287,7 +287,7 @@ class CodeMeta2ZenodoController(object): def converter(codemeta_dict, add_escape2020=True): """ - Convert codemeta metadata into zendoo metadata + Convert codemeta metadata into zenodo metadata :param codemeta_dict: dict :param add_escape2020: bool diff --git a/eossr/ossr.py b/eossr/ossr.py index 7d43025..95541ff 100644 --- a/eossr/ossr.py +++ b/eossr/ossr.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +import json + import requests +from .metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file from .zenodo import ZenodoAPI, get_zenodo_records, zenodo_api_url __all__ = ["escape_community", "get_ossr_records", "get_ossr_pending_requests", "OssrAPI"] @@ -10,42 +13,62 @@ escape_community = 'escape2020' class OssrAPI(ZenodoAPI): - def upload_dir_content(self, directory, record_id=None, metadata=None, erase_previous_files=True, publish=True): + def __init__(self, access_token=None, sandbox=False): + super().__init__(access_token, sandbox) + + def load_metadata(self, directory): """ - Up - - :param directory: - :param record_id: - :param metadata: - :param erase_previous_files: - :param publish: - :return: + Performs the codemeta to zenodo conversion and loads the metadata into a json dict + + :param directory: Path or str + path to the directory containing the metadata file + + :return: metadata loaded dictionary """ - pass + codemeta_metadata_file = self.path_codemeta_file(directory) + if codemeta_metadata_file.exists(): + print(f" - Record metadata based on codemeta file {codemeta_metadata_file}") + with open(codemeta_metadata_file) as file: + codemeta = json.load(file) + metadata = converter(codemeta) + else: + raise FileNotFoundError(" ! No `codemeta.json` metadata file provided") + + return metadata def check_upload_to_zenodo(self, directory): """ - `Tests` the different stages of the GitLab-Zenodo connection. + `Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every + stage is the correct one. - The upload to zenodo is done starting from a `codemeta.json` file and doing the codemeta2zenodo - conversion. + The upload of the directory (passed by argument) to zenodo is done starting from a `codemeta.json` file + and doing the codemeta2zenodo conversion. Checks: - The existence of a `codemeta.json` file in the ROOT dir of the project and - - If not, it checks if it exists a `codemeta.json` file - - If it exists it performs the codemeta2zenodo conversion - - If not, it exits the program + - If it exists it performs the codemeta2zenodo conversion + - If not, it exits the program - The communication with Zenodo through its API to verify that: - - You can fetch a user entries + - You can fetch an user entries - You can create a new entry - The provided zenodo metadata can be digested, and not errors appear - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED ! - :param directory: - :return: + :param directory: Path + Path to directory whose content will be uploaded to Zenodo """ - pass + self._raise_token_status() + path_zenodo_file = self.path_zenodo_file(directory) + path_codemeta_file = self.path_codemeta_file(directory) + + if not path_codemeta_file.exists(): + raise FileNotFoundError(f"No codemeta {path_codemeta_file} file found.") + + print("\n * Performing the codemeta to zenodo conversion... ") + parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, path_zenodo_file) + + self._test_zenodo_connection(path_zenodo_file) def get_ossr_records(search='', sandbox=False, **kwargs): diff --git a/eossr/zenodo/zenodo.py b/eossr/zenodo/zenodo.py index 2a2b70a..9d0326f 100644 --- a/eossr/zenodo/zenodo.py +++ b/eossr/zenodo/zenodo.py @@ -12,7 +12,6 @@ from urllib.request import urlopen import requests from bs4 import BeautifulSoup -from ..metadata.codemeta2zenodo import converter, parse_codemeta_and_write_zenodo_metadata_file from ..utils import get_codemeta_from_zipurl from ..zenodo import http_status @@ -310,7 +309,7 @@ class ZenodoAPI: If a record_id is provided, a new version of the record will be created. :param metadata: dict or None dictionary of zenodo metadata - if None, the metadata will be read from a `.zenodo.json` file or a `codemeta.json` file in `self.root_dir` + if None, the metadata will be read from a `.zenodo.json` file in `self.root_dir` :param erase_previous_files: bool In case of making a new version of an existing record (`record_id` not None), erase files from the previous version @@ -342,25 +341,13 @@ class ZenodoAPI: print(f" * New record id: {new_record_id}") # get metadata - path_codemeta_file = self.path_codemeta_file(directory) - path_zenodo_file = self.path_zenodo_file(directory) - if metadata is not None: + if metadata: print(f" * Record metadata based on provided metadata: {metadata}") - elif path_zenodo_file.exists(): - print(f" - Record metadata based on zenodo file {path_zenodo_file}") - with open(path_zenodo_file) as file: - metadata = json.load(file) - elif path_codemeta_file.exists(): - print(f" - Record metadata based on codemeta file {path_codemeta_file}") - with open(path_codemeta_file) as file: - codemeta = json.load(file) - metadata = converter(codemeta) else: - raise FileNotFoundError(" ! No metadata file provided") + metadata = self.load_metadata(directory) # upload files - dir_to_upload = Path(directory) - for file in dir_to_upload.iterdir(): + for file in Path(directory).iterdir(): self.upload_file_entry(entry_id=new_record_id, name_file=file.name, path_file=file) print(f" * {file.name} uploaded") @@ -381,34 +368,62 @@ class ZenodoAPI: return new_record_id + def load_metadata(self, directory): + """ + Loads the metadata from a `.zenodo.json` file into a json dict + + :param directory: Path or str + path to the directory containing the metadata file + + :return: Loaded metadata dictionary + """ + zenodo_metadata_file = self.path_zenodo_file(directory) + if zenodo_metadata_file.exists(): + print(f" - Record metadata based on zenodo file {zenodo_metadata_file}") + with open(zenodo_metadata_file) as file: + metadata = json.load(file) + else: + raise FileNotFoundError(" ! No `.zenodo.json` metadata file provided") + + return metadata + def check_upload_to_zenodo(self, directory): """ `Tests` the different stages of the GitLab-Zenodo connection and that the status_code returned by every stage is the correct one. + The upload of the directory (passed by argument) to zenodo is done starting from a `.zenodo.json` file + and doing the codemeta2zenodo conversion. + Checks: - The existence of a `.zenodo.json` file in the ROOT dir of the project - - If not, it checks if it exists a `codemeta.json` file - - If it exists it performs the codemeta2zenodo conversion - - If not, it exits the program + - If it exists, it continues with the test + - If not, it exits the program - The communication with Zenodo through its API to verify that: - - You can fetch a user entries + - You can fetch an user entries - You can create a new entry - The provided zenodo metadata can be digested, and not errors appear - Finally erases the test entry - because IT HAS NOT BEEN PUBLISHED ! + + :param directory: Path + Path to directory whose content will be uploaded to Zenodo """ self._raise_token_status() path_zenodo_file = self.path_zenodo_file(directory) - path_codemeta_file = self.path_codemeta_file(directory) if not path_zenodo_file.exists(): - if not path_codemeta_file.exists(): - raise FileNotFoundError(f"No codemeta {path_codemeta_file} nor zenodo {path_zenodo_file} files.") + raise FileNotFoundError(f"No zenodo {path_zenodo_file} file found.") + + self._test_zenodo_connection(path_zenodo_file) - print("\n * Creating a .zenodo.json file from your codemeta.json file...") + def _test_zenodo_connection(self, zenodo_metadata_file): + """ + Performs the checks described in the `check_upload_to_zenodo` method - parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, path_zenodo_file) - print(f"\n * Using {path_zenodo_file} file to simulate a new upload to Zenodo... \n") + :param zenodo_metadata_file: Path + Path to `.zenodo.json` file + """ + print(f"\n * Using {zenodo_metadata_file} file to simulate a new upload to Zenodo... \n") # 1 - Test connection print("1 --> Testing communication with Zenodo...") @@ -430,7 +445,7 @@ class ZenodoAPI: print("3 --> Testing the ingestion of the Zenodo metadata...") test_entry_id = new_entry.json()['id'] - with open(path_zenodo_file) as file: + with open(zenodo_metadata_file) as file: metadata_entry = json.load(file) updated_metadata = self.set_deposit_metadata(test_entry_id, json_metadata=metadata_entry) -- GitLab From 73e5ea6c5de843a7b1f0777721b9a724b1e8a385 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 08:16:31 +0000 Subject: [PATCH 04/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codemeta.json b/codemeta.json index 80abad2..3179fe9 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev17+gc63452a", - "softwareVersion": "v0.7.dev17+gc63452a", + "version": "v0.7.dev20+g52ab1d8", + "softwareVersion": "v0.7.dev20+g52ab1d8", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,11 +43,11 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev17+gc63452a/eossr-v0.7.dev17+gc63452a.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev20+g52ab1d8/eossr-v0.7.dev20+g52ab1d8.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", - "datePublished": "2022-05-04", - "dateModified": "2022-05-04", + "datePublished": "2022-06-10", + "dateModified": "2022-06-10", "operatingSystem": "", "maintainer": { "@type": "Person", -- GitLab From 5dd652ac148c7479b599fafee8c9fb1a996e24b9 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 12:57:09 +0200 Subject: [PATCH 05/26] addapt test and add ossr module tests --- eossr/metadata/codemeta2zenodo.py | 13 ++++++--- eossr/ossr.py | 11 +++++--- eossr/tests/test_ossr.py | 45 +++++++++++++++++++++++++++++-- eossr/zenodo/tests/test_zenodo.py | 6 ++++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index 4a25d8f..72c171c 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -285,22 +285,27 @@ class CodeMeta2ZenodoController(object): json.dump(self.zenodo_data, outfile, indent=4, sort_keys=True) -def converter(codemeta_dict, add_escape2020=True): +def converter(codemeta_dict, add_escape2020=True, dump_zenodo=False): """ Convert codemeta metadata into zenodo metadata :param codemeta_dict: dict :param add_escape2020: bool if True, add escape2020 community and grant - :return: dict - zenodo metadata + :param dump_zenodo: bool + if True, dumps the converted metadata into a file instead of returning a dict + :return: dict or None + zenodo metadata or dumps the metadata file """ meta_converter = CodeMeta2ZenodoController(codemeta_dict) meta_converter.convert() if add_escape2020: meta_converter.add_escape2020_community() meta_converter.add_escape2020_grant() - return meta_converter.zenodo_data + if dump_zenodo: + meta_converter.write_zenodo('.zenodo.json') + else: + return meta_converter.zenodo_data def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add_escape2020=True, overwrite=True): diff --git a/eossr/ossr.py b/eossr/ossr.py index 95541ff..abee090 100644 --- a/eossr/ossr.py +++ b/eossr/ossr.py @@ -23,7 +23,8 @@ class OssrAPI(ZenodoAPI): :param directory: Path or str path to the directory containing the metadata file - :return: metadata loaded dictionary + :return: dict + metadata loaded dictionary """ codemeta_metadata_file = self.path_codemeta_file(directory) if codemeta_metadata_file.exists(): @@ -121,13 +122,15 @@ def get_ossr_records(search='', sandbox=False, **kwargs): return get_zenodo_records(search, sandbox=sandbox, **kwargs) -def get_ossr_pending_requests(**params): +def get_ossr_pending_requests(community=escape_community, **params): """ Get a list of records that have been requested to be added to the OSSR. + :param community: str + Name of the community. DEFAULT: `escape2020` :param params: dict Parameters for the request. Override the class parameters. :return: """ - zen = ZenodoAPI() - return zen.get_community_pending_requests(escape_community, **params) + ossr = OssrAPI() + return ossr.get_community_pending_requests(community, **params) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 20bd228..9a96bf3 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -1,8 +1,49 @@ -from eossr import ossr +import shutil +import tempfile +import unittest +from pathlib import Path + +import pytest + +from eossr import ROOT_DIR +from eossr.ossr import OssrAPI, get_ossr_pending_requests, get_ossr_records def test_get_ossr_records(): - ossr_records = ossr.get_ossr_records(all_versions=1) + ossr_records = get_ossr_records(all_versions=1) assert len(ossr_records) >= 12 # number of records October 01, 2021 all_ids = [rec.data['id'] for rec in ossr_records] assert 5524913 in all_ids # id of the version v0.2 of the eossr + + +def test_get_ossr_pending_requests(): + records = get_ossr_pending_requests('escape2020') + records_ids = [rec.id for rec in records] + assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 + + +class TestOssrAPI(unittest.TestCase): + def __init__(self): + super(TestOssrAPI).__init__() + self.ossr = OssrAPI(sandbox=True) + + @pytest.mark.xfail(raises=FileNotFoundError) + def test_fail_load_metadata(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + self.ossr.load_metadata(tmp_dirname) + + def test_load_metadata(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) + metadata = self.ossr.load_metadata(tmp_dirname) + assert isinstance(metadata, dict) + + def test_check_upload_to_zenodo(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) + _, filename = tempfile.mkstemp(dir=tmp_dirname) + Path(filename).write_text('Hello from test_eossr.py unit tests') + self.ossr.check_upload_to_zenodo(tmp_dirname) diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index 169b5ec..d5b00be 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -10,6 +10,7 @@ import pytest import requests from eossr import ROOT_DIR, zenodo +from eossr.metadata.codemeta2zenodo import converter from eossr.zenodo import HTTPStatusError, Record, ZenodoAPI, get_record, get_zenodo_records, is_live, query_record eossr_test_lib_id = 930570 # test library in sandbox (owner: T. Vuillaume) @@ -91,6 +92,9 @@ class TestZenodoAPIToken(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdirname: print(f"tmpdir {tmpdirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmpdirname) + with open(Path(tmpdirname).joinpath('codemeta.json')) as file: + codemeta = json.load(file) + converter(codemeta, dump_zenodo=True) _, filename = tempfile.mkstemp(dir=tmpdirname) Path(filename).write_text('Hello from eossr unit tests') @@ -160,7 +164,7 @@ def test_get_record_42(): def test_query_record_941144(): - # unsubmitted record in T. Vuillaume's sandbox - for test purposes + # non-submitted record in T. Vuillaume's sandbox - for test purposes answer = query_record(941144, sandbox=True) assert answer.status_code == 404 -- GitLab From 0fe50b7e5287ee2b810a53f07609f08284855786 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 10:58:59 +0000 Subject: [PATCH 06/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index 3179fe9..e39cfe0 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev20+g52ab1d8", - "softwareVersion": "v0.7.dev20+g52ab1d8", + "version": "v0.7.dev23+gcae8f30", + "softwareVersion": "v0.7.dev23+gcae8f30", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev20+g52ab1d8/eossr-v0.7.dev20+g52ab1d8.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev23+gcae8f30/eossr-v0.7.dev23+gcae8f30.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From cd24647f4cef4c34a6d637244b88c51ac5e2e856 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 13:22:28 +0200 Subject: [PATCH 07/26] fix tests --- eossr/tests/test_ossr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 9a96bf3..634d1e4 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -1,3 +1,4 @@ +import os import shutil import tempfile import unittest @@ -25,7 +26,8 @@ def test_get_ossr_pending_requests(): class TestOssrAPI(unittest.TestCase): def __init__(self): super(TestOssrAPI).__init__() - self.ossr = OssrAPI(sandbox=True) + self.token = None + self.ossr = OssrAPI(access_token=self.token, sandbox=True) @pytest.mark.xfail(raises=FileNotFoundError) def test_fail_load_metadata(self): @@ -40,7 +42,9 @@ class TestOssrAPI(unittest.TestCase): metadata = self.ossr.load_metadata(tmp_dirname) assert isinstance(metadata, dict) + @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") def test_check_upload_to_zenodo(self): + self.token = os.getenv('SANDBOX_ZENODO_TOKEN') with tempfile.TemporaryDirectory() as tmp_dirname: print(f"tmpdir {tmp_dirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) -- GitLab From d1f41cb44c75e11d901b92bebe0ae94a8905552e Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 11:23:27 +0000 Subject: [PATCH 08/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index e39cfe0..6a16528 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev23+gcae8f30", - "softwareVersion": "v0.7.dev23+gcae8f30", + "version": "v0.7.dev25+gcd24647", + "softwareVersion": "v0.7.dev25+gcd24647", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev23+gcae8f30/eossr-v0.7.dev23+gcae8f30.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev25+gcd24647/eossr-v0.7.dev25+gcd24647.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From 1b51c1dacfe24cbc8c77e71fb70471930c4af7bb Mon Sep 17 00:00:00 2001 From: GITLAB_TOKEN Date: Fri, 10 Jun 2022 11:24:12 +0000 Subject: [PATCH 09/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index 6a16528..2ccdc04 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev25+gcd24647", - "softwareVersion": "v0.7.dev25+gcd24647", + "version": "v0.7.dev26+gd1f41cb", + "softwareVersion": "v0.7.dev26+gd1f41cb", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev25+gcd24647/eossr-v0.7.dev25+gcd24647.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev26+gd1f41cb/eossr-v0.7.dev26+gd1f41cb.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From c1413e2a69978e11eb11cbb0df22204dba3a83be Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 14:44:05 +0200 Subject: [PATCH 10/26] all OssrAPI test should be with os.getenv(SANDBOX_ZENODO_TOKEN) --- eossr/tests/test_ossr.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 634d1e4..7db4722 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -23,10 +23,11 @@ def test_get_ossr_pending_requests(): assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 +@pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") class TestOssrAPI(unittest.TestCase): def __init__(self): super(TestOssrAPI).__init__() - self.token = None + self.token = os.getenv('SANDBOX_ZENODO_TOKEN') self.ossr = OssrAPI(access_token=self.token, sandbox=True) @pytest.mark.xfail(raises=FileNotFoundError) @@ -42,9 +43,7 @@ class TestOssrAPI(unittest.TestCase): metadata = self.ossr.load_metadata(tmp_dirname) assert isinstance(metadata, dict) - @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") def test_check_upload_to_zenodo(self): - self.token = os.getenv('SANDBOX_ZENODO_TOKEN') with tempfile.TemporaryDirectory() as tmp_dirname: print(f"tmpdir {tmp_dirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) -- GitLab From 55db7455bc0c65914b62e73230f7e1034d584224 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 15:09:30 +0200 Subject: [PATCH 11/26] fix errors in ossr_tests and modify zenodo_test so that they use .zenodo.json --- eossr/tests/test_ossr.py | 56 ++++++++++++++++++++++++------- eossr/zenodo/tests/test_zenodo.py | 20 ++++++++--- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 7db4722..5be8ed6 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -9,6 +9,23 @@ import pytest from eossr import ROOT_DIR from eossr.ossr import OssrAPI, get_ossr_pending_requests, get_ossr_records +zenodo_key_fields = [ + 'access_right', + 'communities', + 'creators', + 'description', + 'grants', + 'keywords', + 'language', + 'license', + 'notes', + 'publication_date', + 'related_identifiers', + 'title', + 'upload_type', + 'version', +] + def test_get_ossr_records(): ossr_records = get_ossr_records(all_versions=1) @@ -23,12 +40,26 @@ def test_get_ossr_pending_requests(): assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 -@pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") -class TestOssrAPI(unittest.TestCase): - def __init__(self): - super(TestOssrAPI).__init__() - self.token = os.getenv('SANDBOX_ZENODO_TOKEN') - self.ossr = OssrAPI(access_token=self.token, sandbox=True) +class TestOssrAPINoToken(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(TestOssrAPINoToken, self).__init__(*args, **kwargs) + self.token = '' + self.ossr = OssrAPI(access_token=self.token, sandbox=False) + + def test_initialization(self): + assert isinstance(self.ossr, OssrAPI) + assert self.ossr.api_url == 'https://zenodo.org/api' + assert self.ossr.access_token == self.token + + @pytest.mark.xfail(raises=ValueError) + def test_raise_token_status(self): + # A value error should be raised as no valid token was provided + self.ossr._raise_token_status() + + def test_load_metadata(self): + metadata = self.ossr.load_metadata(ROOT_DIR) + assert isinstance(metadata, dict) + assert all(item in zenodo_key_fields for item in list(metadata.keys())) @pytest.mark.xfail(raises=FileNotFoundError) def test_fail_load_metadata(self): @@ -36,12 +67,13 @@ class TestOssrAPI(unittest.TestCase): print(f"tmpdir {tmp_dirname}") self.ossr.load_metadata(tmp_dirname) - def test_load_metadata(self): - with tempfile.TemporaryDirectory() as tmp_dirname: - print(f"tmpdir {tmp_dirname}") - shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) - metadata = self.ossr.load_metadata(tmp_dirname) - assert isinstance(metadata, dict) + +@pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") +class TestOssrAPI(unittest.TestCase): + def __init__(self): + super(TestOssrAPI).__init__() + self.token = os.getenv('SANDBOX_ZENODO_TOKEN') + self.ossr = OssrAPI(access_token=self.token, sandbox=True) def test_check_upload_to_zenodo(self): with tempfile.TemporaryDirectory() as tmp_dirname: diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index d5b00be..2b805d8 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -10,7 +10,8 @@ import pytest import requests from eossr import ROOT_DIR, zenodo -from eossr.metadata.codemeta2zenodo import converter +from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file +from eossr.tests.test_ossr import zenodo_key_fields from eossr.zenodo import HTTPStatusError, Record, ZenodoAPI, get_record, get_zenodo_records, is_live, query_record eossr_test_lib_id = 930570 # test library in sandbox (owner: T. Vuillaume) @@ -59,6 +60,17 @@ class TestZenodoAPINoToken(unittest.TestCase): # A value error should be raised as no valid token was provided self.zenodo._raise_token_status() + def test_load_metadata(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + print(f"tmpdir {tmp_dirname}") + shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) + parse_codemeta_and_write_zenodo_metadata_file( + tmp_dirname.joinpath('codemeta.json'), tmp_dirname.joinpath('.zenodo.json') + ) + metadata = self.zenodo.load_metadata(tmp_dirname) + assert isinstance(metadata, dict) + assert all(item in zenodo_key_fields for item in list(metadata.keys())) + @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") class TestZenodoAPIToken(unittest.TestCase): @@ -92,9 +104,9 @@ class TestZenodoAPIToken(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdirname: print(f"tmpdir {tmpdirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmpdirname) - with open(Path(tmpdirname).joinpath('codemeta.json')) as file: - codemeta = json.load(file) - converter(codemeta, dump_zenodo=True) + parse_codemeta_and_write_zenodo_metadata_file( + tmpdirname.joinpath('codemeta.json'), tmpdirname.joinpath('.zenodo.json') + ) _, filename = tempfile.mkstemp(dir=tmpdirname) Path(filename).write_text('Hello from eossr unit tests') -- GitLab From 703d55fbae481042b81a0118fabc9b91b4b0f949 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 13:12:36 +0000 Subject: [PATCH 12/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/codemeta.json b/codemeta.json index 99430bc..9312c0b 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev26+gd1f41cb", - "softwareVersion": "v0.7.dev26+gd1f41cb", + "version": "v0.7.dev33+g66f7966", + "softwareVersion": "v0.7.dev33+g66f7966", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,11 +43,11 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev26+gd1f41cb/eossr-v0.7.dev26+gd1f41cb.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev33+g66f7966/eossr-v0.7.dev33+g66f7966.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", - "datePublished": "2022-05-23", - "dateModified": "2022-05-23", + "datePublished": "2022-06-10", + "dateModified": "2022-06-10", "operatingSystem": "", "maintainer": { "@type": "Person", @@ -115,4 +115,4 @@ } ], "funding": "824064" -} +} \ No newline at end of file -- GitLab From d2c522f6fc68dd646ba5d8abc1f142b7e4261f3d Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 15:23:57 +0200 Subject: [PATCH 13/26] move test_get_ossr_pending_requests to tests that needs token --- eossr/tests/test_ossr.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 5be8ed6..481e48e 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -34,12 +34,6 @@ def test_get_ossr_records(): assert 5524913 in all_ids # id of the version v0.2 of the eossr -def test_get_ossr_pending_requests(): - records = get_ossr_pending_requests('escape2020') - records_ids = [rec.id for rec in records] - assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 - - class TestOssrAPINoToken(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestOssrAPINoToken, self).__init__(*args, **kwargs) @@ -70,11 +64,16 @@ class TestOssrAPINoToken(unittest.TestCase): @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") class TestOssrAPI(unittest.TestCase): - def __init__(self): - super(TestOssrAPI).__init__() + def __init__(self, *args, **kwargs): + super(TestOssrAPI).__init__(*args, **kwargs) self.token = os.getenv('SANDBOX_ZENODO_TOKEN') self.ossr = OssrAPI(access_token=self.token, sandbox=True) + def test_get_ossr_pending_requests(self): + records = get_ossr_pending_requests('escape2020') + records_ids = [rec.id for rec in records] + assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 + def test_check_upload_to_zenodo(self): with tempfile.TemporaryDirectory() as tmp_dirname: print(f"tmpdir {tmp_dirname}") -- GitLab From d890956dbee42fbb39323eabd5ca7e25ad83480c Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 13:24:48 +0000 Subject: [PATCH 14/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index 9312c0b..f6eeefb 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev33+g66f7966", - "softwareVersion": "v0.7.dev33+g66f7966", + "version": "v0.7.dev35+gd2c522f", + "softwareVersion": "v0.7.dev35+gd2c522f", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev33+g66f7966/eossr-v0.7.dev33+g66f7966.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev35+gd2c522f/eossr-v0.7.dev35+gd2c522f.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From 06a0aaeaa37b5a40b26ddbd67d24f9bc0fc814f8 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 15:31:05 +0200 Subject: [PATCH 15/26] add Path to str path - zenodo_tests- and add self to unittest.TestCase super call --- eossr/tests/test_ossr.py | 2 +- eossr/zenodo/tests/test_zenodo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 481e48e..99c1eba 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -65,7 +65,7 @@ class TestOssrAPINoToken(unittest.TestCase): @pytest.mark.skipif(os.getenv('SANDBOX_ZENODO_TOKEN') is None, reason="SANDBOX_ZENODO_TOKEN not defined") class TestOssrAPI(unittest.TestCase): def __init__(self, *args, **kwargs): - super(TestOssrAPI).__init__(*args, **kwargs) + super(TestOssrAPI, self).__init__(*args, **kwargs) self.token = os.getenv('SANDBOX_ZENODO_TOKEN') self.ossr = OssrAPI(access_token=self.token, sandbox=True) diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index 2b805d8..591d150 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -65,7 +65,7 @@ class TestZenodoAPINoToken(unittest.TestCase): print(f"tmpdir {tmp_dirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) parse_codemeta_and_write_zenodo_metadata_file( - tmp_dirname.joinpath('codemeta.json'), tmp_dirname.joinpath('.zenodo.json') + Path(tmp_dirname).joinpath('codemeta.json'), Path(tmp_dirname).joinpath('.zenodo.json') ) metadata = self.zenodo.load_metadata(tmp_dirname) assert isinstance(metadata, dict) -- GitLab From af78f328b3ec1bc6ef812d53bdaacb951e87cb3b Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 14:02:20 +0000 Subject: [PATCH 16/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codemeta.json b/codemeta.json index dd136ae..061e12d 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev22+g2a70d3e", - "softwareVersion": "v0.7.dev22+g2a70d3e", + "version": "v0.7.dev41+gcafd012", + "softwareVersion": "v0.7.dev41+gcafd012", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev22+g2a70d3e/eossr-v0.7.dev22+g2a70d3e.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev41+gcafd012/eossr-v0.7.dev41+gcafd012.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", @@ -115,4 +115,4 @@ } ], "funding": "824064" -} +} \ No newline at end of file -- GitLab From d32f736385cf6c173e8805b47eacef5f6dbece24 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 16:18:22 +0200 Subject: [PATCH 17/26] merge correctly master --- eossr/ossr.py | 4 +++- eossr/tests/test_ossr.py | 4 ++-- eossr/zenodo/tests/test_zenodo.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eossr/ossr.py b/eossr/ossr.py index abee090..ac49fcc 100644 --- a/eossr/ossr.py +++ b/eossr/ossr.py @@ -102,7 +102,9 @@ def get_ossr_records(search='', sandbox=False, **kwargs): """ # make sure we find all OSSR records without limit on the number - r = requests.get(zenodo_api_url + '/records', params={'communities': escape_community}) + params = kwargs + params['communities'] = escape_community + r = requests.get(zenodo_api_url + '/records', params=params) number_of_ossr_entries = r.json()['aggregations']['access_right']['buckets'][0]['doc_count'] kwargs['size'] = number_of_ossr_entries diff --git a/eossr/tests/test_ossr.py b/eossr/tests/test_ossr.py index 99c1eba..6f9f959 100644 --- a/eossr/tests/test_ossr.py +++ b/eossr/tests/test_ossr.py @@ -28,7 +28,7 @@ zenodo_key_fields = [ def test_get_ossr_records(): - ossr_records = get_ossr_records(all_versions=1) + ossr_records = get_ossr_records(all_versions=True) assert len(ossr_records) >= 12 # number of records October 01, 2021 all_ids = [rec.data['id'] for rec in ossr_records] assert 5524913 in all_ids # id of the version v0.2 of the eossr @@ -70,7 +70,7 @@ class TestOssrAPI(unittest.TestCase): self.ossr = OssrAPI(access_token=self.token, sandbox=True) def test_get_ossr_pending_requests(self): - records = get_ossr_pending_requests('escape2020') + records = get_ossr_pending_requests(**{"access_token": self.token}) records_ids = [rec.id for rec in records] assert 532458 in records_ids # pending request in escape2020 sandbox community - 2021-11-28 diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index 643537e..b276c99 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -105,7 +105,7 @@ class TestZenodoAPIToken(unittest.TestCase): print(f"tmpdir {tmpdirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmpdirname) parse_codemeta_and_write_zenodo_metadata_file( - tmpdirname.joinpath('codemeta.json'), tmpdirname.joinpath('.zenodo.json') + Path(tmpdirname).joinpath('codemeta.json'), Path(tmpdirname).joinpath('.zenodo.json') ) _, filename = tempfile.mkstemp(dir=tmpdirname) Path(filename).write_text('Hello from eossr unit tests') -- GitLab From b0e931034eb166285434cc229d6440b9a7dc540a Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 14:20:20 +0000 Subject: [PATCH 18/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index 061e12d..29f4cbb 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev41+gcafd012", - "softwareVersion": "v0.7.dev41+gcafd012", + "version": "v0.7.dev44+gbbcf13b", + "softwareVersion": "v0.7.dev44+gbbcf13b", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev41+gcafd012/eossr-v0.7.dev41+gcafd012.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev44+gbbcf13b/eossr-v0.7.dev44+gbbcf13b.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From 3bc44705d16204c79670a0fa111db2ffa2a340a1 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 17:18:42 +0200 Subject: [PATCH 19/26] test creation of metadata temp files --- eossr/zenodo/tests/test_zenodo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index b276c99..9900ce2 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -67,6 +67,8 @@ class TestZenodoAPINoToken(unittest.TestCase): parse_codemeta_and_write_zenodo_metadata_file( Path(tmp_dirname).joinpath('codemeta.json'), Path(tmp_dirname).joinpath('.zenodo.json') ) + assert Path(tmp_dirname).joinpath('codemeta.json').exists() + assert Path(tmp_dirname).joinpath('.zenodo.json').exists() metadata = self.zenodo.load_metadata(tmp_dirname) assert isinstance(metadata, dict) assert all(item in zenodo_key_fields for item in list(metadata.keys())) -- GitLab From 3b716c72f7a9b2682c6a5b44f68da7d746a524d9 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Fri, 10 Jun 2022 15:19:14 +0000 Subject: [PATCH 20/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codemeta.json b/codemeta.json index 29f4cbb..dee9adc 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev44+gbbcf13b", - "softwareVersion": "v0.7.dev44+gbbcf13b", + "version": "v0.7.dev47+g5e68823", + "softwareVersion": "v0.7.dev47+g5e68823", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,7 +43,7 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev44+gbbcf13b/eossr-v0.7.dev44+gbbcf13b.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev47+g5e68823/eossr-v0.7.dev47+g5e68823.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", "datePublished": "2022-06-10", -- GitLab From 9e533fd89469580b2240ef6896655bc587e1197a Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Mon, 13 Jun 2022 16:04:48 +0200 Subject: [PATCH 21/26] return converter to its original state and make not proud way of testing the failing test --- eossr/metadata/codemeta2zenodo.py | 16 ++++++---------- eossr/zenodo/tests/test_zenodo.py | 14 ++++++++++---- eossr/zenodo/zenodo.py | 3 +++ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index 72c171c..5d56682 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -285,27 +285,22 @@ class CodeMeta2ZenodoController(object): json.dump(self.zenodo_data, outfile, indent=4, sort_keys=True) -def converter(codemeta_dict, add_escape2020=True, dump_zenodo=False): +def converter(codemeta_dict, add_escape2020=True): """ - Convert codemeta metadata into zenodo metadata + Convert an already loaded codemeta metadata json file into a zenodo metadata dict :param codemeta_dict: dict :param add_escape2020: bool if True, add escape2020 community and grant - :param dump_zenodo: bool - if True, dumps the converted metadata into a file instead of returning a dict - :return: dict or None - zenodo metadata or dumps the metadata file + :return: dict + zenodo metadata """ meta_converter = CodeMeta2ZenodoController(codemeta_dict) meta_converter.convert() if add_escape2020: meta_converter.add_escape2020_community() meta_converter.add_escape2020_grant() - if dump_zenodo: - meta_converter.write_zenodo('.zenodo.json') - else: - return meta_converter.zenodo_data + return meta_converter.zenodo_data def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add_escape2020=True, overwrite=True): @@ -330,5 +325,6 @@ def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add outfile = Path(outdir).joinpath('.zenodo.json') if not outfile.exists() or overwrite: meta_converter.write_zenodo(outfile.name) + return outfile else: raise FileExistsError(f"The file {outfile} exists. Use overwrite.") diff --git a/eossr/zenodo/tests/test_zenodo.py b/eossr/zenodo/tests/test_zenodo.py index 9900ce2..d9593a8 100644 --- a/eossr/zenodo/tests/test_zenodo.py +++ b/eossr/zenodo/tests/test_zenodo.py @@ -10,7 +10,7 @@ import pytest import requests from eossr import ROOT_DIR, zenodo -from eossr.metadata.codemeta2zenodo import parse_codemeta_and_write_zenodo_metadata_file +from eossr.metadata.codemeta2zenodo import CodeMeta2ZenodoController, parse_codemeta_and_write_zenodo_metadata_file from eossr.tests.test_ossr import zenodo_key_fields from eossr.zenodo import HTTPStatusError, Record, ZenodoAPI, get_record, get_zenodo_records, is_live, query_record @@ -64,9 +64,15 @@ class TestZenodoAPINoToken(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp_dirname: print(f"tmpdir {tmp_dirname}") shutil.copy(ROOT_DIR.joinpath('codemeta.json'), tmp_dirname) - parse_codemeta_and_write_zenodo_metadata_file( - Path(tmp_dirname).joinpath('codemeta.json'), Path(tmp_dirname).joinpath('.zenodo.json') - ) + + # parse_codemeta_and_write_zenodo_metadata_file does not work fine with temp dirs, + # thus writing explicitly the .zenodo.json file + converter = CodeMeta2ZenodoController.from_file(Path(tmp_dirname).joinpath('codemeta.json')) + converter.convert() + converter.add_escape2020_community() + converter.add_escape2020_grant() + converter.write_zenodo(Path(tmp_dirname).joinpath('.zenodo.json')) + assert Path(tmp_dirname).joinpath('codemeta.json').exists() assert Path(tmp_dirname).joinpath('.zenodo.json').exists() metadata = self.zenodo.load_metadata(tmp_dirname) diff --git a/eossr/zenodo/zenodo.py b/eossr/zenodo/zenodo.py index 9d0326f..7828e2e 100644 --- a/eossr/zenodo/zenodo.py +++ b/eossr/zenodo/zenodo.py @@ -378,6 +378,9 @@ class ZenodoAPI: :return: Loaded metadata dictionary """ zenodo_metadata_file = self.path_zenodo_file(directory) + import os + + os.system(f"ls -lrt {directory}") if zenodo_metadata_file.exists(): print(f" - Record metadata based on zenodo file {zenodo_metadata_file}") with open(zenodo_metadata_file) as file: -- GitLab From 61e666c0a9eed7635e411df25904321367aed315 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Mon, 13 Jun 2022 16:06:12 +0200 Subject: [PATCH 22/26] take out forgotten debug code in zenodo submodule --- eossr/zenodo/zenodo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/eossr/zenodo/zenodo.py b/eossr/zenodo/zenodo.py index 7828e2e..9d0326f 100644 --- a/eossr/zenodo/zenodo.py +++ b/eossr/zenodo/zenodo.py @@ -378,9 +378,6 @@ class ZenodoAPI: :return: Loaded metadata dictionary """ zenodo_metadata_file = self.path_zenodo_file(directory) - import os - - os.system(f"ls -lrt {directory}") if zenodo_metadata_file.exists(): print(f" - Record metadata based on zenodo file {zenodo_metadata_file}") with open(zenodo_metadata_file) as file: -- GitLab From f6a7607d2cf9d83711513d1e53e6bb10bca0afb0 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Mon, 13 Jun 2022 14:07:50 +0000 Subject: [PATCH 23/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codemeta.json b/codemeta.json index dee9adc..71caef6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev47+g5e68823", - "softwareVersion": "v0.7.dev47+g5e68823", + "version": "v0.7.dev51+g2a2a78f", + "softwareVersion": "v0.7.dev51+g2a2a78f", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,11 +43,11 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev47+g5e68823/eossr-v0.7.dev47+g5e68823.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev51+g2a2a78f/eossr-v0.7.dev51+g2a2a78f.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", - "datePublished": "2022-06-10", - "dateModified": "2022-06-10", + "datePublished": "2022-06-13", + "dateModified": "2022-06-13", "operatingSystem": "", "maintainer": { "@type": "Person", -- GitLab From 99bf799469ebe09905b1b8ac189f5eeb6c0a2e04 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Tue, 14 Jun 2022 10:13:50 +0200 Subject: [PATCH 24/26] typo --- eossr/metadata/codemeta2zenodo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eossr/metadata/codemeta2zenodo.py b/eossr/metadata/codemeta2zenodo.py index 5d56682..1ad2dae 100644 --- a/eossr/metadata/codemeta2zenodo.py +++ b/eossr/metadata/codemeta2zenodo.py @@ -315,7 +315,7 @@ def parse_codemeta_and_write_zenodo_metadata_file(codemeta_filename, outdir, add add_escape2020: bool adds escape2020 metadata in zenodo metadata file overwrite: bool - overwrite existing `.zendoo.json` file in `outdir` + overwrite existing `.zenodo.json` file in `outdir` """ meta_converter = CodeMeta2ZenodoController.from_file(codemeta_filename) meta_converter.convert() -- GitLab From 9b9a2146b9e7219e37e5ae6c9436a9aab1da1c5c Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Tue, 14 Jun 2022 11:54:02 +0200 Subject: [PATCH 25/26] fix ossr.check_upload --- eossr/ossr.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eossr/ossr.py b/eossr/ossr.py index ac49fcc..86f5305 100644 --- a/eossr/ossr.py +++ b/eossr/ossr.py @@ -60,16 +60,15 @@ class OssrAPI(ZenodoAPI): Path to directory whose content will be uploaded to Zenodo """ self._raise_token_status() - path_zenodo_file = self.path_zenodo_file(directory) path_codemeta_file = self.path_codemeta_file(directory) if not path_codemeta_file.exists(): raise FileNotFoundError(f"No codemeta {path_codemeta_file} file found.") print("\n * Performing the codemeta to zenodo conversion... ") - parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, path_zenodo_file) + zenodo_file = parse_codemeta_and_write_zenodo_metadata_file(path_codemeta_file, directory) - self._test_zenodo_connection(path_zenodo_file) + self._test_zenodo_connection(zenodo_file) def get_ossr_records(search='', sandbox=False, **kwargs): -- GitLab From 7c4352dc70401c28cbb01a953ca135009a65e8f7 Mon Sep 17 00:00:00 2001 From: Enrique Garcia Date: Tue, 14 Jun 2022 09:54:39 +0000 Subject: [PATCH 26/26] Auto-update codemeta.json [skip ci] --- codemeta.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codemeta.json b/codemeta.json index 71caef6..9bdf66b 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,8 +4,8 @@ "name": "eossr", "description": "

The ESCAPE OSSR library

The eOSSR Python library gathers all the developments made for the OSSR. In particular, it includes:- an API to programmatically access the OSSR, retrieve records and publish content- functions to map and crosswalk metadata between the CodeMeta schema adopted for the OSSR and Zenodo internal schema- functions to help developers automatically contribute to the OSSR, in particular using their continuous integration (see also code snippets)

Code: https://gitlab.in2p3.fr/escape2020/wp3/eossrDocumentation: https://escape2020.pages.in2p3.fr/wp3/eossr/

\"\"\"\"\"CII\"\"\"\"\"\"

Former stable versions

  • v0.6: \"DOI\"
  • v0.5: \"DOI\"
  • v0.4: \"DOI\"
  • v0.3.3: \"\"
  • v0.2 : \"\"

Install

Commands to be run in your terminal.

For users

pip install eossr

You can also run it with docker:

docker run -it gitlab-registry.in2p3.fr/escape2020/wp3/eossr:latest

Visit our registry to see the available docker containers.

Note that latest tag always point to the latest stable released container.

For developers

git clone https://gitlab.in2p3.fr/escape2020/wp3/eossr.gitpip install -e "./eossr"

Running tests

To run tests locally, run:

pip install -e "./eossr[tests]"pytest eossr

Some tests will be skiped if SANDBOX_ZENODO_TOKEN is not defined in your environment variables.If you want to run these tests, you will need to create a sandbox zenodo token and add it to your env:

export SANDBOX_ZENODO_TOKEN="your_sandbox_token"

License

See LICENSE

Cite

To cite this library, use the cite section in the Zenodo page (rightcolumn, below the Versions section).

", "license": "https://spdx.org/licenses/MIT", - "version": "v0.7.dev51+g2a2a78f", - "softwareVersion": "v0.7.dev51+g2a2a78f", + "version": "v0.7.dev55+g3ba6238", + "softwareVersion": "v0.7.dev55+g3ba6238", "codeRepository": "https://gitlab.in2p3.fr/escape2020/wp3/eossr", "developmentStatus": "active", "isAccessibleForFree": true, @@ -43,11 +43,11 @@ "zenodo" ], "runtimePlatform": "Python 3", - "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev51+g2a2a78f/eossr-v0.7.dev51+g2a2a78f.zip", + "downloadUrl": "https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/archive/v0.7.dev55+g3ba6238/eossr-v0.7.dev55+g3ba6238.zip", "releaseNotes": "eossr v0.6 Minor bug fixes and documentation improvements. Full changelog: https://gitlab.in2p3.fr/escape2020/wp3/eossr/-/releases/v0.6", "dateCreated": "2021-08-31", - "datePublished": "2022-06-13", - "dateModified": "2022-06-13", + "datePublished": "2022-06-14", + "dateModified": "2022-06-14", "operatingSystem": "", "maintainer": { "@type": "Person", -- GitLab