diff --git a/.gitlab/ci/tezt.yml b/.gitlab/ci/tezt.yml index 2fc78f4e9cba3259583208b1c8fcc4090601dd68..613d269f0e94fa53bb5779cd3152ca5eb9fdf1df 100644 --- a/.gitlab/ci/tezt.yml +++ b/.gitlab/ci/tezt.yml @@ -3,6 +3,8 @@ .tezt_template: extends: .integration_template artifacts: + reports: + junit: tezt-junit.xml paths: - tezt.log expire_in: 1 day @@ -13,17 +15,17 @@ tezt:1: extends: .tezt_template script: - - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --test ''Edo: current encoding regression test: level'' --test ''Alpha: alpha encoding regression test: level'' --test ''Alpha: alpha encoding regression test: operation.raw'' --test ''Alpha: alpha encoding regression test: seed'' --test ''Edo: current encoding regression test: seed'' --test ''Alpha: alpha encoding regression test: timestamp'' --test ''Edo: current encoding regression test: fitness'' --test ''Edo: current encoding regression test: block_header'' --test ''Edo: current encoding regression test: block_header.raw'' --test ''Alpha: alpha encoding regression test: nonce'' --test ''Edo: current encoding regression test: period'' --test ''Edo: current encoding regression test: cycle'' --test ''Edo: current encoding regression test: gas.cost'' --test ''Edo: current encoding regression test: voting_period'' --test ''Alpha: alpha encoding regression test: period'' --test ''Edo: current encoding regression test: vote.ballot'' --test ''Alpha: alpha encoding regression test: gas'' --test ''Alpha: alpha encoding regression test: voting_period.kind'' --test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --test ''Alpha: normalize data (mockup)'' --test ''Edo: current encoding regression test: operation.internal'' --test ''Alpha: (Mockup) origination fees from unrevealed'' --test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --test ''Alpha: hash data ... of type ... (bad)'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''(Mockup) Migration (transfer)'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Alpha: normalize data'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --test ''Edo: (Proxy) Wrong proto'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Edo: current encoding regression test: operation.unsigned'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --test ''Alpha: alpha (mode client) RPC regression tests: others'' --test ''Alpha: alpha encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Alpha: node initialization (full mode)'' --test ''Edo: (Proxy) Bake'' --test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --test ''Edo: current (mode client) RPC regression tests: votes'' --test ''Alpha: baking ordering'' --test ''Edo: (Proxy) Compare RPC get'' --test ''tezos-codec dump encodings'' --test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / full)'' --test ''Alpha: node synchronization (archive / archive)'' --test ''Alpha: node synchronization (full / full)'' # 304.566508055s' + - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --junit tezt-junit.xml --time --test ''Edo: current encoding regression test: level'' --test ''Alpha: alpha encoding regression test: level'' --test ''Alpha: alpha encoding regression test: operation.raw'' --test ''Alpha: alpha encoding regression test: seed'' --test ''Edo: current encoding regression test: seed'' --test ''Alpha: alpha encoding regression test: timestamp'' --test ''Edo: current encoding regression test: fitness'' --test ''Edo: current encoding regression test: block_header'' --test ''Edo: current encoding regression test: block_header.raw'' --test ''Alpha: alpha encoding regression test: nonce'' --test ''Edo: current encoding regression test: period'' --test ''Edo: current encoding regression test: cycle'' --test ''Edo: current encoding regression test: gas.cost'' --test ''Edo: current encoding regression test: voting_period'' --test ''Alpha: alpha encoding regression test: period'' --test ''Edo: current encoding regression test: vote.ballot'' --test ''Alpha: alpha encoding regression test: gas'' --test ''Alpha: alpha encoding regression test: voting_period.kind'' --test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --test ''Alpha: normalize data (mockup)'' --test ''Edo: current encoding regression test: operation.internal'' --test ''Alpha: (Mockup) origination fees from unrevealed'' --test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --test ''Alpha: hash data ... of type ... (bad)'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''(Mockup) Migration (transfer)'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Alpha: normalize data'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --test ''Edo: (Proxy) Wrong proto'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Edo: current encoding regression test: operation.unsigned'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --test ''Alpha: alpha (mode client) RPC regression tests: others'' --test ''Alpha: alpha encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Alpha: node initialization (full mode)'' --test ''Edo: (Proxy) Bake'' --test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --test ''Edo: current (mode client) RPC regression tests: votes'' --test ''Alpha: baking ordering'' --test ''Edo: (Proxy) Compare RPC get'' --test ''tezos-codec dump encodings'' --test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / full)'' --test ''Alpha: node synchronization (archive / archive)'' --test ''Alpha: node synchronization (full / full)'' # 304.566508055s' tezt:2: extends: .tezt_template script: - - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --test ''Edo: current encoding regression test: nonce'' --test ''Alpha: alpha encoding regression test: fitness'' --test ''Alpha: alpha encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header'' --test ''Edo: current encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header.raw'' --test ''Edo: current encoding regression test: timestamp'' --test ''Edo: current encoding regression test: contract'' --test ''Alpha: alpha encoding regression test: cycle'' --test ''Alpha: alpha encoding regression test: raw_level'' --test ''Alpha: alpha encoding regression test: tez'' --test ''Alpha: alpha encoding regression test: gas.cost'' --test ''Alpha: alpha encoding regression test: contract'' --test ''Alpha: (Mockup) RPC header/shell'' --test ''Alpha: alpha encoding regression test: vote.ballot'' --test ''Edo: current encoding regression test: voting_period.kind'' --test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --test ''Edo: current encoding regression test: delegate.balance_updates'' --test ''Alpha: (Mockup) Transfer (asynchronous)'' --test ''Alpha: alpha encoding regression test: operation.internal'' --test ''CLI under connections cap'' --test ''Edo: (Mockup) Transfer'' --test ''Alpha: (Mockup) Transfer'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --test ''Edo: current encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: current (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) RPC get''\''''s location'' --test ''Alpha: node initialization (archive mode)'' --test ''Alpha: check --connection=1 option'' --test ''Edo: (Proxy) Transfer'' --test ''Edo: current (mode proxy) RPC regression tests: delegates'' --test ''Alpha: Check prevalidator start'' --test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --test p2p-maintenance-init-expected_connections --test ''Alpha: (Proxy) Compare RPC get'' --test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --test ''Alpha: double baking with accuser'' --test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / archive)'' --test ''Alpha: node synchronization (full / archive)'' --test ''Alpha: node synchronization (archive / rolling)'' # 304.513179541s' + - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --junit tezt-junit.xml --time --test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --test ''Edo: current encoding regression test: nonce'' --test ''Alpha: alpha encoding regression test: fitness'' --test ''Alpha: alpha encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header'' --test ''Edo: current encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header.raw'' --test ''Edo: current encoding regression test: timestamp'' --test ''Edo: current encoding regression test: contract'' --test ''Alpha: alpha encoding regression test: cycle'' --test ''Alpha: alpha encoding regression test: raw_level'' --test ''Alpha: alpha encoding regression test: tez'' --test ''Alpha: alpha encoding regression test: gas.cost'' --test ''Alpha: alpha encoding regression test: contract'' --test ''Alpha: (Mockup) RPC header/shell'' --test ''Alpha: alpha encoding regression test: vote.ballot'' --test ''Edo: current encoding regression test: voting_period.kind'' --test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --test ''Edo: current encoding regression test: delegate.balance_updates'' --test ''Alpha: (Mockup) Transfer (asynchronous)'' --test ''Alpha: alpha encoding regression test: operation.internal'' --test ''CLI under connections cap'' --test ''Edo: (Mockup) Transfer'' --test ''Alpha: (Mockup) Transfer'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --test ''Edo: current encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: current (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) RPC get''\''''s location'' --test ''Alpha: node initialization (archive mode)'' --test ''Alpha: check --connection=1 option'' --test ''Edo: (Proxy) Transfer'' --test ''Edo: current (mode proxy) RPC regression tests: delegates'' --test ''Alpha: Check prevalidator start'' --test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --test p2p-maintenance-init-expected_connections --test ''Alpha: (Proxy) Compare RPC get'' --test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --test ''Alpha: double baking with accuser'' --test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / archive)'' --test ''Alpha: node synchronization (full / archive)'' --test ''Alpha: node synchronization (archive / rolling)'' # 304.513179541s' tezt:3: extends: .tezt_template script: - - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --not-test ''Alpha: node synchronization (archive / rolling)'' --not-test ''Alpha: node synchronization (full / archive)'' --not-test ''Alpha: node synchronization (rolling / archive)'' --not-test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: double baking with accuser'' --not-test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --not-test ''Alpha: (Proxy) Compare RPC get'' --not-test p2p-maintenance-init-expected_connections --not-test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --not-test ''Alpha: Check prevalidator start'' --not-test ''Edo: current (mode proxy) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Transfer'' --not-test ''Alpha: check --connection=1 option'' --not-test ''Alpha: node initialization (archive mode)'' --not-test ''Alpha: (Proxy) RPC get''\''''s location'' --not-test ''Edo: current (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Edo: current encoding regression test: operation'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: (Mockup) Transfer'' --not-test ''Edo: (Mockup) Transfer'' --not-test ''CLI under connections cap'' --not-test ''Alpha: alpha encoding regression test: operation.internal'' --not-test ''Alpha: (Mockup) Transfer (asynchronous)'' --not-test ''Edo: current encoding regression test: delegate.balance_updates'' --not-test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --not-test ''Edo: current encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: vote.ballot'' --not-test ''Alpha: (Mockup) RPC header/shell'' --not-test ''Alpha: alpha encoding regression test: contract'' --not-test ''Alpha: alpha encoding regression test: gas.cost'' --not-test ''Alpha: alpha encoding regression test: tez'' --not-test ''Alpha: alpha encoding regression test: raw_level'' --not-test ''Alpha: alpha encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: contract'' --not-test ''Edo: current encoding regression test: timestamp'' --not-test ''Alpha: alpha encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: block_header'' --not-test ''Alpha: alpha encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: fitness'' --not-test ''Edo: current encoding regression test: nonce'' --not-test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --not-test ''Alpha: node synchronization (full / full)'' --not-test ''Alpha: node synchronization (archive / archive)'' --not-test ''Alpha: node synchronization (rolling / full)'' --not-test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --not-test ''tezos-codec dump encodings'' --not-test ''Edo: (Proxy) Compare RPC get'' --not-test ''Alpha: baking ordering'' --not-test ''Edo: current (mode client) RPC regression tests: votes'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --not-test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Bake'' --not-test ''Alpha: node initialization (full mode)'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Alpha: alpha encoding regression test: operation'' --not-test ''Alpha: alpha (mode client) RPC regression tests: others'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: current encoding regression test: operation.unsigned'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) Wrong proto'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --not-test ''Alpha: normalize data'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''(Mockup) Migration (transfer)'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: hash data ... of type ... (bad)'' --not-test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --not-test ''Alpha: (Mockup) origination fees from unrevealed'' --not-test ''Edo: current encoding regression test: operation.internal'' --not-test ''Alpha: normalize data (mockup)'' --not-test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: alpha encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: gas'' --not-test ''Edo: current encoding regression test: vote.ballot'' --not-test ''Alpha: alpha encoding regression test: period'' --not-test ''Edo: current encoding regression test: voting_period'' --not-test ''Edo: current encoding regression test: gas.cost'' --not-test ''Edo: current encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: period'' --not-test ''Alpha: alpha encoding regression test: nonce'' --not-test ''Edo: current encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: block_header'' --not-test ''Edo: current encoding regression test: fitness'' --not-test ''Alpha: alpha encoding regression test: timestamp'' --not-test ''Edo: current encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: operation.raw'' --not-test ''Alpha: alpha encoding regression test: level'' --not-test ''Edo: current encoding regression test: level'' # 304.424400091s' + - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --junit tezt-junit.xml --time --not-test ''Alpha: node synchronization (archive / rolling)'' --not-test ''Alpha: node synchronization (full / archive)'' --not-test ''Alpha: node synchronization (rolling / archive)'' --not-test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: double baking with accuser'' --not-test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --not-test ''Alpha: (Proxy) Compare RPC get'' --not-test p2p-maintenance-init-expected_connections --not-test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --not-test ''Alpha: Check prevalidator start'' --not-test ''Edo: current (mode proxy) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Transfer'' --not-test ''Alpha: check --connection=1 option'' --not-test ''Alpha: node initialization (archive mode)'' --not-test ''Alpha: (Proxy) RPC get''\''''s location'' --not-test ''Edo: current (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Edo: current encoding regression test: operation'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: (Mockup) Transfer'' --not-test ''Edo: (Mockup) Transfer'' --not-test ''CLI under connections cap'' --not-test ''Alpha: alpha encoding regression test: operation.internal'' --not-test ''Alpha: (Mockup) Transfer (asynchronous)'' --not-test ''Edo: current encoding regression test: delegate.balance_updates'' --not-test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --not-test ''Edo: current encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: vote.ballot'' --not-test ''Alpha: (Mockup) RPC header/shell'' --not-test ''Alpha: alpha encoding regression test: contract'' --not-test ''Alpha: alpha encoding regression test: gas.cost'' --not-test ''Alpha: alpha encoding regression test: tez'' --not-test ''Alpha: alpha encoding regression test: raw_level'' --not-test ''Alpha: alpha encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: contract'' --not-test ''Edo: current encoding regression test: timestamp'' --not-test ''Alpha: alpha encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: block_header'' --not-test ''Alpha: alpha encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: fitness'' --not-test ''Edo: current encoding regression test: nonce'' --not-test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --not-test ''Alpha: node synchronization (full / full)'' --not-test ''Alpha: node synchronization (archive / archive)'' --not-test ''Alpha: node synchronization (rolling / full)'' --not-test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --not-test ''tezos-codec dump encodings'' --not-test ''Edo: (Proxy) Compare RPC get'' --not-test ''Alpha: baking ordering'' --not-test ''Edo: current (mode client) RPC regression tests: votes'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --not-test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Bake'' --not-test ''Alpha: node initialization (full mode)'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Alpha: alpha encoding regression test: operation'' --not-test ''Alpha: alpha (mode client) RPC regression tests: others'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: current encoding regression test: operation.unsigned'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) Wrong proto'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --not-test ''Alpha: normalize data'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''(Mockup) Migration (transfer)'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: hash data ... of type ... (bad)'' --not-test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --not-test ''Alpha: (Mockup) origination fees from unrevealed'' --not-test ''Edo: current encoding regression test: operation.internal'' --not-test ''Alpha: normalize data (mockup)'' --not-test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: alpha encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: gas'' --not-test ''Edo: current encoding regression test: vote.ballot'' --not-test ''Alpha: alpha encoding regression test: period'' --not-test ''Edo: current encoding regression test: voting_period'' --not-test ''Edo: current encoding regression test: gas.cost'' --not-test ''Edo: current encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: period'' --not-test ''Alpha: alpha encoding regression test: nonce'' --not-test ''Edo: current encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: block_header'' --not-test ''Edo: current encoding regression test: fitness'' --not-test ''Alpha: alpha encoding regression test: timestamp'' --not-test ''Edo: current encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: operation.raw'' --not-test ''Alpha: alpha encoding regression test: level'' --not-test ''Edo: current encoding regression test: level'' # 304.424400091s' ##END_TEZTTEST## tezt:manual:migration: diff --git a/scripts/update_tezt_test.ml b/scripts/update_tezt_test.ml index 38006534dbd2ed5d868f8063697940b3643abfed..06c477dcb0c9abe80e8e9a8cc9a5b4a478c4a79c 100755 --- a/scripts/update_tezt_test.ml +++ b/scripts/update_tezt_test.ml @@ -64,7 +64,7 @@ let () = output_string output {| extends: .tezt_template script: - - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time |}; + - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --junit tezt-junit.xml --time |}; let double_single_quotes s = String.split_on_char '\'' s |> String.concat "''" in diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml index b7ae35868f3de00561d1e216e6d9b164668d3601..5176b92021d92a0512d089eb05625b1075956c39 100644 --- a/tezt/lib/cli.ml +++ b/tezt/lib/cli.ml @@ -53,6 +53,7 @@ type options = { record : string option; job_count : int; suggest_jobs : string option; + junit : string option; } let options = @@ -80,6 +81,7 @@ let options = let record = ref None in let job_count = ref 3 in let suggest_jobs = ref None in + let junit = ref None in let set_log_level = function | "quiet" -> log_level := Quiet @@ -230,7 +232,10 @@ let options = suggest a partition of the tests that would result in --job-count \ sets of roughly the same total duration. Output each job as a list \ of flags that can be passed to Tezt, followed by a shell comment \ - that denotes the expected duration of the job." ) ] + that denotes the expected duration of the job." ); + ( "--junit", + Arg.String (fun path -> junit := Some path), + " Store test results in FILE using JUnit XML format." ) ] in let usage = (* This was formatted by ocamlformat. Sorry for all the slashes. *) @@ -291,4 +296,5 @@ let options = record = !record; job_count = !job_count; suggest_jobs = !suggest_jobs; + junit = !junit; } diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli index 623456bd1df126c20553ec7f47b812b12c920d52..9d5d08cabcd97cf1f7eb5e669c9c5a94138992b7 100644 --- a/tezt/lib/cli.mli +++ b/tezt/lib/cli.mli @@ -85,6 +85,7 @@ type options = { record : string option; job_count : int; suggest_jobs : string option; + junit : string option; } (** Values for command-line options. *) diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml index c6044268e1ff2926e0697c65ad92a22307befd57..7ecf61f2e51b33e825ac60ebb0c4544ff6b5462f 100644 --- a/tezt/lib/log.ml +++ b/tezt/lib/log.ml @@ -54,7 +54,8 @@ let channel = starting_port = _; record = _; job_count = _; - suggest_jobs } = + suggest_jobs; + junit = _ } = Cli.options in if @@ -267,14 +268,14 @@ let warn x = log ~level:Warn ~color:Color.FG.red ~prefix:"warn" x let error x = log ~level:Error ~color:Color.FG.red ~prefix:"error" x -type test_result = Successful | Failed | Aborted +type test_result = Successful | Failed of string | Aborted let test_result ~progress_state ~iteration test_result test_name = let (prefix, prefix_color) = match test_result with | Successful -> ("SUCCESS", Color.(FG.green ++ bold)) - | Failed -> + | Failed _ -> ("FAILURE", Color.(FG.red ++ bold)) | Aborted -> ("ABORTED", Color.(FG.red ++ bold)) diff --git a/tezt/lib/log.mli b/tezt/lib/log.mli index 9b2fa536eebc62db64f3b098cd0967c50624b874..f910ebcb44e5a210d8f4dc3e1d16603652a7bab8 100644 --- a/tezt/lib/log.mli +++ b/tezt/lib/log.mli @@ -121,7 +121,12 @@ val warn : ('a, unit, string, unit) format4 -> 'a (** Same as [log ~level:Error ~color:red ~prefix:"error"]. *) val error : ('a, unit, string, unit) format4 -> 'a -type test_result = Successful | Failed | Aborted +(** Whether a test succeeded, failed or was aborted by the user. + + [Failed] comes with the error message, which is a string version + of the exception that was raised (usually with [Test.fail]). + This message is unused by the [Log] module itself. *) +type test_result = Successful | Failed of string | Aborted (** Log the result of a test. diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml index da99676f6a799c45620be34e41fb0a88f9bb5b5d..a1d54c1a528b30469be01201b8d796b605da01af 100644 --- a/tezt/lib/test.ml +++ b/tezt/lib/test.ml @@ -73,23 +73,35 @@ let fail x = let global_starting_time = Unix.gettimeofday () -let a_test_failed = ref false +(* Field [id] is used to be able to iterate on tests in order of registration. + Field [time] contains the cumulated time taken by all successful runs of this test. + Field [result] contains the result of the last time the test was run. + If the test was not run, it contains [None]. *) +type test = { + id : int; + file : string; + title : string; + tags : string list; + body : unit -> unit Lwt.t; + mutable time : float; + mutable run_count : int; + mutable result : Log.test_result option; +} -let really_run ~progress_state ~iteration title f = - Log.info "Starting test: %s" title ; +let really_run ~progress_state ~iteration test = + Log.info "Starting test: %s" test.title ; List.iter (fun reset -> reset ()) !reset_functions ; Lwt_main.run @@ let (fail_promise, new_fail_awakener) = Lwt.task () in fail_awakener := Some new_fail_awakener ; let already_logged_exn = ref false in - let test_result = ref Log.Failed in (* Run the test until it succeeds, fails, or we receive SIGINT. *) let main_temporary_directory = Temp.start () in let* () = let run_test () = - let* () = f () in - test_result := Successful ; + let* () = test.body () in + test.result <- Some Successful ; unit in let handle_exception = function @@ -99,14 +111,16 @@ let really_run ~progress_state ~iteration title f = already_logged_exn := true ; unit | exn -> - Log.error "%s" (Printexc.to_string exn) ; + let message = Printexc.to_string exn in + test.result <- Some (Failed message) ; + Log.error "%s" message ; already_logged_exn := true ; unit in let handle_sigint () = let* () = sigint () in Log.debug "Received SIGINT." ; - test_result := Aborted ; + test.result <- Some Aborted ; unit in let global_timeout = @@ -153,7 +167,7 @@ let really_run ~progress_state ~iteration title f = | Delete -> Temp.clean_up () ; false | Delete_if_successful -> - if !test_result = Successful then (Temp.clean_up () ; false) + if test.result = Some Successful then (Temp.clean_up () ; false) else (Temp.stop () ; true) | Keep -> Temp.stop () ; true @@ -175,34 +189,42 @@ let really_run ~progress_state ~iteration title f = It is still possible that the error is actually unrelated, but we already printed an error for the user to debug so it's ok. *) unit - else ( + else (* This could happen if an async promise fails *after* the test was successful. In that case, the test is not that successful after all. *) - Log.error "%s" (Printexc.to_string exn) ; - test_result := Log.Failed ; - unit ) + let message = Printexc.to_string exn in + Log.error "%s" message ; + test.result <- Some (Log.Failed message) ; + unit in Lwt.catch wait_for_async handle_exception in - (* Update progress indicators *) + (* Update progress indicators. *) + let test_result = + match test.result with + | None -> + (* Should not happen: after the test ends we always set [result] to [Some]. + But if it does happen we assume that it failed and that we failed to + maintain this invariant. *) + Log.Failed "unknown error" + | Some result -> + result + in let has_failed = - match !test_result with Successful -> false | Failed | Aborted -> true + match test_result with Successful -> false | Failed _ | Aborted -> true in Progress.update ~has_failed progress_state ; (* Display test result. *) - Log.test_result ~progress_state ~iteration !test_result title ; - match !test_result with + Log.test_result ~progress_state ~iteration test_result test.title ; + match test_result with | Successful -> - unit - | Failed -> + return true + | Failed _ -> Log.report "Try again with: %s --verbose --test %s" Sys.argv.(0) - (Log.quote_shell title) ; - if Cli.options.keep_going then ( - a_test_failed := true ; - unit ) - else exit 1 + (Log.quote_shell test.title) ; + return false | Aborted -> exit 2 @@ -254,18 +276,6 @@ let check_existence kind known specified = (Log.warn "Unknown %s: %s" kind) (String_set.diff (String_set.of_list specified) !known) -(* Field [id] is used to be able to iterate on tests in order of registration. - Field [time] contains the cumulated time taken by all successful runs of this test. *) -type test = { - id : int; - file : string; - title : string; - tags : string list; - body : unit -> unit Lwt.t; - mutable time : float; - mutable run_count : int; -} - (* Tests added using [register] and that match command-line filters. *) let registered : test String_map.t ref = ref String_map.empty @@ -399,7 +409,7 @@ let record_results filename = (* Remove the closure ([body]). *) let marshaled_tests = map_registered_list - @@ fun {id = _; file; title; tags; body = _; time; run_count} -> + @@ fun {id = _; file; title; tags; body = _; time; run_count; result = _} -> {file; title; tags; time = time /. float (max 1 run_count)} in (* Write to file using Marshal. @@ -468,6 +478,77 @@ let suggest_jobs (tests : marshaled_test list) = because it means to run all tests. *) display_job ~negate:true (fst jobs.(job_count - 1), !all_other_tests) +let output_junit filename = + with_open_out filename + @@ fun ch -> + let echo x = + Printf.ksprintf (fun s -> output_string ch s ; output_char ch '\n') x + in + let (count, fail_count, skipped_count, total_time) = + fold_registered (0, 0, 0, 0.) + @@ fun (count, fail_count, skipped_count, total_time) test -> + ( count + 1, + (fail_count + match test.result with Some (Failed _) -> 1 | _ -> 0), + ( skipped_count + + match test.result with None | Some Aborted -> 1 | _ -> 0 ), + total_time +. test.time ) + in + echo {||} ; + echo + {||} + count + fail_count + skipped_count + total_time ; + echo + {| |} + count + fail_count + skipped_count + total_time ; + ( iter_registered + @@ fun test -> + match test.result with + | None | Some Aborted -> + (* Skipped test, do not output. *) + () + | Some (Successful | Failed _) -> + let replace_entities s = + let buffer = Buffer.create (String.length s * 2) in + for i = 0 to String.length s - 1 do + match s.[i] with + | '"' -> + Buffer.add_string buffer """ + | '&' -> + Buffer.add_string buffer "&" + | '\'' -> + Buffer.add_string buffer "'" + | '<' -> + Buffer.add_string buffer "<" + | '>' -> + Buffer.add_string buffer ">" + | c -> + Buffer.add_char buffer c + done ; + Buffer.contents buffer + in + let title = replace_entities test.title in + echo + {| |} + title + (replace_entities test.file) + title + test.time ; + ( match test.result with + | None | Some Successful | Some Aborted -> + () + | Some (Failed message) -> + echo + {| %s|} + (replace_entities message) ) ; + echo " " ) ; + echo " " ; echo "" ; () + let next_id = ref 0 let register ~__FILE__ ~title ~tags body = @@ -493,7 +574,9 @@ let register ~__FILE__ ~title ~tags body = let id = !next_id in incr next_id ; if test_should_be_run ~file ~title ~tags then - let test = {id; file; title; tags; body; time = 0.; run_count = 0} in + let test = + {id; file; title; tags; body; time = 0.; run_count = 0; result = None} + in registered := String_map.add title test !registered let run () = @@ -536,6 +619,8 @@ let run () = prerr_endline "Cannot use both --list and --suggest-jobs at the same time." | (None, None) -> + let exception Stop in + let a_test_failed = ref false in let rec run iteration = match Cli.options.loop_mode with | Count n when n < iteration -> @@ -546,15 +631,19 @@ let run () = in let run_and_measure_time (test : test) = let start = Unix.gettimeofday () in - really_run ~progress_state ~iteration test.title test.body ; + let success = really_run ~progress_state ~iteration test in let time = Unix.gettimeofday () -. start in test.run_count <- test.run_count + 1 ; - test.time <- test.time +. time + test.time <- test.time +. time ; + if not success then ( + a_test_failed := true ; + if not Cli.options.keep_going then raise Stop ) in iter_registered run_and_measure_time ; run (iteration + 1) in - run 1 ; + (try run 1 with Stop -> ()) ; + Option.iter output_junit Cli.options.junit ; Option.iter record_results Cli.options.record ; - if !a_test_failed then exit 1 ; - if Cli.options.time then display_time_summary () + if Cli.options.time then display_time_summary () ; + if !a_test_failed then exit 1