From 622179d44f021321d3326d8eea61589f3bb18c23 Mon Sep 17 00:00:00 2001 From: "adam.khayam" Date: Mon, 8 Sep 2025 11:47:05 +0200 Subject: [PATCH] modifying type of analysis output --- src/api/api_handlers_db.ml | 33 +++- src/api/api_handlers_node.ml | 19 +- src/ethereum_analysis/eth_analyse.ml | 85 +++++---- src/ethereum_analysis/eth_analyse.mli | 2 + src/ethereum_analysis/eth_graph.ml | 163 +++++++++--------- .../eth_main_real_time_store_analyses.ml | 11 +- .../eth_main_real_time_store_transfers.ml | 48 +++--- ...compare_sandwich_analyses_to_scrap_data.ml | 4 +- test/test_common/test_common.ml | 66 ++++--- 9 files changed, 256 insertions(+), 175 deletions(-) diff --git a/src/api/api_handlers_db.ml b/src/api/api_handlers_db.ml index f6ffd684..3be81c12 100644 --- a/src/api/api_handlers_db.ml +++ b/src/api/api_handlers_db.ml @@ -435,7 +435,15 @@ let get_block_tags_db_eth (_, block_number) _ = let> analyses = Db_psql_eth.Eth_db_psql_analysis.get_block_analyses ~block_number () in let> analyses_decode = Lwt_list.map_s Decode_eth.decode_analysis analyses in - EzAPIServer.return_ok (List.flatten live_analyses @ analyses_decode) + let live_analyses = + List.fold_left + (fun acc analysis -> + match analysis with + | None -> acc + | Some res -> acc @ [res]) + [] + @@ List.flatten live_analyses in + EzAPIServer.return_ok (live_analyses @ analyses_decode) [@@service Api_services_db.get_block_tags_db_eth] let get_transaction_tags_db_eth (_, hash) _ = @@ -476,14 +484,20 @@ let get_transaction_tags_db_eth (_, hash) _ = (`tx_traces (block_traces_decode, Int64.to_int block_number)) analysis_type in Lwt.return + @@ List.fold_left + (fun acc res -> + match res with + | None -> acc + | Some res' -> acc @ [res']) + [] @@ List.filter (fun result -> match result with - | A_Arbitrage _ -> true - | A_Transfer _ -> true - | A_FlashLoan _ | A_Liquidate _ -> + | Some (A_Arbitrage _) -> true + | Some (A_Transfer _) -> true + | Some (A_FlashLoan _) | Some (A_Liquidate _) -> Log.log_error_fail ~here:[%here] "Bottom" - | A_Sandwich sandwich -> + | Some (A_Sandwich sandwich) -> String.equal (sandwich.bao_resume.sdo_back_run.sto_transaction_hash :> string) @@ -498,7 +512,8 @@ let get_transaction_tags_db_eth (_, hash) _ = String.equal (x.sto_transaction_hash :> string) tx_hash) - sandwich.bao_resume.sdo_victims) + sandwich.bao_resume.sdo_victims + | None -> false) analysis_result | (`flashloan | `liquidate) as analysis_type -> ( let> tx_receipts_opt = @@ -506,13 +521,15 @@ let get_transaction_tags_db_eth (_, hash) _ = ~tx_hash () in match tx_receipts_opt with | None -> Log.log_error_lwt_fail ~here:[%here] "receipt not found" - | Some receipt -> + | Some receipt -> ( let> receipts_decode = Decode_eth.decode_transaction_receipt receipt in let> analysis = Eth_analysis.Eth_analyse.tx_detect (`tx_receipt receipts_decode) analysis_type in - Lwt.return [analysis]) + match analysis with + | None -> Lwt.return [] + | Some analysis -> Lwt.return [analysis])) | `bottom (_ : bottom) -> .) not_stored_analyses in let> analyses = diff --git a/src/api/api_handlers_node.ml b/src/api/api_handlers_node.ml index 27f9e10a..a2b5692d 100644 --- a/src/api/api_handlers_node.ml +++ b/src/api/api_handlers_node.ml @@ -384,6 +384,13 @@ let get_block_tags_node_eth (_, block_number) _ = (Int64.to_string block_number) @@ fun () -> let> analyses = get_block_analyses_node_aux (Int64.to_int block_number) in + let> analyses = + Lwt_list.fold_left_s + (fun acc analysis -> + match analysis with + | None -> Lwt.return acc + | Some analysis' -> Lwt.return @@ acc @ [analysis']) + [] analyses in EzAPIServer.return_ok analyses [@@service Api_services_node.get_block_tags_node_eth] @@ -443,7 +450,9 @@ let get_transaction_tags_node_eth (_, hash) _ = let> flashloans = Eth_analysis.Eth_analyse.tx_detect (`tx_receipt receipt_decode) `flashloan in - EzAPIServer.return_ok @@ (flashloans :: liquidates :: block_analysis)) + + EzAPIServer.return_ok @@ Option.to_list flashloans + @ Option.to_list liquidates @ block_analysis) [@@service Api_services_node.get_transaction_tags_node_eth] let get_analyses_week_node_eth _ _ = @@ -615,7 +624,13 @@ let update_analyses_cache_node_eth_aux () = (fun block -> let> analyses = get_block_analyses_node_aux block.number in Lwt.return - { la_block_number = block.number; la_analyses = analyses }) + { + la_block_number = block.number; + la_analyses = + List.fold_left + (fun acc opt -> acc @ Option.to_list opt) + [] analyses; + }) !cache_eth_node in let analyses_block = List.filter (fun x -> x.la_analyses <> []) all_analyses_block_result diff --git a/src/ethereum_analysis/eth_analyse.ml b/src/ethereum_analysis/eth_analyse.ml index 3b345c54..64f8bfe5 100644 --- a/src/ethereum_analysis/eth_analyse.ml +++ b/src/ethereum_analysis/eth_analyse.ml @@ -103,7 +103,7 @@ let tx_detect tx tag = match tx with | `tx_receipt tx -> ( match tag with - | (`liquidate | `flashloan) as tag -> + | (`liquidate | `flashloan) as tag -> ( let> lil = abis_detect tag tx.trd_events_info in let txai = { @@ -115,41 +115,49 @@ let tx_detect tx tag = txai_analysis_info_logs = lil; } in let> res = tx_build_analysis_resume tag txai get_event_information in - Lwt.return (tx_return_analysis res tag) + match res.txao_output with + | [] -> Lwt.return None + | _ -> Lwt.return (Some (tx_return_analysis res tag))) | _ -> Log.log_error_lwt_fail ~here:[%here] "The analysis in now a transaction receipt based one") | `tx_trace_with_bb_cost (trace, costs_input) -> ( match tag with - | `arbitrage -> + | `arbitrage -> ( let analysis = Eth_graph.arbitrage_cft trace costs_input in - let resume = - { txaio_log_index = Int64.minus_one; txaio_resume = analysis } in - let output = - { - txao_block_number = costs_input.cfti_block_number; - txao_transaction_hash = trace.dt_tx_hash; - txao_transaction_index = trace.dt_tx_index; - txao_output = [resume]; - } in - Lwt.return (A_Arbitrage output) - | `transfer -> + match analysis.cftao_arbitrage with + | None -> Lwt.return None + | _ -> + let resume = + { txaio_log_index = Int64.minus_one; txaio_resume = analysis } in + let output = + { + txao_block_number = costs_input.cfti_block_number; + txao_transaction_hash = trace.dt_tx_hash; + txao_transaction_index = trace.dt_tx_index; + txao_output = [resume]; + } in + Lwt.return (Some (A_Arbitrage output))) + | `transfer -> ( let block_input = Eth_graph.calculate_transaction_fees costs_input.cfti_block_number costs_input.cfti_block_builder costs_input.cfti_base_fee costs_input.cfti_effective_price costs_input.cfti_max_fee_per_gas costs_input.cfti_gas_used in let analysis = Eth_graph.extract_transfers_from_cft trace block_input in - let resume = - { txaio_log_index = Int64.minus_one; txaio_resume = analysis } in - let output = - { - txao_block_number = block_input.cbi_block_number; - txao_transaction_hash = trace.dt_tx_hash; - txao_transaction_index = trace.dt_tx_index; - txao_output = [resume]; - } in - Lwt.return (A_Transfer output) + match analysis with + | [] -> Lwt.return None + | _ -> + let resume = + { txaio_log_index = Int64.minus_one; txaio_resume = analysis } in + let output = + { + txao_block_number = block_input.cbi_block_number; + txao_transaction_hash = trace.dt_tx_hash; + txao_transaction_index = trace.dt_tx_index; + txao_output = [resume]; + } in + Lwt.return (Some (A_Transfer output))) | _ -> Log.log_error_lwt_fail ~here:[%here] "The analysis in now a trace based one" @@ -163,14 +171,8 @@ let blk_detect txs tag = Lwt_list.filter_s (fun x -> match x with - | A_FlashLoan res -> - Lwt.return @@ not (Int.equal 0 @@ List.length res.txao_output) - | A_Liquidate res -> - Lwt.return @@ not (Int.equal 0 @@ List.length res.txao_output) - | _ -> - Log.log_error_fail ~here:[%here] - "Error : The analysis you're trying to execute is not considered a \ - transaction analysis") + | None -> Lwt.return false + | _ -> Lwt.return true) res | `tx_traces (trace, bnum) -> ( match tag with @@ -178,7 +180,8 @@ let blk_detect txs tag = let> sandwich = Eth_analysis_sandwich.detect_sandwich trace in Lwt.return @@ List.map - (fun x -> A_Sandwich { bao_block_number = bnum; bao_resume = x }) + (fun x -> + Some (A_Sandwich { bao_block_number = bnum; bao_resume = x })) sandwich | _ -> Log.log_error_lwt_fail ~here:[%here] @@ -195,7 +198,13 @@ let blk_detect txs tag = let> res = tx_detect (`tx_trace_with_bb_cost x) tag in Lwt.return @@ acc @ [res]) [] input in - Lwt.return res + Lwt.return + @@ List.filter + (fun x -> + match x with + | None -> false + | _ -> true) + res | `arbitrage -> let input = List.fold_left2 (fun acc x y -> acc @ [(x, y)]) [] traces cft_input_list @@ -206,7 +215,13 @@ let blk_detect txs tag = let> res = tx_detect (`tx_trace_with_bb_cost x) tag in Lwt.return @@ acc @ [res]) [] input in - Lwt.return res + Lwt.return + @@ List.filter + (fun x -> + match x with + | None -> false + | _ -> true) + res | _ -> Log.log_error_lwt_fail ~here:[%here] "You are trying to perform an unknown block analysis") diff --git a/src/ethereum_analysis/eth_analyse.mli b/src/ethereum_analysis/eth_analyse.mli index ee59c889..4020d4cb 100644 --- a/src/ethereum_analysis/eth_analyse.mli +++ b/src/ethereum_analysis/eth_analyse.mli @@ -21,6 +21,7 @@ val tx_detect : Common.Types.Ethereum_decode.known_address_or_evm_value ) Olympus.Types.event_decode_info Common.Types.Ethereum_analysis.analysis + option Lwt.t (** [blk_detect] performs block-level analysis on Ethereum data. @@ -41,5 +42,6 @@ val blk_detect : Common.Types.Ethereum_decode.known_address_or_evm_value ) Olympus.Types.event_decode_info Common.Types.Ethereum_analysis.analysis + option list Lwt.t diff --git a/src/ethereum_analysis/eth_graph.ml b/src/ethereum_analysis/eth_graph.ml index 5d436614..c0ce7781 100644 --- a/src/ethereum_analysis/eth_graph.ml +++ b/src/ethereum_analysis/eth_graph.ml @@ -124,86 +124,91 @@ let extract_erc contract = | Some contract -> contract.ci_type)) let extract_transfer_from_inputs inputs contract transfer_type erc = - let inputs = - List.sort (fun arg1 arg2 -> compare arg1.pda_index arg2.pda_index) inputs - in - let src, dst, operator, amount = - match transfer_type with - | `transfer -> - ( Ca_address (Common.extract_address_from_param (List.nth inputs 0)), - Ca_address (Common.extract_address_from_param (List.nth inputs 1)), - None, - Common.extract_value_from_param (List.nth inputs 2) ) - | `deposit -> - ( contract, - Ca_address (Common.extract_address_from_param (List.nth inputs 0)), - None, - Common.extract_value_from_param (List.nth inputs 1) ) - | `withdrawal -> - ( Ca_address (Common.extract_address_from_param (List.nth inputs 0)), - contract, - None, - Common.extract_value_from_param (List.nth inputs 1) ) - | `transfer_single -> - ( Ca_address (Common.extract_address_from_param (List.nth inputs 1)), - Ca_address (Common.extract_address_from_param (List.nth inputs 2)), - Some - (Ca_address (Common.extract_address_from_param (List.nth inputs 0))), - Tv_complex - (V_erc1155 - (Erc1155_single - { - id = Common.extract_z_value_from_param (List.nth inputs 3); - units = Common.extract_z_value_from_param (List.nth inputs 4); - })) ) - | `transfer_batch -> - ( Ca_address (Common.extract_address_from_param (List.nth inputs 1)), - Ca_address (Common.extract_address_from_param (List.nth inputs 2)), - Some - (Ca_address (Common.extract_address_from_param (List.nth inputs 0))), - Tv_complex - (V_erc1155 - (Erc1155_batch - { - id = - Common.extract_z_array_value_from_param (List.nth inputs 3); - units = - Common.extract_z_array_value_from_param (List.nth inputs 4); - })) ) in - let erc = - match erc with - | Some _ -> erc - | None -> extract_erc contract in - match is_null_address dst with - | true -> - `inputs (src, contract, operator, amount, contract, `burn_transfer, erc) - | false -> ( - match is_null_address src with - | true -> - `inputs (contract, dst, operator, amount, contract, `mint_transfer, erc) - | false -> - `inputs - ( src, - dst, - operator, - amount, + try + let inputs = + List.sort (fun arg1 arg2 -> compare arg1.pda_index arg2.pda_index) inputs + in + let src, dst, operator, amount = + match transfer_type with + | `transfer -> + ( Ca_address (Common.extract_address_from_param (List.nth inputs 0)), + Ca_address (Common.extract_address_from_param (List.nth inputs 1)), + None, + Common.extract_value_from_param (List.nth inputs 2) ) + | `deposit -> + ( contract, + Ca_address (Common.extract_address_from_param (List.nth inputs 0)), + None, + Common.extract_value_from_param (List.nth inputs 1) ) + | `withdrawal -> + ( Ca_address (Common.extract_address_from_param (List.nth inputs 0)), contract, - (match transfer_type with - | `transfer -> ( - match erc with - | None -> ( - match is_erc_contract contract (ErcStandard ERC721) with - | true -> `transfer_nft - | false -> `transfer) - | Some erc -> ( - match erc = ErcStandard ERC721 with - | true -> `transfer_nft - | false -> `transfer)) - | `deposit -> `deposit - | `withdrawal -> `withdrawal - | `transfer_single -> `transfer_nft - | `transfer_batch -> `transfer_nft), - erc )) + None, + Common.extract_value_from_param (List.nth inputs 1) ) + | `transfer_single -> + ( Ca_address (Common.extract_address_from_param (List.nth inputs 1)), + Ca_address (Common.extract_address_from_param (List.nth inputs 2)), + Some + (Ca_address (Common.extract_address_from_param (List.nth inputs 0))), + Tv_complex + (V_erc1155 + (Erc1155_single + { + id = Common.extract_z_value_from_param (List.nth inputs 3); + units = + Common.extract_z_value_from_param (List.nth inputs 4); + })) ) + | `transfer_batch -> + ( Ca_address (Common.extract_address_from_param (List.nth inputs 1)), + Ca_address (Common.extract_address_from_param (List.nth inputs 2)), + Some + (Ca_address (Common.extract_address_from_param (List.nth inputs 0))), + Tv_complex + (V_erc1155 + (Erc1155_batch + { + id = + Common.extract_z_array_value_from_param + (List.nth inputs 3); + units = + Common.extract_z_array_value_from_param + (List.nth inputs 4); + })) ) in + let erc = + match erc with + | Some _ -> erc + | None -> extract_erc contract in + match is_null_address dst with + | true -> + `inputs (src, contract, operator, amount, contract, `burn_transfer, erc) + | false -> ( + match is_null_address src with + | true -> + `inputs (contract, dst, operator, amount, contract, `mint_transfer, erc) + | false -> + `inputs + ( src, + dst, + operator, + amount, + contract, + (match transfer_type with + | `transfer -> ( + match erc with + | None -> ( + match is_erc_contract contract (ErcStandard ERC721) with + | true -> `transfer_nft + | false -> `transfer) + | Some erc -> ( + match erc = ErcStandard ERC721 with + | true -> `transfer_nft + | false -> `transfer)) + | `deposit -> `deposit + | `withdrawal -> `withdrawal + | `transfer_single -> `transfer_nft + | `transfer_batch -> `transfer_nft), + erc )) + with _ -> `none let wrap_address (address : known_address_or_address) = Ca_address address diff --git a/src/ethereum_scripts/eth_main_real_time_store_analyses.ml b/src/ethereum_scripts/eth_main_real_time_store_analyses.ml index 905b77f6..bd75f220 100644 --- a/src/ethereum_scripts/eth_main_real_time_store_analyses.ml +++ b/src/ethereum_scripts/eth_main_real_time_store_analyses.ml @@ -58,8 +58,15 @@ let store_information_block block analysis dbh = let> _ = Lwt_list.iter_s (fun analysis -> - Eth_db_psql.Eth_db_psql_analysis.add_block_analysis ~dbh - ~block_number:block_number_64 ~analysis ()) + match + Option.map + (fun analysis -> + Eth_db_psql.Eth_db_psql_analysis.add_block_analysis ~dbh + ~block_number:block_number_64 ~analysis ()) + analysis + with + | None -> Lwt.return_unit + | Some res -> res) analyses in let|> _ = set_store_block_analysis_finish ~dbh ~block_number:block_number_64 diff --git a/src/ethereum_scripts/eth_main_real_time_store_transfers.ml b/src/ethereum_scripts/eth_main_real_time_store_transfers.ml index 4c99ba63..5360037e 100644 --- a/src/ethereum_scripts/eth_main_real_time_store_transfers.ml +++ b/src/ethereum_scripts/eth_main_real_time_store_transfers.ml @@ -67,26 +67,34 @@ let store_information_block block dbh = let> _ = Lwt_list.iter_s (fun analysis -> - match analysis with - | A_Transfer tr -> ( - let trs = - List.flatten - @@ List.map (fun out -> out.txaio_resume) tr.txao_output in - let> res = - Eth_indexer.Db_neo4j.insert_transfers trs - (tr.txao_transaction_hash :> string) - tr.txao_block_number tr.txao_transaction_index in - match res.errors with - | [] -> Lwt.return_unit - | errors -> - Log.log_error_lwt_fail ~here:[%here] - @@ Format.sprintf "Neo4j Errors : %s " - @@ EzEncoding.construct - (Json_encoding.list Neo4j.error_enc) - errors) - | _ -> - Log.log_error_lwt_fail ~here:[%here] - "Can't store other analyses in Neo4j") + match + Option.map + (fun analysis -> + match analysis with + | A_Transfer tr -> ( + let trs = + List.flatten + @@ List.map (fun out -> out.txaio_resume) tr.txao_output + in + let> res = + Eth_indexer.Db_neo4j.insert_transfers trs + (tr.txao_transaction_hash :> string) + tr.txao_block_number tr.txao_transaction_index in + match res.errors with + | [] -> Lwt.return_unit + | errors -> + Log.log_error_lwt_fail ~here:[%here] + @@ Format.sprintf "Neo4j Errors : %s " + @@ EzEncoding.construct + (Json_encoding.list Neo4j.error_enc) + errors) + | _ -> + Log.log_error_lwt_fail ~here:[%here] + "Can't store other analyses in Neo4j") + analysis + with + | None -> Lwt.return_unit + | Some res -> res) analyses in let|> _ = set_store_transfer_block_finish ~dbh ~block_number:block_number_64 () diff --git a/test/benchmark/compare_sandwich_analyses_to_scrap_data.ml b/test/benchmark/compare_sandwich_analyses_to_scrap_data.ml index 5dbffc99..620c43fc 100644 --- a/test/benchmark/compare_sandwich_analyses_to_scrap_data.ml +++ b/test/benchmark/compare_sandwich_analyses_to_scrap_data.ml @@ -109,11 +109,11 @@ let sandwich_output_to_scrap_type block_number { sj_block = block_number; sj_tx_roles = front_run :: back_run :: victims } let extract_sandwiches_from_analyses - (analyses : 'a Ethereum_analysis.analysis list) = + (analyses : 'a Ethereum_analysis.analysis option list) = List.fold_right (fun analysis acc -> match analysis with - | A_Sandwich s -> s.bao_resume :: acc + | Some (A_Sandwich s) -> s.bao_resume :: acc | _ -> acc) analyses [] diff --git a/test/test_common/test_common.ml b/test/test_common/test_common.ml index df9ff54d..7c0f4095 100644 --- a/test/test_common/test_common.ml +++ b/test/test_common/test_common.ml @@ -103,34 +103,46 @@ module Test_tools = struct let analyses_to_analyses_test (analysis : (known_address_or_address, known_address_or_evm_value) event_decode_info - Ethereum_analysis.analysis) = - match analysis with - | A_FlashLoan x -> - let tx_analyses_test = - { - txat_block_number = x.txao_block_number; - txat_transaction_hash = x.txao_transaction_hash; - } in - AT_Flashloan tx_analyses_test - | A_Liquidate x -> - let tx_analyses_test = - { - txat_block_number = x.txao_block_number; - txat_transaction_hash = x.txao_transaction_hash; - } in - AT_Liquidate tx_analyses_test - | A_Sandwich sandwich -> - AT_Sandwich - { - bat_block_number = sandwich.bao_block_number; - bat_resume = sandwich_output_to_sandwich_test sandwich.bao_resume; - } - | A_Transfer _ -> + Ethereum_analysis.analysis + option) = + match + Option.map + (fun analysis -> + match analysis with + | Common.Types.Ethereum_analysis.A_FlashLoan x -> + let tx_analyses_test = + { + txat_block_number = x.txao_block_number; + txat_transaction_hash = x.txao_transaction_hash; + } in + AT_Flashloan tx_analyses_test + | A_Liquidate x -> + let tx_analyses_test = + { + txat_block_number = x.txao_block_number; + txat_transaction_hash = x.txao_transaction_hash; + } in + AT_Liquidate tx_analyses_test + | A_Sandwich sandwich -> + AT_Sandwich + { + bat_block_number = sandwich.bao_block_number; + bat_resume = + sandwich_output_to_sandwich_test sandwich.bao_resume; + } + | A_Transfer _ -> + Log.log_warning_fail ~here:[%here] + "Transfer analysis has still no tests designed for it." + | A_Arbitrage _ -> + Log.log_warning_fail ~here:[%here] + "Arbitrage analysis has still no tests designed for it.") + analysis + with + | None -> Log.log_warning_fail ~here:[%here] - "Transfer analysis has still no tests designed for it." - | A_Arbitrage _ -> - Log.log_warning_fail ~here:[%here] - "Arbitrage analysis has still no tests designed for it." + "The analysis returned a None analysis. This result should have been \ + filtered out." + | Some res -> res let perform_analysis database analysis_type = match database with -- GitLab