From 6b73b505f4fbb1dd934baa3a3208d90d0855737a Mon Sep 17 00:00:00 2001
From: "iguerNL@Functori" <iguer@functori.com>
Date: Tue, 13 May 2025 15:40:52 +0200
Subject: [PATCH 1/2] DAL/Node: add --http-backup CLI option

---
 src/bin_dal_node/main.ml                |  6 +++++
 src/lib_dal_node/cli.ml                 | 30 +++++++++++++++++--------
 src/lib_dal_node/cli.mli                |  4 ++++
 src/lib_dal_node/configuration_file.ml  | 25 ++++++++++++++++-----
 src/lib_dal_node/configuration_file.mli |  2 ++
 5 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/src/bin_dal_node/main.ml b/src/bin_dal_node/main.ml
index ee8451916b28..52bbca0d05c3 100644
--- a/src/bin_dal_node/main.ml
+++ b/src/bin_dal_node/main.ml
@@ -34,6 +34,7 @@ let merge
         listen_addr;
         public_addr;
         endpoint;
+        http_backup_uris;
         metrics_addr;
         profile;
         peers;
@@ -65,6 +66,11 @@ let merge
     public_addr = Option.value ~default:configuration.public_addr public_addr;
     expected_pow = Option.value ~default:configuration.expected_pow expected_pow;
     endpoint = Option.value ~default:configuration.endpoint endpoint;
+    http_backup_uris =
+      (* backup URIs from the CLI, if any, are favored over the ones in the
+         config file. *)
+      (if List.is_empty http_backup_uris then configuration.http_backup_uris
+       else http_backup_uris);
     profile;
     (* metrics are disabled unless a metrics_addr option is specified *)
     metrics_addr;
diff --git a/src/lib_dal_node/cli.ml b/src/lib_dal_node/cli.ml
index d11ee70dc9ea..cd733a214de1 100644
--- a/src/lib_dal_node/cli.ml
+++ b/src/lib_dal_node/cli.ml
@@ -120,7 +120,7 @@ module Term = struct
       & opt (some (p2p_point_arg ~default_port)) None
       & info ~docs ~doc ~docv:"ADDR[:PORT]" ["public-addr"])
 
-  let endpoint_arg =
+  let uri_arg =
     let open Cmdliner in
     let decoder string =
       try Uri.of_string string |> Result.ok
@@ -137,9 +137,18 @@ module Term = struct
     in
     Arg.(
       value
-      & opt (some endpoint_arg) None
+      & opt (some uri_arg) None
       & info ~docs ~doc ~docv:"URI" ["endpoint"; "E"])
 
+  let http_backup_uris =
+    let open Cmdliner in
+    let doc =
+      "List of HTTP base URIs to fetch missing DAL slots if they are \
+       unavailable locally or cannot be reconstructed from shards. This option \
+       can be specified multiple times to provide fallback sources."
+    in
+    Arg.(value & opt_all uri_arg [] & info ~doc ~docv:"URI" ["http-backup"])
+
   let ignore_l1_config_peers =
     let open Cmdliner in
     let doc = "Ignore the boot(strap) peers provided by L1 config." in
@@ -367,10 +376,11 @@ module Term = struct
     Cmdliner.Term.(
       ret
         (const process $ data_dir $ rpc_addr $ expected_pow $ net_addr
-       $ public_addr $ endpoint $ metrics_addr $ attester_profile
-       $ operator_profile $ observer_profile $ bootstrap_profile $ peers
-       $ history_mode $ service_name $ service_namespace $ fetch_trusted_setup
-       $ verbose $ ignore_l1_config_peers))
+       $ public_addr $ endpoint $ http_backup_uris $ metrics_addr
+       $ attester_profile $ operator_profile $ observer_profile
+       $ bootstrap_profile $ peers $ history_mode $ service_name
+       $ service_namespace $ fetch_trusted_setup $ verbose
+       $ ignore_l1_config_peers))
 end
 
 type t = Run | Config_init | Config_update | Debug_print_store_schemas
@@ -520,6 +530,7 @@ type options = {
   listen_addr : P2p_point.Id.t option;
   public_addr : P2p_point.Id.t option;
   endpoint : Uri.t option;
+  http_backup_uris : Uri.t list;
   profile : Profile_manager.unresolved_profile option;
   metrics_addr : P2p_point.Id.t option;
   peers : string list;
@@ -534,9 +545,9 @@ type options = {
 
 let make ~run =
   let run subcommand data_dir rpc_addr expected_pow listen_addr public_addr
-      endpoint metrics_addr attesters operators observers bootstrap_flag peers
-      history_mode service_name service_namespace fetch_trusted_setup verbose
-      ignore_l1_config_peers =
+      endpoint http_backup_uris metrics_addr attesters operators observers
+      bootstrap_flag peers history_mode service_name service_namespace
+      fetch_trusted_setup verbose ignore_l1_config_peers =
     let run profile =
       run
         subcommand
@@ -547,6 +558,7 @@ let make ~run =
           listen_addr;
           public_addr;
           endpoint;
+          http_backup_uris;
           profile;
           metrics_addr;
           peers;
diff --git a/src/lib_dal_node/cli.mli b/src/lib_dal_node/cli.mli
index df0f8cba3b4a..d27bda4e1b8f 100644
--- a/src/lib_dal_node/cli.mli
+++ b/src/lib_dal_node/cli.mli
@@ -46,6 +46,10 @@ type options = {
   public_addr : P2p_point.Id.t option;
       (** The endpoint on which the DAL node can be contacted by other DAL nodes. *)
   endpoint : Uri.t option;  (** The endpoint on which to contact the L1 node. *)
+  http_backup_uris : Uri.t list;
+      (** (Optional) URIs to use as HTTP backup sources for slot data retrieval,
+          in case the slot is missing locally and reconstruction from shards is
+          not possible. *)
   profile : Profile_manager.unresolved_profile option;
       (** Profiles of the DAL node used for tracking shards. *)
   metrics_addr : P2p_point.Id.t option;  (** Metrics server endpoint. *)
diff --git a/src/lib_dal_node/configuration_file.ml b/src/lib_dal_node/configuration_file.ml
index eef10665c574..ea75a60a2623 100644
--- a/src/lib_dal_node/configuration_file.ml
+++ b/src/lib_dal_node/configuration_file.ml
@@ -72,6 +72,7 @@ type t = {
   peers : string list;
   expected_pow : float;
   endpoint : Uri.t;
+  http_backup_uris : Uri.t list;
   metrics_addr : P2p_point.Id.t option;
   profile : Profile_manager.unresolved_profile;
   history_mode : history_mode;
@@ -132,6 +133,7 @@ let default =
     peers = default_peers;
     expected_pow = default_expected_pow;
     endpoint = default_endpoint;
+    http_backup_uris = [];
     metrics_addr = None;
     history_mode = default_history_mode;
     profile = Profile_manager.Empty;
@@ -151,7 +153,7 @@ let neighbor_encoding : neighbor Data_encoding.t =
     (fun (addr, port) -> {addr; port})
     (obj2 (req "rpc-addr" string) (req "rpc-port" uint16))
 
-let endpoint_encoding : Uri.t Data_encoding.t =
+let uri_encoding : Uri.t Data_encoding.t =
   let open Data_encoding in
   conv_with_guard
     (fun uri -> Uri.to_string uri)
@@ -159,7 +161,7 @@ let endpoint_encoding : Uri.t Data_encoding.t =
       try Uri.of_string str |> Result.ok
       with exn ->
         Format.asprintf
-          "endpoint decoding failed:@.%a@."
+          "uri decoding failed:@.%a@."
           Error_monad.pp_print_trace
           [Exn exn]
         |> Result.error)
@@ -179,6 +181,7 @@ let encoding : t Data_encoding.t =
            peers;
            expected_pow;
            endpoint;
+           http_backup_uris;
            metrics_addr;
            history_mode;
            profile;
@@ -197,6 +200,7 @@ let encoding : t Data_encoding.t =
           peers,
           expected_pow,
           endpoint,
+          http_backup_uris,
           metrics_addr ),
         ( history_mode,
           profile,
@@ -214,6 +218,7 @@ let encoding : t Data_encoding.t =
              peers,
              expected_pow,
              endpoint,
+             http_backup_uris,
              metrics_addr ),
            ( history_mode,
              profile,
@@ -232,6 +237,7 @@ let encoding : t Data_encoding.t =
         peers;
         expected_pow;
         endpoint;
+        http_backup_uris;
         metrics_addr;
         history_mode;
         profile;
@@ -244,7 +250,7 @@ let encoding : t Data_encoding.t =
         ignore_l1_config_peers;
       })
     (merge_objs
-       (obj8
+       (obj9
           (dft
              "data-dir"
              ~description:"Location of the data dir"
@@ -278,8 +284,13 @@ let encoding : t Data_encoding.t =
           (dft
              "endpoint"
              ~description:"The Tezos node endpoint"
-             endpoint_encoding
+             uri_encoding
              default_endpoint)
+          (dft
+             "http_backup_uris"
+             ~description:"Optional HTTP endpoints to fetch missing slots from."
+             (list uri_encoding)
+             [])
           (dft
              "metrics-addr"
              ~description:"The point for the DAL node metrics server"
@@ -412,7 +423,7 @@ module V0 = struct
          (dft "peers" (list string) default_peers)
          (dft "expected-pow" float default_expected_pow)
          (dft "network-name" string legacy_network_name)
-         (dft "endpoint" endpoint_encoding default_endpoint)
+         (dft "endpoint" uri_encoding default_endpoint)
          (dft "metrics-addr" P2p_point.Id.encoding default_metrics_addr))
       (obj2
          (dft "history_mode" history_mode_encoding default_history_mode)
@@ -451,6 +462,7 @@ module V0 = struct
       fetch_trusted_setup = true;
       verbose = false;
       ignore_l1_config_peers = false;
+      http_backup_uris = [];
     }
 end
 
@@ -507,7 +519,7 @@ module V1 = struct
          (dft "peers" (list string) default_peers)
          (dft "expected-pow" float default_expected_pow)
          (dft "network-name" string legacy_network_name)
-         (dft "endpoint" endpoint_encoding default_endpoint)
+         (dft "endpoint" uri_encoding default_endpoint)
          (dft "metrics-addr" (Encoding.option P2p_point.Id.encoding) None))
       (obj8
          (dft "history_mode" history_mode_encoding default_history_mode)
@@ -565,6 +577,7 @@ module V1 = struct
       fetch_trusted_setup;
       verbose;
       ignore_l1_config_peers = false;
+      http_backup_uris = [];
     }
 end
 
diff --git a/src/lib_dal_node/configuration_file.mli b/src/lib_dal_node/configuration_file.mli
index 14dd1330c254..f7ba9fbf286d 100644
--- a/src/lib_dal_node/configuration_file.mli
+++ b/src/lib_dal_node/configuration_file.mli
@@ -52,6 +52,8 @@ type t = {
   expected_pow : float;
       (** The expected PoW difficulty level for the peers' identity. *)
   endpoint : Uri.t;  (** The endpoint of a Tezos L1 node. *)
+  http_backup_uris : Uri.t list;
+      (** Backup URIs to fetch slot data if missing and unrecoverable from shards. *)
   metrics_addr : P2p_point.Id.t option;
       (** The TCP address of the node's server used to export metrics. *)
   profile : Profile_manager.unresolved_profile;
-- 
GitLab


From 3be5cd4d04a5ecfc2c6e9caee3808b7e78669a7b Mon Sep 17 00:00:00 2001
From: "iguerNL@Functori" <iguer@functori.com>
Date: Mon, 19 May 2025 11:54:37 +0200
Subject: [PATCH 2/2] DAL/Node: add --trust-http-backup-uris CLI option

---
 src/bin_dal_node/main.ml                | 16 +++++++++++-----
 src/lib_dal_node/cli.ml                 | 23 ++++++++++++++++++-----
 src/lib_dal_node/cli.mli                |  2 ++
 src/lib_dal_node/configuration_file.ml  | 17 ++++++++++++++++-
 src/lib_dal_node/configuration_file.mli |  2 ++
 5 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/src/bin_dal_node/main.ml b/src/bin_dal_node/main.ml
index 52bbca0d05c3..711ea3509d7c 100644
--- a/src/bin_dal_node/main.ml
+++ b/src/bin_dal_node/main.ml
@@ -35,6 +35,7 @@ let merge
         public_addr;
         endpoint;
         http_backup_uris;
+        trust_http_backup_uris;
         metrics_addr;
         profile;
         peers;
@@ -58,6 +59,14 @@ let merge
           ~lower_prio:configuration.profile
           ~higher_prio:from_cli
   in
+  let http_backup_uris, trust_http_backup_uris =
+    (* backup URIs from the CLI, if any, are favored over the ones in the
+       config file. *)
+    if List.is_empty http_backup_uris then
+      (configuration.http_backup_uris, configuration.trust_http_backup_uris)
+    else (http_backup_uris, trust_http_backup_uris)
+  in
+
   {
     configuration with
     data_dir = Option.value ~default:configuration.data_dir data_dir;
@@ -66,11 +75,8 @@ let merge
     public_addr = Option.value ~default:configuration.public_addr public_addr;
     expected_pow = Option.value ~default:configuration.expected_pow expected_pow;
     endpoint = Option.value ~default:configuration.endpoint endpoint;
-    http_backup_uris =
-      (* backup URIs from the CLI, if any, are favored over the ones in the
-         config file. *)
-      (if List.is_empty http_backup_uris then configuration.http_backup_uris
-       else http_backup_uris);
+    http_backup_uris;
+    trust_http_backup_uris;
     profile;
     (* metrics are disabled unless a metrics_addr option is specified *)
     metrics_addr;
diff --git a/src/lib_dal_node/cli.ml b/src/lib_dal_node/cli.ml
index cd733a214de1..78dfa34d8b8d 100644
--- a/src/lib_dal_node/cli.ml
+++ b/src/lib_dal_node/cli.ml
@@ -149,6 +149,17 @@ module Term = struct
     in
     Arg.(value & opt_all uri_arg [] & info ~doc ~docv:"URI" ["http-backup"])
 
+  let trust_http_backup_uris =
+    let open Cmdliner in
+    let doc =
+      "If set, skip cryptographic verification of slots downloaded from HTTP \
+       backup URIs. Default is false. This can speed up slot retrieval when \
+       replaying history or for debugging purposes, but should be used with \
+       caution for normal operation or in the context of refutation games \
+       (unless the HTTP source is fully trusted)."
+    in
+    Arg.(value & flag & info ~doc ["trust-http-backup-uris"])
+
   let ignore_l1_config_peers =
     let open Cmdliner in
     let doc = "Ignore the boot(strap) peers provided by L1 config." in
@@ -376,8 +387,8 @@ module Term = struct
     Cmdliner.Term.(
       ret
         (const process $ data_dir $ rpc_addr $ expected_pow $ net_addr
-       $ public_addr $ endpoint $ http_backup_uris $ metrics_addr
-       $ attester_profile $ operator_profile $ observer_profile
+       $ public_addr $ endpoint $ http_backup_uris $ trust_http_backup_uris
+       $ metrics_addr $ attester_profile $ operator_profile $ observer_profile
        $ bootstrap_profile $ peers $ history_mode $ service_name
        $ service_namespace $ fetch_trusted_setup $ verbose
        $ ignore_l1_config_peers))
@@ -531,6 +542,7 @@ type options = {
   public_addr : P2p_point.Id.t option;
   endpoint : Uri.t option;
   http_backup_uris : Uri.t list;
+  trust_http_backup_uris : bool;
   profile : Profile_manager.unresolved_profile option;
   metrics_addr : P2p_point.Id.t option;
   peers : string list;
@@ -545,9 +557,9 @@ type options = {
 
 let make ~run =
   let run subcommand data_dir rpc_addr expected_pow listen_addr public_addr
-      endpoint http_backup_uris metrics_addr attesters operators observers
-      bootstrap_flag peers history_mode service_name service_namespace
-      fetch_trusted_setup verbose ignore_l1_config_peers =
+      endpoint http_backup_uris trust_http_backup_uris metrics_addr attesters
+      operators observers bootstrap_flag peers history_mode service_name
+      service_namespace fetch_trusted_setup verbose ignore_l1_config_peers =
     let run profile =
       run
         subcommand
@@ -559,6 +571,7 @@ let make ~run =
           public_addr;
           endpoint;
           http_backup_uris;
+          trust_http_backup_uris;
           profile;
           metrics_addr;
           peers;
diff --git a/src/lib_dal_node/cli.mli b/src/lib_dal_node/cli.mli
index d27bda4e1b8f..2cda1c475864 100644
--- a/src/lib_dal_node/cli.mli
+++ b/src/lib_dal_node/cli.mli
@@ -50,6 +50,8 @@ type options = {
       (** (Optional) URIs to use as HTTP backup sources for slot data retrieval,
           in case the slot is missing locally and reconstruction from shards is
           not possible. *)
+  trust_http_backup_uris : bool;
+      (** Whether to trust the data downlaoded from the provided HTTP backup URIs. *)
   profile : Profile_manager.unresolved_profile option;
       (** Profiles of the DAL node used for tracking shards. *)
   metrics_addr : P2p_point.Id.t option;  (** Metrics server endpoint. *)
diff --git a/src/lib_dal_node/configuration_file.ml b/src/lib_dal_node/configuration_file.ml
index ea75a60a2623..708dcc285ae1 100644
--- a/src/lib_dal_node/configuration_file.ml
+++ b/src/lib_dal_node/configuration_file.ml
@@ -73,6 +73,7 @@ type t = {
   expected_pow : float;
   endpoint : Uri.t;
   http_backup_uris : Uri.t list;
+  trust_http_backup_uris : bool;
   metrics_addr : P2p_point.Id.t option;
   profile : Profile_manager.unresolved_profile;
   history_mode : history_mode;
@@ -134,6 +135,7 @@ let default =
     expected_pow = default_expected_pow;
     endpoint = default_endpoint;
     http_backup_uris = [];
+    trust_http_backup_uris = false;
     metrics_addr = None;
     history_mode = default_history_mode;
     profile = Profile_manager.Empty;
@@ -182,6 +184,7 @@ let encoding : t Data_encoding.t =
            expected_pow;
            endpoint;
            http_backup_uris;
+           trust_http_backup_uris;
            metrics_addr;
            history_mode;
            profile;
@@ -201,6 +204,7 @@ let encoding : t Data_encoding.t =
           expected_pow,
           endpoint,
           http_backup_uris,
+          trust_http_backup_uris,
           metrics_addr ),
         ( history_mode,
           profile,
@@ -219,6 +223,7 @@ let encoding : t Data_encoding.t =
              expected_pow,
              endpoint,
              http_backup_uris,
+             trust_http_backup_uris,
              metrics_addr ),
            ( history_mode,
              profile,
@@ -238,6 +243,7 @@ let encoding : t Data_encoding.t =
         expected_pow;
         endpoint;
         http_backup_uris;
+        trust_http_backup_uris;
         metrics_addr;
         history_mode;
         profile;
@@ -250,7 +256,7 @@ let encoding : t Data_encoding.t =
         ignore_l1_config_peers;
       })
     (merge_objs
-       (obj9
+       (obj10
           (dft
              "data-dir"
              ~description:"Location of the data dir"
@@ -291,6 +297,13 @@ let encoding : t Data_encoding.t =
              ~description:"Optional HTTP endpoints to fetch missing slots from."
              (list uri_encoding)
              [])
+          (dft
+             "trust_http_backup_uris"
+             ~description:
+               "Whether to trust the data downlaoded from the provided HTTP \
+                backup URIs."
+             bool
+             false)
           (dft
              "metrics-addr"
              ~description:"The point for the DAL node metrics server"
@@ -463,6 +476,7 @@ module V0 = struct
       verbose = false;
       ignore_l1_config_peers = false;
       http_backup_uris = [];
+      trust_http_backup_uris = false;
     }
 end
 
@@ -578,6 +592,7 @@ module V1 = struct
       verbose;
       ignore_l1_config_peers = false;
       http_backup_uris = [];
+      trust_http_backup_uris = false;
     }
 end
 
diff --git a/src/lib_dal_node/configuration_file.mli b/src/lib_dal_node/configuration_file.mli
index f7ba9fbf286d..2e2ea8194365 100644
--- a/src/lib_dal_node/configuration_file.mli
+++ b/src/lib_dal_node/configuration_file.mli
@@ -54,6 +54,8 @@ type t = {
   endpoint : Uri.t;  (** The endpoint of a Tezos L1 node. *)
   http_backup_uris : Uri.t list;
       (** Backup URIs to fetch slot data if missing and unrecoverable from shards. *)
+  trust_http_backup_uris : bool;
+      (** Whether to trust the data downlaoded from the provided HTTP backup URIs. *)
   metrics_addr : P2p_point.Id.t option;
       (** The TCP address of the node's server used to export metrics. *)
   profile : Profile_manager.unresolved_profile;
-- 
GitLab