diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_t5.h b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_t5.h
index 65e6cac2aba09..afb98d8365abd 100644
--- a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_t5.h
+++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_t5.h
@@ -290,13 +290,13 @@ Status BeamSearchT5<T>::Execute(const FeedsFetchesManager& encoder_feeds_fetches
       dumper->Print("decoder_feeds", i, true);
       dumper->Print("", decoder_feeds[i]);
     }
-    auto offset = decoder_subgraph_.GetFirstPastInputIndex() + 4 * decoder_subgraph_.num_layers;
-    dumper->Print("past_sequence_length", offset, true);
-    dumper->Print("", decoder_feeds[offset]);
-    dumper->Print("beam_width", offset + 1, true);
-    dumper->Print("", decoder_feeds[offset + 1]);
-    dumper->Print("cache_redir", offset + 2, true);
-    dumper->Print("", decoder_feeds[offset + 2]);
+    // auto offset = decoder_subgraph_.GetFirstPastInputIndex() + 4 * decoder_subgraph_.num_layers;
+    // dumper->Print("past_sequence_length", offset, true);
+    // dumper->Print("", decoder_feeds[offset]);
+    // dumper->Print("beam_width", offset + 1, true);
+    // dumper->Print("", decoder_feeds[offset + 1]);
+    // dumper->Print("past_sequence_length", offset + 2, true);
+    // dumper->Print("", decoder_feeds[offset + 2]);
 #endif
 
 #ifdef DEBUG_NODE_INPUTS_OUTPUTS
diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc
index ae87961dc8e8a..bd334753e3517 100644
--- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc
+++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc
@@ -41,10 +41,11 @@ GetCapability::GetCapability(const GraphViewer& graph_viewer_param, std::string
 std::vector<std::unique_ptr<ComputeCapability>> GetCapability::Execute() {
   std::vector<std::unique_ptr<ComputeCapability>> result;
 
+  if (graph_viewer_.Name() == "decoder subgraph" || graph_viewer_.Name() == "beam-search-test")
+    return result;
   // Check if it is a subgraph
-  if (graph_viewer_.IsSubgraph() && graph_viewer_.Name() == "tf2onnx") {
+  if (graph_viewer_.IsSubgraph() && graph_viewer_.Name() == "tf2onnx")
     return result;
-  }
 
   // This is a list of initializers that nGraph considers as constants. Example weights, reshape shape etc.
   std::unordered_set<std::string> ng_required_initializers;
diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc
index 30ba5a319830f..46c76ba4a05f6 100644
--- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc
+++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc
@@ -162,7 +162,7 @@ std::vector<SupportedOp> supported_op_mode = {
     {"InstanceNormalization", V_2023_0, {"VPUX"}},
     {"HardSigmoid", V_2020_4, {"CPU", "GPU"}},
     {"HardMax", V_2022_1, {"CPU", "GPU"}},
-    {"LayerNormalization", V_2023_0, {"CPU", "GPU"}},
+    {"LayerNormalization", V_2023_0, {"CPU", "GPU", "VPUX"}},
     {"LeakyRelu", V_2020_4, {"CPU", "GPU"}},
     {"LeakyRelu", V_2023_0, {"VPUX"}},
     {"Less", V_2020_4, {"CPU", "GPU"}},
diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc
index aa29a20c87fbb..e235b3accd9e5 100644
--- a/onnxruntime/core/session/inference_session.cc
+++ b/onnxruntime/core/session/inference_session.cc
@@ -1428,6 +1428,31 @@ common::Status InferenceSession::Initialize() {
 
     // Verify that there are no external initializers in the graph if external data is disabled.
     onnxruntime::Graph& graph = model_->MainGraph();
+
+#ifdef USE_OPENVINO
+    /// Move initializers from the MainGraph to subgraphs
+    std::unordered_set<std::string> initializer_names_to_preserve;
+    for (auto& node: graph.Nodes()) {
+
+      // Preserve implicitInputDefs in the subgraphs
+      for (auto& def : node.ImplicitInputDefs())
+        initializer_names_to_preserve.insert(def->Name());
+
+      for(auto& entry: node.GetAttributeNameToMutableSubgraphMap()) {
+        Graph *subgraph = entry.second;
+
+        for(const auto& parent_graph_initializer: graph.GetAllInitializedTensors()) {
+          if (initializer_names_to_preserve.find(parent_graph_initializer.first) != initializer_names_to_preserve.cend())
+            subgraph->AddInitializedTensor(*parent_graph_initializer.second);
+      }
+    }
+  }
+
+  // Now remove those initializers from the MainGraph
+  for(auto& name: initializer_names_to_preserve)
+    graph.RemoveInitializedTensor(name);
+#endif
+
 #ifdef DISABLE_EXTERNAL_INITIALIZERS
     const InitializedTensorSet& initializers = graph.GetAllInitializedTensors();
     for (const auto& it : initializers) {