diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6de999af --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.history/ diff --git a/src/Paillave.Etl.Http/HttpConnectionInfoEx.cs b/src/Paillave.Etl.Http/HttpConnectionInfoEx.cs index 8fbfaed1..e804fb08 100644 --- a/src/Paillave.Etl.Http/HttpConnectionInfoEx.cs +++ b/src/Paillave.Etl.Http/HttpConnectionInfoEx.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Net.Http; +using Fluid; +using Paillave.Etl.Core; namespace Paillave.Etl.Http; @@ -8,11 +10,13 @@ public static class IHttpConnectionInfoEx { public static HttpClient CreateHttpClient( IHttpConnectionInfo httpConnectionInfo, - IHttpAdapterParameters adapterParameters + IHttpAdapterParameters adapterParameters, + IFileValueMetadata? additionalMetadata=null ) { HttpClient client = new HttpClient(); client = ManageHeaders(client, httpConnectionInfo.HttpHeaders); + client = ManageAdditionalParameters(client, httpConnectionInfo, adapterParameters); client = ManageAuthentication(client, httpConnectionInfo, adapterParameters); return client; } @@ -28,17 +32,46 @@ private static HttpClient ManageHeaders(HttpClient client, Dictionary private readonly IHttpConnectionInfo _connectionInfo; private readonly IHttpAdapterParameters _parameters; + private readonly IFileValueMetadata? _additionalMetadata = null; + public HttpFileValue( string name, string url, @@ -18,7 +21,8 @@ public HttpFileValue( string connectionName, string connectorName, IHttpConnectionInfo? connectionInfo = null, - IHttpAdapterParameters? parameters = null + IHttpAdapterParameters? parameters = null, + IFileValueMetadata? additionalMetadata = null ) : base( new HttpFileValueMetadata @@ -42,7 +46,7 @@ public override Stream GetContent() => private Stream GetContentSingleTime() { - var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters); + var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters, _additionalMetadata); var response = HttpHelpers.GetResponse(_connectionInfo, _parameters, httpClient).Result; @@ -60,7 +64,7 @@ private Stream GetContentSingleTime() protected override void DeleteFile() { - var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters); + var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters, _additionalMetadata); var parameters = new HttpAdapterProviderParameters(_parameters) { diff --git a/src/Paillave.Etl.Http/HttpFileValueProcessor.cs b/src/Paillave.Etl.Http/HttpFileValueProcessor.cs index c1c97522..79c2fa85 100644 --- a/src/Paillave.Etl.Http/HttpFileValueProcessor.cs +++ b/src/Paillave.Etl.Http/HttpFileValueProcessor.cs @@ -34,7 +34,8 @@ IExecutionContext context var httpClient = IHttpConnectionInfoEx.CreateHttpClient( connectionParameters, - processorParameters + processorParameters, + fileValue.Metadata ); var response = HttpHelpers @@ -67,7 +68,8 @@ IExecutionContext context push( new HttpPostFileValue( - content, + //content, + connectionParameters.Url.ToStream(), fileName, new HttpPostFileValueMetadata { diff --git a/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj b/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj index 241ae1b5..6b4afef9 100644 --- a/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj +++ b/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj @@ -14,7 +14,7 @@ all - + diff --git a/src/Paillave.Etl.Mail/MailFileValueProcessor.cs b/src/Paillave.Etl.Mail/MailFileValueProcessor.cs index b0a8f07c..2b33f2c0 100644 --- a/src/Paillave.Etl.Mail/MailFileValueProcessor.cs +++ b/src/Paillave.Etl.Mail/MailFileValueProcessor.cs @@ -109,6 +109,12 @@ private void SendSingleFile(MailAdapterConnectionParameters connectionParameters smtpClient.Send(mailMessage); } } + /// + /// TODO : replace with liquid syntax + /// + /// + /// + /// private static string FormatText(string text, JToken metadata) { var regex = new Regex("\\{(?.+?)(:(.+?))?}", RegexOptions.Singleline); diff --git a/src/Tutorials/BlogTutorial/Program copy.cs b/src/Tutorials/BlogTutorial/Program copy.cs index e677e0b2..166d46a2 100644 --- a/src/Tutorials/BlogTutorial/Program copy.cs +++ b/src/Tutorials/BlogTutorial/Program copy.cs @@ -9,7 +9,7 @@ namespace BlogTutorial { class Program { - static async Task Main(string[] args) + static async Task Main1(string[] args) { var processRunner = StreamProcessRunner.Create(DefineProcess102); var executionOptions = new ExecutionOptions diff --git a/src/Tutorials/BlogTutorial/Program.cs b/src/Tutorials/BlogTutorial/Program.cs index bbd7b37e..c76f03cc 100644 --- a/src/Tutorials/BlogTutorial/Program.cs +++ b/src/Tutorials/BlogTutorial/Program.cs @@ -12,7 +12,7 @@ namespace BlogTutorial { class Program9 { - static async Task Main9(string[] args) + static async Task Main(string[] args) { var processRunner = StreamProcessRunner.Create(DefineProcess2); processRunner.DebugNodeStream += (sender, e) diff --git a/src/Tutorials/Paillave.Etl.Samples/Program10.cs b/src/Tutorials/Paillave.Etl.Samples/Program10.cs new file mode 100644 index 00000000..dd42de55 --- /dev/null +++ b/src/Tutorials/Paillave.Etl.Samples/Program10.cs @@ -0,0 +1,91 @@ + +using Paillave.Etl.Core; +using Paillave.Etl.Http; + +namespace Paillave.Etl.Samples +{ + + class Program10 + { + static async Task Main(string[] args) + { + var httpAdapterProviderParameters = new HttpAdapterProviderParameters + { + Method = HttpMethodCustomEnum.Post, + Body = new Dictionary + { + {"name", "{{AdditionalParameters.bodyValueName}}" }, + {"age", 30}, + {"city", "New York"} + } + }; + var httpAdapterConnectionParameters = new HttpAdapterConnectionParameters + { + //Url = "https://www.google.com/search?q={{AdditionalParameters.query}}", + //Url = "https://www.w3schools.com/{{AdditionalParameters.format}}/{{AdditionalParameters.file}}" + + Url = "http://echo.free.beeceptor.com/sample-request?author={{AdditionalParameters.author}}", + }; + + var connectors = new FileValueConnectors(); + connectors.Register( + new HttpFileValueProvider( + "MyHttpSourceForAdditionalParameters", + "Input", + "HttpConnection", + httpAdapterConnectionParameters, + httpAdapterProviderParameters + ) + ); + + var executionOptions = new ExecutionOptions { Connectors = connectors }; + var processRunner = StreamProcessRunner.Create(Import); + var res = await processRunner.ExecuteAsync(args, executionOptions); + } + + public static void Import(ISingleStream contextStream) + { + contextStream + .FromConnector("get from http", "MyHttpSourceForAdditionalParameters") + .Select( + "select", + i => + { + if (((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters == null) + { + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters = new Dictionary(); + } + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("file", "xmlhttp_info.txt"); + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("format", "xml"); + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("query", "Voiture"); + + + + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("author", "beeceptor"); + ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("bodyValueName", "Mickael"); + + + + + return i; + } + ) + .Do( + "print to console", + i => + { + using (var memoryStream = new MemoryStream()) + { + i.GetContent().CopyTo(memoryStream); + byte[] contentBytes = memoryStream.ToArray(); + string contentString = System.Text.Encoding.UTF8.GetString( + contentBytes + ); + Console.WriteLine($"result ({contentString}"); + Console.WriteLine($"result ({i.Name})"); + } + } + ); + } + } +} diff --git a/src/Tutorials/Paillave.Etl.Samples/Program9.cs b/src/Tutorials/Paillave.Etl.Samples/Program9.cs index ac750705..f9e1d6d7 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Program9.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Program9.cs @@ -23,7 +23,7 @@ public class TestHeaders class Program9 { - static async Task Main(string[] args) + static async Task Main9(string[] args) { var httpAdapterConnectionParameters = new HttpAdapterConnectionParameters {