From d99bf2d2e69a7866c232d8a2e66642c614a569d8 Mon Sep 17 00:00:00 2001 From: xietao Date: Wed, 11 Jul 2018 20:35:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=BA=20dp2Capo=20=E7=9A=84=20Z39.50=20Serv?= =?UTF-8?q?ice=20=E5=A2=9E=E5=8A=A0=20IpTable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DigitalPlatform.Net.csproj | 1 + DigitalPlatform.Net/IpTable.cs | 395 ++++++++++++++++++ .../DigitalPlatform.Z3950.Server.csproj | 1 + DigitalPlatform.Z3950.Server/ZServer.cs | 60 ++- dp2Capo/ServerConnection.cs | 172 +++++++- dp2Capo/ServerInfo.cs | 2 + 6 files changed, 616 insertions(+), 15 deletions(-) create mode 100644 DigitalPlatform.Net/IpTable.cs diff --git a/DigitalPlatform.Net/DigitalPlatform.Net.csproj b/DigitalPlatform.Net/DigitalPlatform.Net.csproj index 045890d4..ea549f05 100644 --- a/DigitalPlatform.Net/DigitalPlatform.Net.csproj +++ b/DigitalPlatform.Net/DigitalPlatform.Net.csproj @@ -42,6 +42,7 @@ + diff --git a/DigitalPlatform.Net/IpTable.cs b/DigitalPlatform.Net/IpTable.cs new file mode 100644 index 00000000..05e316e6 --- /dev/null +++ b/DigitalPlatform.Net/IpTable.cs @@ -0,0 +1,395 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace DigitalPlatform.Net +{ + /// + /// 跟踪和限制前端每个 IP 连接数量的工具类 + /// + public class IpTable + { + ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); + // static int m_nLockTimeout = 5000; // 5000=5秒 + + // IpTable 中允许存储的 IpEntry 最大个数 + int _nMaxEntryCount = 10000; + + // 判定攻击行为的两个参数： + int _maxTotalConnectRequest = 1000; // (同一 IP)一段时间内累计的 Connect 请求个数 + TimeSpan _period = TimeSpan.FromSeconds(10); // 时间段长度 + + /// + /// 每个(来自前端的) IP 最多允许访问服务器的前端机器数量 + /// + public int MaxClientsPerIp = 1000; // -1 表示不限制 + + Dictionary _ipTable = new Dictionary(); // IP -- 信息 对照表 + + // 查找一个 IpEntry + public IpEntry FindIpEntry(string ip) + { + // TODO: localhost 注意做归一化处理 + _lock.EnterReadLock(); + try + { + if (_ipTable.TryGetValue(ip, out IpEntry value) == false) + return null; + return value; + } + finally + { + _lock.ExitReadLock(); + } + } + + public bool RemoveIpEntry(string ip, bool condition) + { + if (condition) + { + IpEntry entry = FindIpEntry(ip); + if (entry == null) + return false; + + if (entry.CanRemove() == false) + return false; + } + + // 然后用写锁定尝试 + _lock.EnterWriteLock(); + try + { + return _ipTable.Remove(ip); + } + finally + { + _lock.ExitWriteLock(); + } + } + + // 确保获得一个 IpEntry + // exception: + // 可能会抛出异常 + public IpEntry GetIpEntry(string ip) + { + // TODO: localhost 注意做归一化处理 + // 先用读锁定尝试一次 + _lock.EnterReadLock(); + try + { + if (_ipTable.TryGetValue(ip, out IpEntry value) == false) + goto NEXT; + return value; + } + finally + { + _lock.ExitReadLock(); + } + + NEXT: + // 然后用写锁定尝试 + _lock.EnterWriteLock(); + try + { + if (_ipTable.TryGetValue(ip, out IpEntry value) == false) + { + if (this._ipTable.Count > _nMaxEntryCount) + throw new Exception("IP 表超过条目配额 " + _nMaxEntryCount); + + IpEntry entry = new IpEntry(); + _ipTable.Add(ip, entry); + return entry; + } + return value; + } + finally + { + _lock.ExitWriteLock(); + } + } + + // 清除所有黑名单事项 + public void ClearBlackList(TimeSpan delta) + { + List list = new List(); + _lock.EnterReadLock(); + try + { + foreach (var pair in _ipTable.ToList()) + { + list.Add(pair.Value); + } + } + finally + { + _lock.ExitReadLock(); + } + + foreach (IpEntry entry in list) + { + if (entry.IsBackListExpire(delta)) + entry.SetInBlackList(false); + } + } + + // 检查 IP。 + // exception: + // 可能会抛出异常 + // return: + // null 许可 IP 被使用 + // 其他 禁止 IP 使用。字符串内容为禁止理由 + public string CheckIp(string ip) + { + // exception: + // 可能会抛出异常 + IpEntry entry = GetIpEntry(ip); + if (entry.IsInBlackList() == true) + return "IP [" + ip + "] 在黑名单之中"; + + // return: + // false 总量超过 max + // true 总量没有超过 max + if (entry.CheckTotal(_maxTotalConnectRequest, _period) == false) + { + // TODO: 何时从黑名单中自动清除? + entry.SetInBlackList(); + return "IP [" + ip + "] 短时间 (" + _period.TotalSeconds.ToString() + "秒) 内连接请求数超过极限 (" + _maxTotalConnectRequest + ")，已被加入黑名单"; + } + + long value = entry.IncOnline(); + if (MaxClientsPerIp != -1 && value >= MaxClientsPerIp) + { + entry.DecOnline(); + return "IP 地址为 '" + ip + "' 的前端数量超过配额 " + MaxClientsPerIp; + } + + return null; + } + + // 释放 IpEntry + public void FinishIp(string ip) + { + IpEntry entry = FindIpEntry(ip); + if (entry == null) + return; + long value = entry.DecOnline(); + if (value == 0) + { + // 发出清除 0 值条目的请求。不过也许不该清除，因为后面还要判断 Total。可以改为清除 Total 值小于某个阈值的条目 + RemoveIpEntry(ip, true); + } + } + +#if NO + // 增量 IP 统计数字 + // 如果 IP 事项总数超过限额，会抛出异常 + // parameters: + // strIP 前端机器的 IP 地址。还用于辅助判断是否超过 MaxClients。localhost 是不计算在内的 + long _incIpCount(string strIP, int nDelta) + { + // this.MaxClients = 0; // test + if (nDelta != 0 && nDelta != 1 && nDelta != -1) + throw new ArgumentException("nDelta 参数值应为 0 -1 1 之一"); + + IpEntry entry = GetIpEntry(strIP); + + long oldOnline = 0; + if (nDelta == 1) + { + oldOnline = entry.IncOnline(); + if (oldOnline >= MaxClientsPerIp) + { + entry.DecOnline(); + throw new Exception("IP 地址为 '" + strIP + "' 的前端数量超过配额 " + MaxClientsPerIp); + } + + return oldOnline; + } + else if (nDelta == -1) + { + oldOnline = entry.DecOnline(); + if (oldOnline == 1) + { + // 发出清除 0 值条目的请求。不过也许不该清除，因为后面还要判断 Total。可以改为清除 Total 值小于某个阈值的条目 + RemoveIpEntry(strIP); + } + + return oldOnline; + } + else + { + Debug.Assert(nDelta == 0, ""); + } + + return entry.OnlineCount; +#if NO + long v = 0; + if (this._ipTable.ContainsKey(strIP) == true) + v = (long)this._ipTable[strIP]; + else + { + if (this.Count > _nMaxCount + && v + nDelta != 0) + throw new OutofSessionException("IP 条目数量超过 " + _nMaxCount.ToString()); + + // 判断前端机器台数是否超过限制数额 2014/8/23 + if (this.MaxClients != -1 + && IsLocalhost(strIP) == false + && this.GetClientIpAmount() >= this.MaxClients + && v + nDelta != 0) + throw new OutofClientsException("前端机器数量已经达到 " + this.GetClientIpAmount().ToString() + " 个 ( 现有IP: " + StringUtil.MakePathList(GetIpList(), ", ") + " 试图申请的IP: " + strIP + ")。请先释放出通道然后重新访问"); + + } + + if (v + nDelta == 0) + this._ipTable.Remove(strIP); // 及时移走计数器为 0 的条目，避免 hashtable 尺寸太大 + else + this._ipTable[strIP] = v + nDelta; + + return v; // 返回增量前的数字 +#endif + } + +#endif + + } + + /// + /// 一个 IP 信息事项 + /// + public class IpEntry + { + private long _onlineCount; + + // 一段时间内累计的连接请求个数 + private long _totalConnectCount; + // 最近一次累计的开始时间 + private DateTime _totalStartTime = DateTime.Now; + + private AttackInfo _attackInfo = null; + + // 当前在线的数量 + public long OnlineCount { get => _onlineCount; set => _onlineCount = value; } + + // 发生过 Connect 的总次数 + public long TotalConnectCount { get => _totalConnectCount; set => _totalConnectCount = value; } + + public long IncOnline() + { + return Interlocked.Increment(ref _onlineCount); + } + + public long DecOnline() + { + return Interlocked.Decrement(ref _onlineCount); + } + + public long IncTotal() + { + return Interlocked.Increment(ref _totalConnectCount); + } + + public long DecTotal() + { + return Interlocked.Decrement(ref _totalConnectCount); + } + + // 检查条目是否处于黑名单内 + public bool IsInBlackList() + { + if (this._attackInfo != null) + return true; + return false; + } + + // 判断黑名单是否过期 + public bool IsBackListExpire(TimeSpan delta) + { + if (this._attackInfo == null) + return false; + return this._attackInfo.IsExpired(delta); + } + + public void SetInBlackList(bool set = true) + { + if (set) + { + if (this._attackInfo == null) + this._attackInfo = new AttackInfo(); + + this._attackInfo.Memo(); + } + else + { + if (this._attackInfo != null) + this._attackInfo = null; + } + } + + // 如果累计开始时间超过指定长度，则清除 total 值 + public void TryClearTotal(TimeSpan delta) + { + TimeSpan length = DateTime.Now - _totalStartTime; + + //Debug.WriteLine("length=" + length.ToString() + ", delta=" + delta.ToString()); + + if (length > delta) + ClearTotal(); + } + + public void ClearTotal() + { + _totalStartTime = DateTime.Now; + _totalConnectCount = 0; + //Debug.WriteLine("Clear() _totalConnectCount"); + } + + // 检查总量是否超出 + // 注意返回前，_totalConnectCount 已经被加 1 + // return: + // false 总量超过 max + // true 总量没有超过 max。 + public bool CheckTotal(int max, TimeSpan delta) + { + TryClearTotal(delta); + + IncTotal(); + //Debug.WriteLine("_totalConnectCount=" + _totalConnectCount); + if (_totalConnectCount > max) + return false; + return true; + } + + // 是否允许清理？ + public bool CanRemove() + { + if (this._totalConnectCount <= 1) + return true; + return false; + } + } + + // 攻击信息 + public class AttackInfo + { + public DateTime AttackTime = DateTime.Now; // 最后一次攻击的时间 + public long AttackCount = 0; // 一共发生的攻击次数 + + // 记载一次攻击 + public void Memo() + { + this.AttackTime = DateTime.Now; + this.AttackCount++; + } + + // parameters: + // delta 从最后一次攻击的时间计算，多长以后清除攻击记忆 + public bool IsExpired(TimeSpan delta) + { + if (DateTime.Now > this.AttackTime + delta) + return true; + return false; + } + } +} diff --git a/DigitalPlatform.Z3950.Server/DigitalPlatform.Z3950.Server.csproj b/DigitalPlatform.Z3950.Server/DigitalPlatform.Z3950.Server.csproj index 5e10e169..b243eade 100644 --- a/DigitalPlatform.Z3950.Server/DigitalPlatform.Z3950.Server.csproj +++ b/DigitalPlatform.Z3950.Server/DigitalPlatform.Z3950.Server.csproj @@ -13,6 +13,7 @@ + diff --git a/DigitalPlatform.Z3950.Server/ZServer.cs b/DigitalPlatform.Z3950.Server/ZServer.cs index a6fd2fd0..29fd00ad 100644 --- a/DigitalPlatform.Z3950.Server/ZServer.cs +++ b/DigitalPlatform.Z3950.Server/ZServer.cs @@ -6,7 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; - +using DigitalPlatform.Net; using log4net; namespace DigitalPlatform.Z3950.Server @@ -48,6 +48,8 @@ public class ZServer : IDisposable private TcpListener Listener; private bool IsActive = true; + private IpTable IpTable; + #endregion // public static ILog _log = null; @@ -66,6 +68,7 @@ static string GetClientIP(TcpClient s) public async void Listen(int backlog) { + this.IpTable = new IpTable(); this.Listener = new TcpListener(IPAddress.Any, this.Port); this.Listener.Start(backlog); // TODO: 要捕获异常 @@ -73,20 +76,45 @@ public async void Listen(int backlog) while (this.IsActive) { + TcpClient tcpClient = null; try { - TcpClient tcpClient = await this.Listener.AcceptTcpClientAsync(); + tcpClient = await this.Listener.AcceptTcpClientAsync(); // string ip = ((IPEndPoint)s.Client.RemoteEndPoint).Address.ToString(); string ip = GetClientIP(tcpClient); - ZManager.Log?.Info("*** ip [" + ip + "] request"); + // ZManager.Log?.Info("*** ip [" + ip + "] request"); + + if (this.IpTable != null) + { + string error = this.IpTable.CheckIp(ip); + if (error != null) + { + tcpClient.Close(); + // TODO: 可以在首次出现这种情况的时候记入错误日志 + // ZManager.Log?.Info("*** ip [" + ip + "] 被禁止 Connect。原因: " + error); + continue; + } + } + + Task task = // 用来消除警告 // https://stackoverflow.com/questions/18577054/alternative-to-task-run-that-doesnt-throw-warning + Task.Run(() => + HandleClient(tcpClient, + () => + { + if (this.IpTable != null) + this.IpTable.FinishIp(ip); + tcpClient = null; + }, + _cancelToken)); - Task.Run(() => HandleClient(tcpClient, _cancelToken)); - // TestHandleClient(tcpClient, ServerInfo._cancel.Token); } catch (Exception ex) { + if (tcpClient != null) + tcpClient.Close(); + if (this.IsActive == false) break; ZManager.Log?.Error("Listen() 出现异常: " + ExceptionUtil.GetExceptionMessage(ex)); @@ -102,6 +130,15 @@ public void Close() _zChannels.Clear(); } + public void TryClearBlackList() + { + if (this.IpTable == null) + return; + + // 清理一次黑名单 + this.IpTable.ClearBlackList(TimeSpan.FromMinutes(10)); + } + #if NO public virtual async void TestHandleClient(TcpClient tcpClient, CancellationToken token) @@ -117,6 +154,7 @@ public void Dispose() // 处理一个通道的通讯活动 public async void HandleClient(TcpClient tcpClient, + Action close_action, CancellationToken token) { ZServerChannel channel = _zChannels.Add(tcpClient); @@ -139,6 +177,8 @@ public async void HandleClient(TcpClient tcpClient, bool running = true; while (running) { + if (token != null && token.IsCancellationRequested) + return; // 注意调用返回后如果发现返回 null 或者抛出了异常，调主要主动 Close 和重新分配 TcpClient BerTree request = await ZProcessor.GetIncomingRequest(tcpClient); if (request == null) @@ -149,6 +189,8 @@ public async void HandleClient(TcpClient tcpClient, Console.WriteLine("request " + i); channel.Touch(); + if (token != null && token.IsCancellationRequested) + return; byte[] response = null; if (this.ProcessRequest == null) @@ -162,6 +204,8 @@ public async void HandleClient(TcpClient tcpClient, } channel.Touch(); + if (token != null && token.IsCancellationRequested) + return; // 注意调用返回 result.Value == -1 情况下，要及时 Close TcpClient Result result = await ZProcessor.SendResponse(response, tcpClient); @@ -177,8 +221,9 @@ public async void HandleClient(TcpClient tcpClient, } catch (Exception ex) { - // 2016/11/14 - ZManager.Log?.Error("ip:" + ip + " HandleClient() 异常: " + ExceptionUtil.GetExceptionText(ex)); + string strError = "ip:" + ip + " HandleClient() 异常: " + ExceptionUtil.GetExceptionText(ex); + ZManager.Log?.Error(strError); + // Console.WriteLine(strError); } finally { @@ -208,6 +253,7 @@ public async void HandleClient(TcpClient tcpClient, } #endif channel.Close(); + close_action.Invoke(); } } diff --git a/dp2Capo/ServerConnection.cs b/dp2Capo/ServerConnection.cs index 1edc5b1c..155c2ac9 100644 --- a/dp2Capo/ServerConnection.cs +++ b/dp2Capo/ServerConnection.cs @@ -1409,6 +1409,10 @@ void SearchAndResponse(SearchRequest searchParam) return; } + SearchBiblio(searchParam); + return; + +#if NO string strError = ""; string strErrorCode = ""; IList records = new List(); @@ -1556,9 +1560,165 @@ void SearchAndResponse(SearchRequest searchParam) this.dp2library.LibraryUID, records, strError, +strErrorCode)); +#endif + } + + void SearchBiblio(SearchRequest searchParam) + { + string strError = ""; + string strErrorCode = ""; + IList records = new List(); + + string strResultSetName = searchParam.ResultSetName; + if (string.IsNullOrEmpty(strResultSetName) == true) + strResultSetName = "default"; // "#" + searchParam.TaskID; // "default"; + else + strResultSetName = "#" + strResultSetName; // 如果请求方指定了结果集名，则在 dp2library 中处理为全局结果集名 + + try + { + LibraryChannel channel = GetChannel(searchParam.LoginInfo); + TimeSpan old_timeout = channel.Timeout; + if (searchParam.Timeout != TimeSpan.FromMilliseconds(0)) + channel.Timeout = searchParam.Timeout; + try + { + string strQueryXml = ""; + long lRet = 0; + + if (searchParam.QueryWord == "!getResult") + { + lRet = -1; + } + else + { + if (searchParam.Operation == "searchBiblio") + { + // TODO: 根据数据库名列表，分析出需要针对 dp2library 服务器检索和针对 Z39.50 服务器检索的部分，分别调用 + // 等待所有 Task 完成后，统一发出一个表示结束的包 + lRet = channel.SearchBiblio(// null, + searchParam.DbNameList, + searchParam.QueryWord, + (int)searchParam.MaxResults, + searchParam.UseList, + searchParam.MatchStyle, + "zh", + strResultSetName, + "", // strSearchStyle + "", // strOutputStyle + searchParam.Filter, + out strQueryXml, + out strError); + writeDebug("searchBiblio() lRet=" + lRet + + ", strQueryXml=" + strQueryXml + + ", strError=" + strError + + ",errorcode=" + channel.ErrorCode.ToString()); + } + else if (searchParam.Operation == "searchPatron") + { + lRet = channel.SearchReader(// null, + searchParam.DbNameList, + searchParam.QueryWord, + (int)searchParam.MaxResults, + searchParam.UseList, + searchParam.MatchStyle, + "zh", + strResultSetName, + "", + out strError); + } + else + { + lRet = -1; + strError = "无法识别的 Operation 值 '" + searchParam.Operation + "'"; + } + + strErrorCode = channel.ErrorCode.ToString(); + + if (lRet == -1 || lRet == 0) + { + if (lRet == 0 + || (lRet == -1 && channel.ErrorCode == DigitalPlatform.LibraryClient.localhost.ErrorCode.NotFound)) + { + // 没有命中 + TryResponseSearch( + new SearchResponse( + searchParam.TaskID, + 0, + 0, + this.dp2library.LibraryUID, + records, + strError, // 出错信息大概为 not found。 + strErrorCode)); + return; + } + goto ERROR1; + } + } + + + { + long lHitCount = lRet; + + if (searchParam.Count == 0) + { + // 返回命中数 + TryResponseSearch( + new SearchResponse( + searchParam.TaskID, + lHitCount, + 0, + this.dp2library.LibraryUID, + records, + "本次没有返回任何记录", + strErrorCode)); + return; + } + + // 注: SendResults() 负责 ReturnChannel() + LibraryChannel temp_channel = channel; + channel = null; + Task.Run(() => SendResults( + temp_channel, + searchParam, + strResultSetName, + lHitCount)); + } + } + finally + { + if (channel != null) + { + channel.Timeout = old_timeout; + this.ReturnChannel(channel); + } + } + + this.AddInfoLine("search and response end"); + return; + } + catch (Exception ex) + { + AddErrorLine("SearchAndResponse() 出现异常: " + ex.Message); + strError = ExceptionUtil.GetDebugText(ex); + goto ERROR1; + } + + ERROR1: + // 报错 + TryResponseSearch( + new SearchResponse( +searchParam.TaskID, +-1, +0, +this.dp2library.LibraryUID, +records, +strError, strErrorCode)); } + // 注: 本函数最后要主动 ReturnChannel() void SendResults( LibraryChannel channel, @@ -1614,7 +1774,7 @@ void SendResults( + ",lStart=" + lStart + ",lPerCount=" + lPerCount + ",strBrowseStyle=" + strBrowseStyle - + ", strError=" + strError + + ",strError=" + strError + ",errorcode=" + strErrorCode); if (lRet == -1) @@ -2201,16 +2361,12 @@ void GetBiblioInfo(SearchRequest searchParam) nPathFormatIndex = Array.IndexOf(formats, "outputpath"); } - string[] results = null; - // string strRecPath = ""; - byte[] baTimestamp = null; - long lRet = channel.GetBiblioInfos( searchParam.QueryWord, searchParam.UseList, // strBiblioXml formats, - out results, - out baTimestamp, + out string[] results, + out byte[] baTimestamp, out strError); strErrorCode = channel.ErrorCode.ToString(); if (lRet == -1 || lRet == 0) @@ -2688,7 +2844,7 @@ void GetUserInfo(SearchRequest searchParam) strErrorCode)); } - #endregion +#endregion #if NO // 连接成功后被调用，执行登录功能。重载时要调用 Login(...) 向 server 发送 login 消息 diff --git a/dp2Capo/ServerInfo.cs b/dp2Capo/ServerInfo.cs index a3058834..057d78be 100644 --- a/dp2Capo/ServerInfo.cs +++ b/dp2Capo/ServerInfo.cs @@ -742,6 +742,8 @@ public static void BackgroundWork() // 清除废弃的全局结果集 Task.Run(() => instance.FreeGlobalResultSets()); + + ZServer.TryClearBlackList(); } }