g82tt/KinginfoGateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化业务插件缓存逻辑

Browse Source
This commit is contained in:
Kimdiego2098
2024-03-01 13:33:43 +08:00
parent e69ea0b9dc
commit 76fef9c807
11 changed files with 187 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/ThingsGateway.Admin.Application/Util/UserTokenCacheUtil.cs
View File

@@ -15,7 +15,7 @@ namespace ThingsGateway.Admin.Application;
/// </summary>
public class UserTokenCacheUtil
{
public static LiteDBCache<UserTokenCache> UserTokenCache = LiteDBCacheUtil.GetDB<UserTokenCache>(nameof(UserTokenCache), $"{typeof(UserTokenCache).FullName}", false);
public static LiteDBCache<UserTokenCache> UserTokenCache = LiteDBCacheUtil.GetDB<UserTokenCache>(nameof(UserTokenCache), $"{typeof(UserTokenCache).FullName}", true, false);
public static List<VerificatInfo>? HashGetOne(long id)
{

94
src/ThingsGateway.Cache/Cache/LiteDBCache/LiteDB/LiteDBCache.cs
View File

@@ -61,73 +61,104 @@ public class LiteDBCache<T> : IDisposable where T : IPrimaryIdEntity
public void Dispose()
{
try { _dbProvider.Dispose(); } catch { }
lock (this)
{
try { _dbProvider.Dispose(); } catch { }
}
}
public IEnumerable<T>? GetPage(int skipCount, int pageSize)
public List<T>? GetPage(int skipCount, int pageSize)
{
var results = Collection.Find(Query.All(), skipCount, pageSize);
return results;
lock (this)
{
var results = Collection.Find(Query.All(), skipCount, pageSize).ToList();
return results;
}
}
public T? GetOne(long id)
{
var results = Collection.FindById(id);
return results;
lock (this)
{
var results = Collection.FindById(id);
return results;
}
}
public IEnumerable<T>? Get(long[] ids)
{
var results = Collection.Find(a => ids.Contains(a.Id));
return results;
lock (this)
{
var results = Collection.Find(a => ids.Contains(a.Id));
return results;
}
}
public IEnumerable<T>? GetAll()
{
var results = Collection.FindAll();
return results;
lock (this)
{
var results = Collection.FindAll();
return results;
}
}
public int AddRange(IEnumerable<T> data, int batchSize = 5000)
{
var results = Collection.InsertBulk(data, batchSize);
return results;
lock (this)
{
var results = Collection.InsertBulk(data, batchSize);
return results;
}
}
public void Add(T data)
{
Collection.Insert(data);
lock (this)
{
Collection.Insert(data);
}
}
public int DeleteMany(IEnumerable<T> data)
{
var results = Collection.DeleteMany(a => data.Select(item => item.Id).Contains(a.Id));
return results;
lock (this)
{
var results = Collection.DeleteMany(a => data.Select(item => item.Id).Contains(a.Id));
return results;
}
}
public int DeleteMany(Expression<Func<T, bool>> predicate)
{
var results = Collection.DeleteMany(predicate);
return results;
lock (this)
{
var results = Collection.DeleteMany(predicate);
return results;
}
}
public int DeleteMany(BsonExpression bsonExpression)
{
var results = Collection.DeleteMany(bsonExpression);
return results;
lock (this)
{
var results = Collection.DeleteMany(bsonExpression);
return results;
}
}
/// <summary>
/// init database
/// </summary>
public void InitDb()
public void InitDb(bool isRebuild = false)
{
lock (_dbProvider)
lock (this)
{
try
{
_dbProvider.Checkpoint();
//_dbProvider.Rebuild();
if (isRebuild)
_dbProvider.Rebuild();
lock (Collection)
{
//建立索引
@@ -141,15 +172,18 @@ public class LiteDBCache<T> : IDisposable where T : IPrimaryIdEntity
private LiteDatabase GetConnection(LiteDBOptions options)
{
ConnectionString builder = new ConnectionString()
lock (this)
{
Filename = options.DataSource,
//InitialSize = options.InitialSize,
Connection = options.ConnectionType,
Password = options.Password
};
var _conn = new LiteDatabase(builder);
ConnectionString builder = new ConnectionString()
{
Filename = options.DataSource,
//InitialSize = options.InitialSize,
Connection = options.ConnectionType,
Password = options.Password
};
var _conn = new LiteDatabase(builder);
return _conn;
return _conn;
}
}
}

102
src/ThingsGateway.Cache/Cache/LiteDBCache/LiteDB/LiteDBCacheUtil.cs
View File

@@ -9,6 +9,7 @@
//------------------------------------------------------------------------------
using Furion;
using Furion.Logging.Extensions;
using NewLife;
@@ -32,7 +33,7 @@ public static class LiteDBCacheUtil
/// 如果当前文件的大小超限，则返回新的链接
/// 如果当前文件的数量超限，则删除部分旧文件
/// </summary>
public static LiteDBCache<T>? GetDB<T>(string id, string typeName, bool isDeleteRule = true) where T : IPrimaryIdEntity
public static LiteDBCache<T>? GetDB<T>(string id, string typeName, bool isInsert, bool isDeleteRule = true) where T : IPrimaryIdEntity
{
lock (_dictObject)
{
@@ -49,6 +50,7 @@ public static class LiteDBCacheUtil
var driveUsage = (100 - (drive.TotalFreeSpace * 100.00 / drive.TotalSize));
if (driveUsage > LiteDBCacheUtil.config.MaxDriveUsage)
{
$"磁盘使用率超限，将删除缓存文件".LogInformation();
//删除全部文件夹中旧文件
string[] dirs = Directory.GetDirectories(GetFileBasePath());
//遍历全部文件夹，删除90%的文件
@@ -89,6 +91,7 @@ public static class LiteDBCacheUtil
string[] files = Directory.GetFiles(dir, searchPattern);
if (files.Length > LiteDBCacheUtil.config.MaxFileCount)
{
$"{dir}缓存文件数量超限，将删除文件".LogInformation();
//数量超限就删除旧文件
//按文件更改时间降序排序
var sortedFiles = files.OrderBy(file => File.GetLastWriteTime(file)).ToArray();
@@ -124,8 +127,9 @@ public static class LiteDBCacheUtil
}
var mb1 = Math.Round((double)length1 / (double)1024 / (double)1024, 2);
if (mb1 > LiteDBCacheUtil.config.MaxFileLength)
if (isInsert && mb1 > LiteDBCacheUtil.config.MaxFileLength)
{
$"{fullName}缓存文件大小超限，将产生新文件".LogInformation();
//大小超限就返回新的文件
var newFullName = dir.CombinePath($"{fileStart}_{maxNum + 1}{ex}");
{
@@ -162,23 +166,54 @@ public static class LiteDBCacheUtil
}
}
}
{
if (_dict.TryGetValue(fullName, out object cache1))
{
//返回原连接
try
{
return (LiteDBCache<T>)cache1;
var connect = (LiteDBCache<T>)cache1;
if (maxNum > 1 && !isInsert)
{
if (connect.GetPage(1, 1).Count == 0)
{
//无内容时，删除文件
DisposeAndDeleteFile(fullName, cache1);
return GetDB<T>(id, typeName, isInsert, isDeleteRule);
}
}
else
{
if (maxNum == 1)
{
if (isDeleteRule)
{
long? length1 = null;
if (!File.Exists(fullName))
{
length1 = 0;
}
else
{
length1 = new FileInfo(fullName).Length;
}
var mb1 = Math.Round((double)length1 / (double)1024 / (double)1024, 2);
if (mb1 > LiteDBCacheUtil.config.MaxFileLength)
{
connect.InitDb(true);
}
}
}
}
return connect;
}
catch (Exception)
{
//可能类型变换导致错误，此时返回null,并释放连接
//可能类型变换导致错误，此时释放连接
DisposeAndDeleteFile(fullName, cache1);
var cache = new LiteDBCache<T>(id, typeName, fullName);
_dict.TryAdd(fullName, cache);
return cache;
return GetDB<T>(id, typeName, isInsert, isDeleteRule);
}
}
{
@@ -194,6 +229,7 @@ public static class LiteDBCacheUtil
{
if (File.Exists(file))
{
$"删除{file}缓存文件".LogInformation();
File.SetAttributes(file, FileAttributes.Normal);
File.Delete(file);
}
@@ -238,24 +274,40 @@ public static class LiteDBCacheUtil
{
var dir = GetFilePath(id);
var fileStart = GetFileStartName(typeName);
string[] files = Directory.GetFiles(dir, $"{fileStart}_*.ldb");
int maxNumber = 1;
Regex regex = new Regex(@"_(\d+)\.ldb$");
foreach (var file in files)
{
Match match = regex.Match(file);
if (match.Success && int.TryParse(match.Groups[1].Value, out int number))
{
if (number > maxNumber)
{
maxNumber = number;
}
}
}
return maxNumber;
//搜索全部符合条件的文件
if (!File.Exists(dir.CombinePath($"{fileStart}_{ex}")))
{
return null;
}
var index = 1;
while (true)
{
var newFileName = dir.CombinePath($"{fileStart}_{index}{ex}");
if (System.IO.File.Exists(newFileName))
{
index++;
}
else
{
return (index == 1 ? null : index - 1);
}
}
//if (!File.Exists(dir.CombinePath($"{fileStart}_1{ex}")))
//{
// return 1;
//}
//var index = 2;
//while (true)
//{
// var newFileName = dir.CombinePath($"{fileStart}_{index}{ex}");
// if (System.IO.File.Exists(newFileName))
// {
// index++;
// }
// else
// {
// return (index - 1);
// }
//}
}
}

8
src/ThingsGateway.Cache/Config/Cache.Development.json
View File

@@ -1,8 +1,8 @@
{
//缓存设置
"LiteDBConfig": {
"MaxFileLength": "400",
"MaxFileCount": "20",
"MaxDriveUsage": "90"
"LiteDBConfig": { //单个缓存文件夹
"MaxFileLength": "64", //最大文件大小
"MaxFileCount": "10", //最大文件数量
"MaxDriveUsage": "90" //最大磁盘使用率
}
}

8
src/ThingsGateway.Cache/Config/Cache.Production.json
View File

@@ -1,8 +1,8 @@
{
//缓存设置
"LiteDBConfig": {
"MaxFileLength": "400",
"MaxFileCount": "20",
"MaxDriveUsage": "90"
"LiteDBConfig": { //单个缓存文件夹
"MaxFileLength": "64", //最大文件大小
"MaxFileCount": "10", //最大文件数量
"MaxDriveUsage": "90" //最大磁盘使用率
}
}

13
src/ThingsGateway.Gateway.Application/Plugin/Business/Cache/BusinessBaseWithCacheT.cs
View File

@@ -33,7 +33,10 @@ public abstract class BusinessBaseWithCacheT<T> : BusinessBase
/// <summary>
/// 获取缓存对象，注意每次获取的对象可能不一样，如顺序操作，需固定引用
/// </summary>
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T>> LiteDBCacheT => LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T).FullName}_{nameof(T)}");
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T>> LiteDBCacheT(bool isInsert)
{
return LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T).FullName}_{nameof(T)}", isInsert);
}
protected override IProtocol? Protocol => null;
@@ -101,7 +104,7 @@ public abstract class BusinessBaseWithCacheT<T> : BusinessBase
{
#region //成功上传时，补上传缓存数据
if (success)
if (IsConnected())
{
List<long> successIds = new();
@@ -111,7 +114,7 @@ public abstract class BusinessBaseWithCacheT<T> : BusinessBase
while (!cancellationToken.IsCancellationRequested)
{
//循环获取
var varList = LiteDBCacheT.GetPage(successCount, _businessPropertyWithCache.SplitSize).ToList(); //按最大列表数量分页
var varList = LiteDBCacheT(false).GetPage(successCount, _businessPropertyWithCache.SplitSize); //按最大列表数量分页
if (varList?.Count != 0)
{
try
@@ -151,7 +154,7 @@ public abstract class BusinessBaseWithCacheT<T> : BusinessBase
LogMessage?.LogWarning(ex);
}
if (successIds.Count > 0)
LiteDBCacheT.DeleteMany(a => successIds.Contains(a.Id));
LiteDBCacheT(false).DeleteMany(a => successIds.Contains(a.Id));
}
#endregion //成功上传时，补上传缓存数据
@@ -165,7 +168,7 @@ public abstract class BusinessBaseWithCacheT<T> : BusinessBase
protected virtual void AddCache(List<LiteDBDefalutCacheItem<T>> data)
{
if (data?.Count > 0)
LiteDBCacheT.AddRange(data);
LiteDBCacheT(true).AddRange(data);
}
/// <summary>

13
src/ThingsGateway.Gateway.Application/Plugin/Business/Cache/BusinessBaseWithCacheTT.cs
View File

@@ -24,7 +24,10 @@ public abstract class BusinessBaseWithCacheTT<T, T2> : BusinessBaseWithCacheT<T>
/// <summary>
/// 获取缓存对象，注意每次获取的对象可能不一样，如顺序操作，需固定引用
/// </summary>
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T2>> LiteDBCacheT2 => LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T2>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T2).FullName}_{nameof(T2)}");
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T2>> LiteDBCacheT2(bool isInsert)
{
return LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T2>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T2).FullName}_{nameof(T2)}", isInsert);
}
protected ConcurrentQueue<LiteDBDefalutCacheItem<T2>> _memoryT2Queue = new();
@@ -35,7 +38,7 @@ public abstract class BusinessBaseWithCacheTT<T, T2> : BusinessBaseWithCacheT<T>
protected virtual void AddCache(List<LiteDBDefalutCacheItem<T2>> data)
{
if (data?.Count > 0)
LiteDBCacheT2.AddRange(data);
LiteDBCacheT2(true).AddRange(data);
}
/// <summary>
@@ -130,7 +133,7 @@ public abstract class BusinessBaseWithCacheTT<T, T2> : BusinessBaseWithCacheT<T>
{
#region //成功上传时，补上传缓存数据
if (success)
if (IsConnected())
{
List<long> successIds = new();
@@ -140,7 +143,7 @@ public abstract class BusinessBaseWithCacheTT<T, T2> : BusinessBaseWithCacheT<T>
while (!cancellationToken.IsCancellationRequested)
{
//循环获取
var varList = LiteDBCacheT2.GetPage(successCount, _businessPropertyWithCache.SplitSize).ToList(); //按最大列表数量分页
var varList = LiteDBCacheT2(false).GetPage(successCount, _businessPropertyWithCache.SplitSize); //按最大列表数量分页
if (varList?.Count != 0)
{
try
@@ -180,7 +183,7 @@ public abstract class BusinessBaseWithCacheTT<T, T2> : BusinessBaseWithCacheT<T>
success = false;
}
if (successIds.Count > 0)
LiteDBCacheT2.DeleteMany(a => successIds.Contains(a.Id));
LiteDBCacheT2(false).DeleteMany(a => successIds.Contains(a.Id));
}
#endregion //成功上传时，补上传缓存数据

13
src/ThingsGateway.Gateway.Application/Plugin/Business/Cache/BusinessBaseWithCacheTTT.cs
View File

@@ -24,7 +24,10 @@ public abstract class BusinessBaseWithCacheTTT<T, T2, T3> : BusinessBaseWithCach
/// <summary>
/// 获取缓存对象，注意每次获取的对象可能不一样，如顺序操作，需固定引用
/// </summary>
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T3>> LiteDBCacheT3 => LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T3>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T3).FullName}_{nameof(T3)}");
protected virtual LiteDBCache<LiteDBDefalutCacheItem<T3>> LiteDBCacheT3(bool isInsert)
{
return LiteDBCacheUtil.GetDB<LiteDBDefalutCacheItem<T3>>(CurrentDevice.Id.ToString(), $"{CurrentDevice.PluginName}{typeof(T3).FullName}_{nameof(T3)}", isInsert);
}
protected ConcurrentQueue<LiteDBDefalutCacheItem<T3>> _memoryT3Queue = new();
@@ -35,7 +38,7 @@ public abstract class BusinessBaseWithCacheTTT<T, T2, T3> : BusinessBaseWithCach
protected virtual void AddCache(List<LiteDBDefalutCacheItem<T3>> data)
{
if (data?.Count > 0)
LiteDBCacheT3.AddRange(data);
LiteDBCacheT3(true).AddRange(data);
}
/// <summary>
@@ -132,7 +135,7 @@ public abstract class BusinessBaseWithCacheTTT<T, T2, T3> : BusinessBaseWithCach
{
#region //成功上传时，补上传缓存数据
if (success)
if (IsConnected())
{
List<long> successIds = new();
@@ -142,7 +145,7 @@ public abstract class BusinessBaseWithCacheTTT<T, T2, T3> : BusinessBaseWithCach
while (!cancellationToken.IsCancellationRequested)
{
//循环获取
var varList = LiteDBCacheT3.GetPage(successCount, _businessPropertyWithCache.SplitSize).ToList(); //按最大列表数量分页
var varList = LiteDBCacheT3(false).GetPage(successCount, _businessPropertyWithCache.SplitSize); //按最大列表数量分页
if (varList?.Count != 0)
{
try
@@ -182,7 +185,7 @@ public abstract class BusinessBaseWithCacheTTT<T, T2, T3> : BusinessBaseWithCach
LogMessage?.LogWarning(ex);
}
if (successIds.Count > 0)
LiteDBCacheT3.DeleteMany(a => successIds.Contains(a.Id));
LiteDBCacheT3(false).DeleteMany(a => successIds.Contains(a.Id));
}
#endregion //成功上传时，补上传缓存数据

2
src/ThingsGateway.Gateway.Application/Worker/HardwareInfoWorker.cs
View File

@@ -164,7 +164,7 @@ public class HardwareInfoWorker : BackgroundService
return data.ToList();
}
private LiteDBCache<HisHardwareInfo> cache = LiteDBCacheUtil.GetDB<HisHardwareInfo>(nameof(APPInfo), $"{typeof(HisHardwareInfo).FullName}", false);
private LiteDBCache<HisHardwareInfo> cache = LiteDBCacheUtil.GetDB<HisHardwareInfo>(nameof(APPInfo), $"{typeof(HisHardwareInfo).FullName}", true, false);
}
/// <inheritdoc/>

2
src/plugin/ThingsGateway.Plugin.SqlDB/SqlDB/SqlDBProducer.cs
View File

@@ -77,7 +77,7 @@ YitIdHelper.NextId())
var varList = CurrentDevice.VariableRunTimes.ToList().Adapt<List<SQLRealValue>>();
var result = await UpdateAsync(varList, cancellationToken);
if (success != result.IsSuccess)
if (result != null && success != result.IsSuccess)
{
if (!result.IsSuccess)
LogMessage.LogWarning(result.ToString());

16
src/plugin/ThingsGateway.Plugin.SqlDB/SqlDB/SqlDbProducer.other.cs
View File

@@ -83,10 +83,15 @@ public partial class SqlDBProducer : BusinessBaseWithCacheInterval<SQLHistoryVal
db.Ado.CancellationToken = cancellationToken;
if (!_initRealData)
{
var result = await db.Storageable(datas).As(_driverPropertys.ReadDBTableName).PageSize(5000).ExecuteSqlBulkCopyAsync();
if (result > 0)
LogMessage.Trace($"主题：{nameof(SQLRealValue)}{Environment.NewLine} ，数量：{result}");
_initRealData = true;
if (datas?.Count != 0)
{
var result = await db.Storageable(datas).As(_driverPropertys.ReadDBTableName).PageSize(5000).ExecuteSqlBulkCopyAsync();
if (result > 0)
LogMessage.Trace($"主题：{nameof(SQLRealValue)}{Environment.NewLine} ，数量：{result}");
_initRealData = true;
return new();
}
return null;
}
else
{
@@ -94,10 +99,11 @@ public partial class SqlDBProducer : BusinessBaseWithCacheInterval<SQLHistoryVal
{
var result = await db.Fastest<SQLRealValue>().AS(_driverPropertys.ReadDBTableName).PageSize(100000).BulkUpdateAsync(datas);
LogMessage.Trace($"主题：{nameof(SQLRealValue)}{Environment.NewLine} ，数量：{result}");
return new();
}
return null;
}
return new();
}
catch (Exception ex)
{