Skip to content

GitLab

Commit 705de1f3 authored by Biasmey's avatar Biasmey
Browse files

add cache for duplicates and add support for move directory

parent 79528dc3
Hide whitespace changes
Inline Side-by-side
NotificationWriterService/Extensions.cs 0 → 100644
View file @ 705de1f3
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
namespace NotificationWriterService
{
public static class Extensions
{
public static void EnqueueList<T>(this ConcurrentQueue<T> queue, IEnumerable<T> items)
{
foreach (var item in items)
{
queue.Enqueue(item);
}
}
public static bool IsDirectory(this FileSystemEventArgs @event)
{
var attr = File.GetAttributes(@event.FullPath);
return attr.HasFlag(FileAttributes.Directory);
}
}
}
NotificationWriterService/FileChangeDetail.cs
View file @ 705de1f3
......@@ -3,7 +3,8 @@ using System.IO;
internal class FileChangeDetail
{
public FileSystemEventArgs GetFileSystemEventArgs { get; set; }
public WatcherChangeTypes ChangeType { get; set; }
public string FullPath { get; set; }
public DateTimeOffset DateTime { get; set; }
public FileChangeDetail()
......
NotificationWriterService/FileWatchManager.cs
View file @ 705de1f3
using NotificationWriterService.Options;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Serilog;
namespace NotificationWriterService
{
internal class FileWatchManager : IFileWatchManager,IDisposable
internal class FileWatchManager : IFileWatchManager
{
const string changeListFolder = "__sc_changelist__";
private readonly TimeSpan _expirationTime = TimeSpan.FromSeconds(1);
private Object _cacheLock = new Object();
const string ChangeListFolder = "__sc_changelist__";
ConcurrentQueue<FileChangeDetail> queue = new ConcurrentQueue<FileChangeDetail>();
private readonly AppOptions _options;
private readonly ILogger _logger;
private FileSystemWatcher _watcher;
private FileSystemWatcher _watcherDirectory;
private IMemoryCache _cache;
public FileWatchManager(AppOptions options, ILogger logger)
public FileWatchManager(AppOptions options, ILogger logger, IMemoryCache cache)
{
_options = options;
_logger = logger;
_watcher = new FileSystemWatcher();
_watcherDirectory = new FileSystemWatcher();
_cache = cache;
}
public Task Run(CancellationToken token)
{
_watcher.NotifyFilter = NotifyFilters.LastWrite;
_watcher.Path = _options.WatchDirectoryPath;
_watcher.NotifyFilter = NotifyFilters.Size | NotifyFilters.FileName | NotifyFilters.DirectoryName;
_watcher.Filter = _options.FileFilter;
//_watcher.NotifyFilter = NotifyFilters.LastWrite;
_watcherDirectory.Path = _watcher.Path = _options.WatchDirectoryPath;
_watcher.NotifyFilter = NotifyFilters.Size | NotifyFilters.FileName;
_watcherDirectory.NotifyFilter = NotifyFilters.DirectoryName;
_watcherDirectory.Filter = _watcher.Filter = _options.FileFilter;
// Add event handlers.
_watcher.Changed += new FileSystemEventHandler(OnChanged);
// _watcher.Created += new FileSystemEventHandler(OnChanged);
_watcher.Deleted += new FileSystemEventHandler(OnChanged);
_watcher.Renamed += new RenamedEventHandler(OnRenamed);
_watcher.IncludeSubdirectories = true;
_watcher.EnableRaisingEvents = true;
_watcher.Changed += OnChanged;
_watcher.Created += OnChanged;
_watcher.Deleted += OnChanged;
_watcher.Renamed += OnRenamed;
_watcherDirectory.IncludeSubdirectories = _watcher.IncludeSubdirectories = true;
_watcherDirectory.EnableRaisingEvents = _watcher.EnableRaisingEvents = true;
_watcher.InternalBufferSize = 16;
_watcherDirectory.Created += (sender, e) =>
{
var dateTime = DateTimeOffset.UtcNow;
var files = GetFileList(e.FullPath).ToList();
queue.EnqueueList(files.Where(q =>
{
lock (_cacheLock)
{
if (_cache.TryGetValue(q, out var _))
{
return false;
}
_cache.Set(q, WatcherChangeTypes.Created, _expirationTime);
}
return true;
}).Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Renamed += (sender, e) =>
{
var files = GetFileList(e.FullPath).ToList(); // check this maybe is better move to ProcessQueue
var dateTime = DateTimeOffset.UtcNow;
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Deleted,
FullPath = q.Replace(e.FullPath, e.OldFullPath),
DateTime = dateTime
}));
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Deleted += (sender, e) =>
{
queue.Enqueue(new FileChangeDetail
{
ChangeType = 0,
FullPath = e.FullPath
});
};
return ProcessQueueAsync(token);
}
......@@ -49,12 +118,22 @@ namespace NotificationWriterService
private void OnChanged(object source, FileSystemEventArgs e)
{
if (e.FullPath.Contains(changeListFolder))
if (e.FullPath.Contains(ChangeListFolder))
return;
lock (_cacheLock)
{
if (_cache.TryGetValue(e.FullPath, out var _))
{
return;
}
_cache.Set(e.FullPath, e.ChangeType, _expirationTime);
}
queue.Enqueue(new FileChangeDetail
{
GetFileSystemEventArgs = e,
ChangeType = e.ChangeType,
FullPath = e.FullPath
});
......@@ -62,20 +141,23 @@ namespace NotificationWriterService
private void OnRenamed(object source, RenamedEventArgs e)
{
if (e.FullPath.Contains(changeListFolder))
if (e.FullPath.Contains(ChangeListFolder))
return;
var dateTime = DateTimeOffset.UtcNow;
queue.Enqueue(new FileChangeDetail
{
GetFileSystemEventArgs = new FileSystemEventArgs(WatcherChangeTypes.Deleted, e.OldFullPath.Replace(e.OldName, string.Empty), e.OldName),
ChangeType = WatcherChangeTypes.Deleted,
FullPath = e.OldFullPath,
DateTime = dateTime
});
queue.Enqueue(new FileChangeDetail
{
GetFileSystemEventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, e.FullPath.Replace(e.Name, string.Empty), e.Name),
ChangeType = WatcherChangeTypes.Created,
FullPath = e.FullPath,
DateTime = dateTime
});
......@@ -101,33 +183,34 @@ namespace NotificationWriterService
var directoryPath = Regex.Replace(completedhexTime.Substring(8, 4), ".{2}", @"$0\");
string firstHexString = completedhexTime.Substring(0, 8);
var firstHexString = completedhexTime.Substring(0, 8);
var num = long.Parse(firstHexString, System.Globalization.NumberStyles.HexNumber);
var content = new string[]
var content = new[]
{
$"{getModifier(fileChangeDetail.GetFileSystemEventArgs.ChangeType)} {hexTime} {Path.GetRelativePath(_options.WatchDirectoryPath,fileChangeDetail.GetFileSystemEventArgs.FullPath).Replace(Path.DirectorySeparatorChar,'/')}"
$"{GetModifier(fileChangeDetail.ChangeType)} {hexTime} {Path.GetRelativePath(_options.WatchDirectoryPath,fileChangeDetail.FullPath).Replace(Path.DirectorySeparatorChar,'/')}"
};
var path = Path.Combine(_options.WatchDirectoryPath, changeListFolder, num.ToString("X"), directoryPath);
var path = Path.Combine(_options.WatchDirectoryPath, ChangeListFolder, num.ToString("X"), directoryPath);
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
File.AppendAllLines($@"{path}{completedhexTime.Substring(12, 2)}", content, Encoding.UTF8);
}catch(Exception e)
}
catch (Exception e)
{
_logger.Error(e, $"Processing {fileChangeDetail.GetFileSystemEventArgs.FullPath} with changeType: {fileChangeDetail.GetFileSystemEventArgs.ChangeType}");
_logger.Error(e, $"Processing {fileChangeDetail?.FullPath} with changeType: {fileChangeDetail?.ChangeType}");
}
}
await Task.Delay(_options.WriteChangesDelay);
}
}
private string getModifier(WatcherChangeTypes changeType)
private string GetModifier(WatcherChangeTypes changeType)
{
switch (changeType)
{
......@@ -136,14 +219,46 @@ namespace NotificationWriterService
return "~";
case WatcherChangeTypes.Deleted:
return "-";
}
throw new Exception($"{nameof(WatcherChangeTypes)} not registered");
return "*-";//directory delete
}
private static IEnumerable<string> GetFileList(string rootFolderPath)
{
var pending = new Queue<string>();
pending.Enqueue(rootFolderPath);
while (pending.Count > 0)
{
rootFolderPath = pending.Dequeue();
string[] tmp;
try
{
tmp = Directory.GetFiles(rootFolderPath);
}
catch (UnauthorizedAccessException)
{
continue;
}
foreach (var t in tmp)
{
yield return t;
}
tmp = Directory.GetDirectories(rootFolderPath);
foreach (var t in tmp)
{
pending.Enqueue(t);
}
}
}
public void Dispose()
{
_watcher.Dispose();
_watcherDirectory.Dispose();
}
}
}
NotificationWriterService/NotificationWriterService.csproj
View file @ 705de1f3
......@@ -5,18 +5,19 @@
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="Serilog" Version="2.5.0" />
<PackageReference Include="Serilog" Version="2.7.1" />
<PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
......
NotificationWriterService/Program.cs
View file @ 705de1f3
......@@ -6,6 +6,7 @@ using Serilog;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace NotificationWriterService
{
......@@ -29,7 +30,10 @@ namespace NotificationWriterService
Log.Information("The app is shutting down.");
t.Wait();// wait for the queue is empty
};
do
{
Task.Delay(100).Wait();
}
while (!t.IsCompleted);
}
......@@ -64,6 +68,8 @@ namespace NotificationWriterService
serviceCollection.AddSingleton<Serilog.ILogger>(Log.Logger);
serviceCollection.AddSingleton<IFileWatchManager, FileWatchManager>();
serviceCollection.AddMemoryCache();
return serviceCollection.BuildServiceProvider();
}
......
WnScNewsAgent.v3.ncrunchsolution.user 0 → 100644
View file @ 705de1f3
<SolutionConfiguration>
<Settings>
<CurrentEngineMode>Run all tests automatically [Global]</CurrentEngineMode>
</Settings>
</SolutionConfiguration>
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment