Commit b77add75 authored by Alcides Viamontes E's avatar Alcides Viamontes E
Browse files

Merge remote-tracking branch 'origin/dev' into dev

parents 85605226 705de1f3
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
namespace NotificationWriterService
{
public static class Extensions
{
public static void EnqueueList<T>(this ConcurrentQueue<T> queue, IEnumerable<T> items)
{
foreach (var item in items)
{
queue.Enqueue(item);
}
}
public static bool IsDirectory(this FileSystemEventArgs @event)
{
var attr = File.GetAttributes(@event.FullPath);
return attr.HasFlag(FileAttributes.Directory);
}
}
}
...@@ -3,7 +3,8 @@ using System.IO; ...@@ -3,7 +3,8 @@ using System.IO;
internal class FileChangeDetail internal class FileChangeDetail
{ {
public FileSystemEventArgs GetFileSystemEventArgs { get; set; } public WatcherChangeTypes ChangeType { get; set; }
public string FullPath { get; set; }
public DateTimeOffset DateTime { get; set; } public DateTimeOffset DateTime { get; set; }
public FileChangeDetail() public FileChangeDetail()
......
using NotificationWriterService.Options; using NotificationWriterService.Options;
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Serilog; using Serilog;
namespace NotificationWriterService namespace NotificationWriterService
{ {
internal class FileWatchManager : IFileWatchManager,IDisposable internal class FileWatchManager : IFileWatchManager
{ {
const string changeListFolder = "__sc_changelist__";
private readonly TimeSpan _expirationTime = TimeSpan.FromSeconds(1);
private Object _cacheLock = new Object();
const string ChangeListFolder = "__sc_changelist__";
ConcurrentQueue<FileChangeDetail> queue = new ConcurrentQueue<FileChangeDetail>(); ConcurrentQueue<FileChangeDetail> queue = new ConcurrentQueue<FileChangeDetail>();
private readonly AppOptions _options; private readonly AppOptions _options;
private readonly ILogger _logger; private readonly ILogger _logger;
private FileSystemWatcher _watcher; private FileSystemWatcher _watcher;
private FileSystemWatcher _watcherDirectory;
private IMemoryCache _cache;
public FileWatchManager(AppOptions options, ILogger logger) public FileWatchManager(AppOptions options, ILogger logger, IMemoryCache cache)
{ {
_options = options; _options = options;
_logger = logger; _logger = logger;
_watcher = new FileSystemWatcher(); _watcher = new FileSystemWatcher();
_watcherDirectory = new FileSystemWatcher();
_cache = cache;
} }
public Task Run(CancellationToken token) public Task Run(CancellationToken token)
{ {
_watcher.NotifyFilter = NotifyFilters.LastWrite; //_watcher.NotifyFilter = NotifyFilters.LastWrite;
_watcher.Path = _options.WatchDirectoryPath; _watcherDirectory.Path = _watcher.Path = _options.WatchDirectoryPath;
_watcher.NotifyFilter = NotifyFilters.Size | NotifyFilters.FileName | NotifyFilters.DirectoryName; _watcher.NotifyFilter = NotifyFilters.Size | NotifyFilters.FileName;
_watcher.Filter = _options.FileFilter; _watcherDirectory.NotifyFilter = NotifyFilters.DirectoryName;
_watcherDirectory.Filter = _watcher.Filter = _options.FileFilter;
// Add event handlers. // Add event handlers.
_watcher.Changed += new FileSystemEventHandler(OnChanged); _watcher.Changed += OnChanged;
// _watcher.Created += new FileSystemEventHandler(OnChanged); _watcher.Created += OnChanged;
_watcher.Deleted += new FileSystemEventHandler(OnChanged); _watcher.Deleted += OnChanged;
_watcher.Renamed += new RenamedEventHandler(OnRenamed); _watcher.Renamed += OnRenamed;
_watcher.IncludeSubdirectories = true; _watcherDirectory.IncludeSubdirectories = _watcher.IncludeSubdirectories = true;
_watcher.EnableRaisingEvents = true; _watcherDirectory.EnableRaisingEvents = _watcher.EnableRaisingEvents = true;
_watcher.InternalBufferSize = 16;
_watcherDirectory.Created += (sender, e) =>
{
var dateTime = DateTimeOffset.UtcNow;
var files = GetFileList(e.FullPath).ToList();
queue.EnqueueList(files.Where(q =>
{
lock (_cacheLock)
{
if (_cache.TryGetValue(q, out var _))
{
return false;
}
_cache.Set(q, WatcherChangeTypes.Created, _expirationTime);
}
return true;
}).Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Renamed += (sender, e) =>
{
var files = GetFileList(e.FullPath).ToList(); // check this maybe is better move to ProcessQueue
var dateTime = DateTimeOffset.UtcNow;
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Deleted,
FullPath = q.Replace(e.FullPath, e.OldFullPath),
DateTime = dateTime
}));
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Deleted += (sender, e) =>
{
queue.Enqueue(new FileChangeDetail
{
ChangeType = 0,
FullPath = e.FullPath
});
};
return ProcessQueueAsync(token); return ProcessQueueAsync(token);
} }
...@@ -49,12 +118,22 @@ namespace NotificationWriterService ...@@ -49,12 +118,22 @@ namespace NotificationWriterService
private void OnChanged(object source, FileSystemEventArgs e) private void OnChanged(object source, FileSystemEventArgs e)
{ {
if (e.FullPath.Contains(changeListFolder)) if (e.FullPath.Contains(ChangeListFolder))
return; return;
lock (_cacheLock)
{
if (_cache.TryGetValue(e.FullPath, out var _))
{
return;
}
_cache.Set(e.FullPath, e.ChangeType, _expirationTime);
}
queue.Enqueue(new FileChangeDetail queue.Enqueue(new FileChangeDetail
{ {
GetFileSystemEventArgs = e, ChangeType = e.ChangeType,
FullPath = e.FullPath
}); });
...@@ -62,20 +141,23 @@ namespace NotificationWriterService ...@@ -62,20 +141,23 @@ namespace NotificationWriterService
private void OnRenamed(object source, RenamedEventArgs e) private void OnRenamed(object source, RenamedEventArgs e)
{ {
if (e.FullPath.Contains(changeListFolder)) if (e.FullPath.Contains(ChangeListFolder))
return; return;
var dateTime = DateTimeOffset.UtcNow; var dateTime = DateTimeOffset.UtcNow;
queue.Enqueue(new FileChangeDetail queue.Enqueue(new FileChangeDetail
{ {
GetFileSystemEventArgs = new FileSystemEventArgs(WatcherChangeTypes.Deleted, e.OldFullPath.Replace(e.OldName, string.Empty), e.OldName), ChangeType = WatcherChangeTypes.Deleted,
FullPath = e.OldFullPath,
DateTime = dateTime DateTime = dateTime
}); });
queue.Enqueue(new FileChangeDetail queue.Enqueue(new FileChangeDetail
{ {
GetFileSystemEventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, e.FullPath.Replace(e.Name, string.Empty), e.Name), ChangeType = WatcherChangeTypes.Created,
FullPath = e.FullPath,
DateTime = dateTime DateTime = dateTime
}); });
...@@ -101,33 +183,34 @@ namespace NotificationWriterService ...@@ -101,33 +183,34 @@ namespace NotificationWriterService
var directoryPath = Regex.Replace(completedhexTime.Substring(8, 4), ".{2}", @"$0\"); var directoryPath = Regex.Replace(completedhexTime.Substring(8, 4), ".{2}", @"$0\");
string firstHexString = completedhexTime.Substring(0, 8); var firstHexString = completedhexTime.Substring(0, 8);
var num = long.Parse(firstHexString, System.Globalization.NumberStyles.HexNumber); var num = long.Parse(firstHexString, System.Globalization.NumberStyles.HexNumber);
var content = new string[] var content = new[]
{ {
$"{getModifier(fileChangeDetail.GetFileSystemEventArgs.ChangeType)} {hexTime} {Path.GetRelativePath(_options.WatchDirectoryPath,fileChangeDetail.GetFileSystemEventArgs.FullPath).Replace(Path.DirectorySeparatorChar,'/')}" $"{GetModifier(fileChangeDetail.ChangeType)} {hexTime} {Path.GetRelativePath(_options.WatchDirectoryPath,fileChangeDetail.FullPath).Replace(Path.DirectorySeparatorChar,'/')}"
}; };
var path = Path.Combine(_options.WatchDirectoryPath, changeListFolder, num.ToString("X"), directoryPath); var path = Path.Combine(_options.WatchDirectoryPath, ChangeListFolder, num.ToString("X"), directoryPath);
if (!Directory.Exists(path)) if (!Directory.Exists(path))
Directory.CreateDirectory(path); Directory.CreateDirectory(path);
File.AppendAllLines($@"{path}{completedhexTime.Substring(12, 2)}", content, Encoding.UTF8); File.AppendAllLines($@"{path}{completedhexTime.Substring(12, 2)}", content, Encoding.UTF8);
}catch(Exception e) }
catch (Exception e)
{ {
_logger.Error(e, $"Processing {fileChangeDetail.GetFileSystemEventArgs.FullPath} with changeType: {fileChangeDetail.GetFileSystemEventArgs.ChangeType}"); _logger.Error(e, $"Processing {fileChangeDetail?.FullPath} with changeType: {fileChangeDetail?.ChangeType}");
} }
} }
await Task.Delay(_options.WriteChangesDelay); await Task.Delay(_options.WriteChangesDelay);
} }
} }
private string getModifier(WatcherChangeTypes changeType) private string GetModifier(WatcherChangeTypes changeType)
{ {
switch (changeType) switch (changeType)
{ {
...@@ -136,14 +219,46 @@ namespace NotificationWriterService ...@@ -136,14 +219,46 @@ namespace NotificationWriterService
return "~"; return "~";
case WatcherChangeTypes.Deleted: case WatcherChangeTypes.Deleted:
return "-"; return "-";
} }
throw new Exception($"{nameof(WatcherChangeTypes)} not registered"); return "*-";//directory delete
}
private static IEnumerable<string> GetFileList(string rootFolderPath)
{
var pending = new Queue<string>();
pending.Enqueue(rootFolderPath);
while (pending.Count > 0)
{
rootFolderPath = pending.Dequeue();
string[] tmp;
try
{
tmp = Directory.GetFiles(rootFolderPath);
}
catch (UnauthorizedAccessException)
{
continue;
}
foreach (var t in tmp)
{
yield return t;
}
tmp = Directory.GetDirectories(rootFolderPath);
foreach (var t in tmp)
{
pending.Enqueue(t);
}
}
} }
public void Dispose() public void Dispose()
{ {
_watcher.Dispose(); _watcher.Dispose();
_watcherDirectory.Dispose();
} }
} }
} }
...@@ -5,18 +5,19 @@ ...@@ -5,18 +5,19 @@
<TargetFramework>netcoreapp2.0</TargetFramework> <TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.0.0" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" /> <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="Serilog" Version="2.5.0" /> <PackageReference Include="Serilog" Version="2.7.1" />
<PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" /> <PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0" /> <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
......
...@@ -6,6 +6,7 @@ using Serilog; ...@@ -6,6 +6,7 @@ using Serilog;
using System; using System;
using System.IO; using System.IO;
using System.Threading; using System.Threading;
using System.Threading.Tasks;
namespace NotificationWriterService namespace NotificationWriterService
{ {
...@@ -29,7 +30,10 @@ namespace NotificationWriterService ...@@ -29,7 +30,10 @@ namespace NotificationWriterService
Log.Information("The app is shutting down."); Log.Information("The app is shutting down.");
t.Wait();// wait for the queue is empty t.Wait();// wait for the queue is empty
}; };
do
{
Task.Delay(100).Wait();
}
while (!t.IsCompleted); while (!t.IsCompleted);
} }
...@@ -64,6 +68,8 @@ namespace NotificationWriterService ...@@ -64,6 +68,8 @@ namespace NotificationWriterService
serviceCollection.AddSingleton<Serilog.ILogger>(Log.Logger); serviceCollection.AddSingleton<Serilog.ILogger>(Log.Logger);
serviceCollection.AddSingleton<IFileWatchManager, FileWatchManager>(); serviceCollection.AddSingleton<IFileWatchManager, FileWatchManager>();
serviceCollection.AddMemoryCache();
return serviceCollection.BuildServiceProvider(); return serviceCollection.BuildServiceProvider();
} }
......
<SolutionConfiguration>
<Settings>
<CurrentEngineMode>Run all tests automatically [Global]</CurrentEngineMode>
</Settings>
</SolutionConfiguration>
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment