Commit ccdaea65 authored by Alcides's avatar Alcides
Browse files

Merge branch 'dev' into 'master'

watch service v1

See merge request acceleration-program/WnSCNewsAgent!1
parents b08d5c70 7c407095
/NotificationWriterService/bin
/NotificationWriterService/obj
/.vs
\.idea/\.idea\.WnScNewsAgent/\.idea/
\.idea/\.idea\.WnScNewsAgent/
UpgradeLog\.htm
WnScNewsAgent\.sln\.DotSettings\.user
maria/
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ContentModelStore">
<e p="C:\Users\alcides\.Rider2018.1\system\resharper-host\local\Transient\ReSharperHost\v12\SolutionCaches\_WnScNewsAgent.1544923448.00" t="ExcludeRecursive" />
<e p="C:\Users\alcides\RiderProjects\WnScNewsAgent" t="IncludeFlat">
<e p="NotificationWriterService" t="IncludeRecursive">
<e p="bin" t="ExcludeRecursive" />
<e p="NotificationWriterService.csproj" t="IncludeRecursive" />
<e p="obj" t="ExcludeRecursive" />
<e p="Program.cs" t="Include" />
<e p="Properties" t="Include">
<e p="AssemblyInfo.cs" t="Include" />
</e>
</e>
<e p="packages" t="ExcludeRecursive" />
<e p="WnScNewsAgent.sln" t="IncludeFlat" />
</e>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ContentModelUserStore">
<explicitIncludes />
<explicitExcludes />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/.idea.WnScNewsAgent/riderModule.iml" filepath="$PROJECT_DIR$/.idea/.idea.WnScNewsAgent/riderModule.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="RIDER_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$/../.." />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
namespace NotificationWriterService
{
public static class Extensions
{
public static void EnqueueList<T>(this ConcurrentQueue<T> queue, IEnumerable<T> items)
{
foreach (var item in items)
{
queue.Enqueue(item);
}
}
public static bool IsDirectory(this FileSystemEventArgs @event)
{
var attr = File.GetAttributes(@event.FullPath);
return attr.HasFlag(FileAttributes.Directory);
}
}
}
using System;
using System.IO;
internal class FileChangeDetail
{
public WatcherChangeTypes ChangeType { get; set; }
public string FullPath { get; set; }
public DateTimeOffset DateTime { get; set; }
public FileChangeDetail()
{
DateTime = DateTimeOffset.UtcNow;
}
}
\ No newline at end of file
using NotificationWriterService.Options;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Serilog;
namespace NotificationWriterService
{
internal class FileWatchManager : IFileWatchManager
{
private readonly TimeSpan _expirationTime = TimeSpan.FromSeconds(1);
private Object _cacheLock = new Object();
const string ChangeListFolder = "__sc_changelist__";
ConcurrentQueue<FileChangeDetail> queue = new ConcurrentQueue<FileChangeDetail>();
private readonly AppOptions _options;
private readonly ILogger _logger;
private FileSystemWatcher _watcher;
private FileSystemWatcher _watcherDirectory;
private IMemoryCache _cache;
public FileWatchManager(AppOptions options, ILogger logger, IMemoryCache cache)
{
_options = options;
_logger = logger;
_watcher = new FileSystemWatcher();
_watcherDirectory = new FileSystemWatcher();
_cache = cache;
}
public Task Run(CancellationToken token)
{
//_watcher.NotifyFilter = NotifyFilters.LastWrite;
_watcherDirectory.Path = _watcher.Path = _options.WatchDirectoryPath;
_watcher.NotifyFilter = NotifyFilters.Size | NotifyFilters.FileName;
_watcherDirectory.NotifyFilter = NotifyFilters.DirectoryName;
_watcherDirectory.Filter = _watcher.Filter = _options.FileFilter;
// Add event handlers.
_watcher.Changed += OnChanged;
_watcher.Created += OnChanged;
_watcher.Deleted += OnChanged;
_watcher.Renamed += OnRenamed;
_watcherDirectory.IncludeSubdirectories = _watcher.IncludeSubdirectories = true;
_watcherDirectory.EnableRaisingEvents = _watcher.EnableRaisingEvents = true;
_watcher.InternalBufferSize = 16;
_watcherDirectory.Created += (sender, e) =>
{
var dateTime = DateTimeOffset.UtcNow;
var files = GetFileList(e.FullPath).ToList();
queue.EnqueueList(files.Where(q =>
{
lock (_cacheLock)
{
if (_cache.TryGetValue(q, out var _))
{
return false;
}
_cache.Set(q, WatcherChangeTypes.Created, _expirationTime);
}
return true;
}).Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Renamed += (sender, e) =>
{
var files = GetFileList(e.FullPath).ToList(); // check this maybe is better move to ProcessQueue
var dateTime = DateTimeOffset.UtcNow;
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Deleted,
FullPath = q.Replace(e.FullPath, e.OldFullPath),
DateTime = dateTime
}));
queue.EnqueueList(files.Select(q => new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = q,
DateTime = dateTime
}));
};
_watcherDirectory.Deleted += (sender, e) =>
{
queue.Enqueue(new FileChangeDetail
{
ChangeType = 0,
FullPath = e.FullPath
});
};
return ProcessQueueAsync(token);
}
private void OnChanged(object source, FileSystemEventArgs e)
{
if (e.FullPath.Contains(ChangeListFolder))
return;
lock (_cacheLock)
{
if (_cache.TryGetValue(e.FullPath, out var _))
{
return;
}
_cache.Set(e.FullPath, e.ChangeType, _expirationTime);
}
queue.Enqueue(new FileChangeDetail
{
ChangeType = e.ChangeType,
FullPath = e.FullPath
});
}
private void OnRenamed(object source, RenamedEventArgs e)
{
if (e.FullPath.Contains(ChangeListFolder))
return;
var dateTime = DateTimeOffset.UtcNow;
queue.Enqueue(new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Deleted,
FullPath = e.OldFullPath,
DateTime = dateTime
});
queue.Enqueue(new FileChangeDetail
{
ChangeType = WatcherChangeTypes.Created,
FullPath = e.FullPath,
DateTime = dateTime
});
}
private async Task ProcessQueueAsync(CancellationToken token)
{
while (!token.IsCancellationRequested || !queue.IsEmpty)
{
var count = queue.Count;
for (var i = 0; i < count; i++)
{
FileChangeDetail fileChangeDetail = null;
try
{
if (!queue.TryDequeue(out fileChangeDetail))
continue;
var hexTime = fileChangeDetail.DateTime.ToUnixTimeSeconds().ToString("X");
var completedhexTime = hexTime.PadLeft(16, '0');
var directoryPath = Regex.Replace(completedhexTime.Substring(8, 4), ".{2}", @"$0\");
var firstHexString = completedhexTime.Substring(0, 8);
var num = long.Parse(firstHexString, System.Globalization.NumberStyles.HexNumber);
var content = new[]
{
$"{GetModifier(fileChangeDetail.ChangeType)} {hexTime} {Path.GetRelativePath(_options.WatchDirectoryPath,fileChangeDetail.FullPath).Replace(Path.DirectorySeparatorChar,'/')}"
};
var path = Path.Combine(_options.WatchDirectoryPath, ChangeListFolder, num.ToString("X"), directoryPath);
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
File.AppendAllLines($@"{path}{completedhexTime.Substring(12, 2)}", content, Encoding.UTF8);
}
catch (Exception e)
{
_logger.Error(e, $"Processing {fileChangeDetail?.FullPath} with changeType: {fileChangeDetail?.ChangeType}");
}
}
await Task.Delay(_options.WriteChangesDelay);
}
}
private string GetModifier(WatcherChangeTypes changeType)
{
switch (changeType)
{
case WatcherChangeTypes.Created:
case WatcherChangeTypes.Changed:
return "~";
case WatcherChangeTypes.Deleted:
return "-";
}
// return "*-";//directory delete
return "~"; // Not the correct mark for directory delete, but it
// won't cause problems upstream...
}
private static IEnumerable<string> GetFileList(string rootFolderPath)
{
var pending = new Queue<string>();
pending.Enqueue(rootFolderPath);
while (pending.Count > 0)
{
rootFolderPath = pending.Dequeue();
string[] tmp;
try
{
tmp = Directory.GetFiles(rootFolderPath);
}
catch (UnauthorizedAccessException)
{
continue;
}
foreach (var t in tmp)
{
yield return t;
}
tmp = Directory.GetDirectories(rootFolderPath);
foreach (var t in tmp)
{
pending.Enqueue(t);
}
}
}
public void Dispose()
{
_watcher.Dispose();
_watcherDirectory.Dispose();
}
}
}
using System;
using System.Threading;
using System.Threading.Tasks;
namespace NotificationWriterService
{
internal interface IFileWatchManager:IDisposable
{
Task Run(CancellationToken token);
}
}
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props"
Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')"/>
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{93BA2042-C559-4FAC-A798-59AEFA4953EA}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>NotificationWriterService</RootNamespace>
<AssemblyName>NotificationWriterService</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System"/>
<Reference Include="System.Core"/>
<Reference Include="System.Data"/>
<Reference Include="System.Xml"/>
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs"/>
<Compile Include="Properties\AssemblyInfo.cs"/>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets"/>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="Serilog" Version="2.7.1" />
<PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
using System;
using System.Collections.Generic;
using System.Text;
namespace NotificationWriterService.Options
{
class AppOptions
{
public string WatchDirectoryPath { get; set; }
public int WriteChangesDelay { get; set; }
public string FileFilter { get; set; }
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using NotificationWriterService.Options;
using Serilog;
using System;
using System.IO;
using System.Security.Permissions;
using System.Threading;
using System.Threading.Tasks;
public class Watcher
namespace NotificationWriterService
{
public static void Main()
//to use https://nssm.cc/ to install as a service in windows
class Program
{
Run();
static void Main(string[] args)
{
var source = new CancellationTokenSource();
var token = source.Token;
var serviceProvider = ConfigureServices();
}
var fileWatchManager = serviceProvider.GetRequiredService<IFileWatchManager>();
[PermissionSet(SecurityAction.Demand, Name="FullTrust")]
public static void Run()
{
string[] args = System.Environment.GetCommandLineArgs();
var t = fileWatchManager.Run(token);
Log.Information("WnSCNewsAgent is running");
Console.CancelKeyPress += (s, e) => { // https://nssm.cc/ trigger this event when the service is stoped
source.Cancel();
Log.Information("The app is shutting down.");
t.Wait();// wait for the queue is empty
Log.Information("Finished.");
};
do
{
Task.Delay(100).Wait();
}
while (!t.IsCompleted);
}
// If a directory is not specified, exit program.
if(args.Length != 2)
private static IServiceProvider ConfigureServices()
{
// Display the proper way to call the program.
Console.WriteLine("Usage: Watcher.exe (directory)");
return;
}
var serviceCollection = new ServiceCollection();
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetParent(AppContext.BaseDirectory).FullName)
.AddJsonFile("appsettings.json", false)
.Build();
// Create a new FileSystemWatcher and set its properties.
FileSystemWatcher watcher = new FileSystemWatcher();
watcher.Path = args[1];
/* Watch for changes in LastAccess and LastWrite times, and
the renaming of files or directories. */
watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite
| NotifyFilters.FileName | NotifyFilters.DirectoryName;
// Only watch text files.
watcher.Filter = "*.txt";
// Add event handlers.
watcher.Changed += new FileSystemEventHandler(OnChanged);
watcher.Created += new FileSystemEventHandler(OnChanged);
watcher.Deleted += new FileSystemEventHandler(OnChanged);
watcher.Renamed += new RenamedEventHandler(OnRenamed);
// Begin watching.
watcher.EnableRaisingEvents = true;
// Wait for the user to quit the program.
Console.WriteLine("Press \'q\' to quit the sample.");
while(Console.Read()!='q');
}
// Add logging
serviceCollection.AddSingleton(new LoggerFactory()
.AddConsole()
.AddSerilog()
.AddDebug());
serviceCollection.AddLogging();
// Define the event handlers.
private static void OnChanged(object source, FileSystemEventArgs e)
{
// Specify what is done when a file is changed, created, or deleted.
Console.WriteLine("File: " + e.FullPath + " " + e.ChangeType);
}
var options = configuration.Get<AppOptions>();
private static void OnRenamed(object source, RenamedEventArgs e)
{
// Specify what is done when a file is renamed.
Console.WriteLine("File: {0} renamed to {1}", e.OldFullPath, e.FullPath);
// Build configuration
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()