From 6c2ce2133e5c2289a5c62f0ae502c7578a533edb Mon Sep 17 00:00:00 2001 From: femsci Date: Tue, 22 Aug 2023 15:06:00 +0200 Subject: [PATCH] Spaghetti code. --- .gitignore | 2 + Dockerfile | 38 ++++++ Pacmeow.sln | 27 ++++ src/Nyanbyte.Pacmeow.Server/Data/Mirror.cs | 23 ++++ src/Nyanbyte.Pacmeow.Server/Data/Package.cs | 15 ++ .../Data/Repository.cs | 11 ++ src/Nyanbyte.Pacmeow.Server/LoggingService.cs | 14 ++ .../Nyanbyte.Pacmeow.Server.csproj | 9 ++ src/Nyanbyte.Pacmeow.Server/PackageService.cs | 129 ++++++++++++++++++ src/Nyanbyte.Pacmeow.Server/PacmeowOptions.cs | 15 ++ src/Nyanbyte.Pacmeow.Server/Program.cs | 47 +++++++ .../Properties/launchSettings.json | 37 +++++ .../RepositoryService.cs | 49 +++++++ .../appsettings.Development.json | 8 ++ src/Nyanbyte.Pacmeow.Server/appsettings.json | 9 ++ src/Nyanbyte.Pacmeow.Server/config.json | 7 + 16 files changed, 440 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Pacmeow.sln create mode 100644 src/Nyanbyte.Pacmeow.Server/Data/Mirror.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/Data/Package.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/Data/Repository.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/LoggingService.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/Nyanbyte.Pacmeow.Server.csproj create mode 100644 src/Nyanbyte.Pacmeow.Server/PackageService.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/PacmeowOptions.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/Program.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/Properties/launchSettings.json create mode 100644 src/Nyanbyte.Pacmeow.Server/RepositoryService.cs create mode 100644 src/Nyanbyte.Pacmeow.Server/appsettings.Development.json create mode 100644 src/Nyanbyte.Pacmeow.Server/appsettings.json create mode 100644 src/Nyanbyte.Pacmeow.Server/config.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5967294 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/bin/ +**/obj/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..eb08aa4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM mcr.microsoft.com/dotnet/sdk:7.0-alpine AS build + +WORKDIR /source + +COPY Pacmeow.sln ./ + +COPY src/Nyanbyte.Pacmeow.Server/*.csproj ./src/Nyanbyte.Pacmeow.Server/ + +RUN dotnet restore + +COPY src/Nyanbyte.Pacmeow.Server/. ./src/Nyanbyte.Pacmeow.Server/ + +RUN dotnet publish ./src/Nyanbyte.Pacmeow.Server/Nyanbyte.Pacmeow.Server.csproj -c Release -o /build/pacmeow + +FROM mcr.microsoft.com/dotnet/aspnet:7.0-alpine AS runtime + +# Environment vars + +RUN mkdir -p /srv/certs /srv/log + +# Files + +COPY --from=build /build/pacmeow /srv/pacmeow + +VOLUME [ "/srv/pacmeow", "/srv/certs", "/srv/log", "/var/cache/pacmeow" ] + +WORKDIR /srv/pacmeow + +# Net + +EXPOSE 80/tcp +EXPOSE 443/tcp + +# Process handling + +STOPSIGNAL SIGTERM + +ENTRYPOINT [ "dotnet", "/srv/pacmeow/Nyanbyte.Pacmeow.Server.dll" ] diff --git a/Pacmeow.sln b/Pacmeow.sln new file mode 100644 index 0000000..559e00e --- /dev/null +++ b/Pacmeow.sln @@ -0,0 +1,27 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4BC3F182-A882-4BD7-BB87-5A63E4428D5A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nyanbyte.Pacmeow.Server", "src\Nyanbyte.Pacmeow.Server\Nyanbyte.Pacmeow.Server.csproj", "{7993A4C3-1034-40CB-A7EB-D7833E24973C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7993A4C3-1034-40CB-A7EB-D7833E24973C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7993A4C3-1034-40CB-A7EB-D7833E24973C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7993A4C3-1034-40CB-A7EB-D7833E24973C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7993A4C3-1034-40CB-A7EB-D7833E24973C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {7993A4C3-1034-40CB-A7EB-D7833E24973C} = {4BC3F182-A882-4BD7-BB87-5A63E4428D5A} + EndGlobalSection +EndGlobal diff --git a/src/Nyanbyte.Pacmeow.Server/Data/Mirror.cs b/src/Nyanbyte.Pacmeow.Server/Data/Mirror.cs new file mode 100644 index 0000000..9108452 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Data/Mirror.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Threading.Tasks; + +namespace Nyanbyte.Pacmeow.Server.Data; + +public record Mirror +{ + public required string Platform { get; set; } + public required string Address { get; set; } + + public Uri GetUri(string repo, string arch) + { + if (!Address.Contains("$repo") || !Address.Contains("$arch")) + { + throw new DataException($"Invalid server: {Address}"); + } + + return new Uri(Address.Replace("$repo", repo).Replace("$arch", arch)); + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/Data/Package.cs b/src/Nyanbyte.Pacmeow.Server/Data/Package.cs new file mode 100644 index 0000000..e57a5ae --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Data/Package.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Nyanbyte.Pacmeow.Server.Data; + +public record Package +{ + public required string Name { get; set; } + public required string Repo { get; set; } + public required string Arch { get; set; } + public required string Platform { get; set; } + +} diff --git a/src/Nyanbyte.Pacmeow.Server/Data/Repository.cs b/src/Nyanbyte.Pacmeow.Server/Data/Repository.cs new file mode 100644 index 0000000..319d8d2 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Data/Repository.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Nyanbyte.Pacmeow.Server.Data; + +public record Repository +{ + +} diff --git a/src/Nyanbyte.Pacmeow.Server/LoggingService.cs b/src/Nyanbyte.Pacmeow.Server/LoggingService.cs new file mode 100644 index 0000000..05a3b23 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/LoggingService.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Nyanbyte.Pacmeow.Server; + +public class LoggingService +{ + public void LogFileDelete(string path, string reason) + { + + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/Nyanbyte.Pacmeow.Server.csproj b/src/Nyanbyte.Pacmeow.Server/Nyanbyte.Pacmeow.Server.csproj new file mode 100644 index 0000000..4c2bb77 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Nyanbyte.Pacmeow.Server.csproj @@ -0,0 +1,9 @@ + + + + net7.0 + enable + enable + + + diff --git a/src/Nyanbyte.Pacmeow.Server/PackageService.cs b/src/Nyanbyte.Pacmeow.Server/PackageService.cs new file mode 100644 index 0000000..2494c55 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/PackageService.cs @@ -0,0 +1,129 @@ +using System.Buffers; +using System.Data; +using System.IO.Pipelines; +using System.Net; +using System.Net.Http.Headers; +using Microsoft.Extensions.Options; +using Nyanbyte.Pacmeow.Server.Data; + +namespace Nyanbyte.Pacmeow.Server; + +public class PackageService : IDisposable +{ + public PackageService(IOptions opts, LoggingService log, RepositoryService rep) + { + _config = opts.Value; + _log = log; + _rep = rep; + if (opts.Value.PruneTime > 0) + { + _timer = new(PurgeCache, null, TimeSpan.Zero, TimeSpan.FromHours(1)); + } + else _timer = null!; + var handler = new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(15) // Recreate every 15 minutes + }; + cli = new HttpClient(handler); + cli.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Pacmeow", "1.0e")); + } + + private PacmeowOptions _config; + private readonly Timer _timer; + private readonly LoggingService _log; + private readonly RepositoryService _rep; + private readonly HttpClient cli; + + public async Task TryDownloadPackage(Package pkg, HttpContext ctx) + { + Console.WriteLine($"pkg: {pkg.Arch}"); + + foreach (var mirror in _rep.GetMirrors(pkg.Platform)) + { + UriBuilder uri = new(mirror.GetUri(pkg.Repo, pkg.Arch)); + uri.Path = $"{uri.Path.TrimEnd('/')}/{pkg.Name}"; + + Console.WriteLine($"Trying {uri}"); + + //Console.WriteLine($"{uri.Host} :=> {string.Join(", ", (await Dns.GetHostAddressesAsync(uri.Host)).Select(x => x.ToString()))}"); + + var response = await cli.GetAsync(uri.Uri, HttpCompletionOption.ResponseHeadersRead); + + Console.WriteLine("Response..."); + + if (!response.IsSuccessStatusCode) + { + Console.WriteLine($"Cannot retrieve {pkg.Name} from {uri.Host}. Status code: {response.StatusCode}"); + continue; + } + + Console.WriteLine($"Status: {response.StatusCode}, len: {response.Content.Headers.ContentLength}"); + + ctx.Response.StatusCode = 200; + ctx.Response.ContentLength = response.Content.Headers.ContentLength ?? 0; + ctx.Response.ContentType = response.Content.Headers.ContentType?.ToString(); + await ctx.Response.StartAsync(); + + int len = (int)(response.Content.Headers.ContentLength ?? 0); + var stream = await response.Content.ReadAsStreamAsync(); + Console.WriteLine($"Stream read."); + + using var fs = File.OpenWrite(Path.Combine(_config.CacheDirectory, "repo", pkg.Platform, pkg.Name)); + var buf = ArrayPool.Shared.Rent(32768); + + long total = 0; + int read; + while ((read = stream.Read(buf, 0, buf.Length)) > 0) + { + total += read; + //Console.WriteLine($"Progress: {total}/{len}"); + await ctx.Response.Body.WriteAsync(buf, 0, read); + await ctx.Response.Body.FlushAsync(); + await fs.WriteAsync(buf, 0, read); + } + + ArrayPool.Shared.Return(buf); + + fs.Close(); + + return true; + } + + ctx.Response.StatusCode = 404; + return false; + } + + public async Task TryWritePackage(Package pkg, HttpContext ctx) + { + string path = Path.Combine(_config.CacheDirectory, "repo", pkg.Platform, pkg.Name); + + if (File.Exists(path)) + { + ctx.Response.StatusCode = 200; + await ctx.Response.StartAsync(); + await File.OpenRead(path).CopyToAsync(ctx.Response.Body); + return true; + } + + return await TryDownloadPackage(pkg, ctx); + } + + private void PurgeCache(object? _) + { + DateTime now = DateTime.UtcNow; + + foreach (var repo in Directory.EnumerateDirectories(Path.Combine(_config.CacheDirectory, "repo"))) + { + foreach (var file in Directory.EnumerateFiles(repo).Where(f => (now - File.GetLastWriteTimeUtc(f)).TotalSeconds > _config.PruneTime)) + { + File.Delete(file); + _log.LogFileDelete(file, "File expired"); + } + } + } + + public void Dispose() + { + _timer?.Dispose(); + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/PacmeowOptions.cs b/src/Nyanbyte.Pacmeow.Server/PacmeowOptions.cs new file mode 100644 index 0000000..9885b77 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/PacmeowOptions.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Threading.Tasks; + +namespace Nyanbyte.Pacmeow.Server; + +public class PacmeowOptions +{ + public string CacheDirectory { get; set; } = "/tmp/cache/pacmeow"; // "/var/cache/pacmeow"; + [Range(0, 3600 * 24 * 100)] + public int PruneTime { get; set; } = 0; + public required Dictionary> Mirrors { get; set; } +} diff --git a/src/Nyanbyte.Pacmeow.Server/Program.cs b/src/Nyanbyte.Pacmeow.Server/Program.cs new file mode 100644 index 0000000..8a6ff78 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Program.cs @@ -0,0 +1,47 @@ +using System.Net; +using System.Text.Json; +using Microsoft.AspNetCore.Http.HttpResults; +using Nyanbyte.Pacmeow.Server; +using Nyanbyte.Pacmeow.Server.Data; + + +var builder = WebApplication.CreateBuilder(args); +builder.Configuration.AddJsonFile("./config.json", false); +builder.Services.Configure(builder.Configuration); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +var app = builder.Build(); + +app.MapGet("/version", () => "1.0"); +app.MapGet("/repo/{platform}/{repo}/os/{arch}/{name}", async (HttpContext ctx, PackageService pkg, string name, string platform, string repo, string arch) => +{ + if (name.EndsWith(".db.sig")) + { + ctx.Response.StatusCode = 404; + await ctx.Response.CompleteAsync(); + return; + } + + Package p = new() + { + Arch = arch, + Repo = repo, + Name = name, + Platform = platform + }; + + + var result = await pkg.TryWritePackage(p, ctx); + if (result) + { + await ctx.Response.CompleteAsync(); + } + else + { + ctx.Response.StatusCode = 404; + await ctx.Response.CompleteAsync(); + } +}); + +app.Run(); diff --git a/src/Nyanbyte.Pacmeow.Server/Properties/launchSettings.json b/src/Nyanbyte.Pacmeow.Server/Properties/launchSettings.json new file mode 100644 index 0000000..82b85da --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/Properties/launchSettings.json @@ -0,0 +1,37 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:43446", + "sslPort": 44393 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5248", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7113;http://localhost:5248", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/RepositoryService.cs b/src/Nyanbyte.Pacmeow.Server/RepositoryService.cs new file mode 100644 index 0000000..593ea40 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/RepositoryService.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Nyanbyte.Pacmeow.Server.Data; + +namespace Nyanbyte.Pacmeow.Server; + +public class RepositoryService +{ + public RepositoryService(IOptions opt) + { + _opt = opt.Value; + _mirrors = _opt.Mirrors.SelectMany(x => + x.Value.Select(m => + new Mirror() + { + Address = m, + Platform = x.Key + } + ) + ).ToList(); + Console.WriteLine($"{_opt.PruneTime}, {_opt.CacheDirectory}, {_opt.Mirrors.Count}"); + Console.WriteLine(string.Join(", ", _mirrors.Select(x => x.Address))); + + foreach (var mirror in _mirrors) + { + var path = Path.Combine(_opt.CacheDirectory, "repo", mirror.Platform); + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + } + } + + private readonly PacmeowOptions _opt; + private readonly List _mirrors = new(); + + public IEnumerable GetMirrors(string platform) + { + return _mirrors.Where(m => m.Platform == platform); + } + + public IEnumerable GetAll() + { + return _mirrors.AsEnumerable(); + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/appsettings.Development.json b/src/Nyanbyte.Pacmeow.Server/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/Nyanbyte.Pacmeow.Server/appsettings.json b/src/Nyanbyte.Pacmeow.Server/appsettings.json new file mode 100644 index 0000000..10f68b8 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/src/Nyanbyte.Pacmeow.Server/config.json b/src/Nyanbyte.Pacmeow.Server/config.json new file mode 100644 index 0000000..7f11f53 --- /dev/null +++ b/src/Nyanbyte.Pacmeow.Server/config.json @@ -0,0 +1,7 @@ +{ + "cacheDirectory": "/var/cache/pacmeow", + "pruneTime": 518400, + "mirrors": { + "arch": ["https://mirror.selfnet.de/archlinux/$repo/os/$arch"] + } +}