Skip to content

Commit c29f260

Browse files
VanderCatVanderCat
committed
[FEAT] Compressing CLI embedded in library itself
1 parent 26a39f4 commit c29f260

6 files changed

Lines changed: 248 additions & 23 deletions

File tree

‎NekoLib.Archive/Compression/ZstdProvider.cs‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ namespace NekoLib.Archive;
55

66
[CompressionId("ZSTD")]
77
public class ZstdProvider() : IDecompressor, ICompressor, IDisposable {
8-
private Decompressor Decompressor = new();
9-
private Compressor Compressor = new();
8+
private Decompressor _decompressor = new();
9+
private Compressor _compressor = new();
1010
public int DictCapacity = 112640;
1111
public byte[]? Dict;
1212
private int _compressionLevel = Compressor.DefaultCompressionLevel;
@@ -21,37 +21,38 @@ public int CompressionLevel {
2121
if (value > Compressor.MaxCompressionLevel || value < Compressor.MinCompressionLevel)
2222
value = Compressor.DefaultCompressionLevel;
2323
_compressionLevel = value;
24-
Compressor.SetParameter(ZSTD_cParameter.ZSTD_c_compressionLevel, CompressionLevel);
24+
_compressor.SetParameter(ZSTD_cParameter.ZSTD_c_compressionLevel, CompressionLevel);
2525
}
2626
}
2727

2828
public void LoadData(ReadOnlySpan<byte> data) {
29-
Decompressor.LoadDictionary(data);
30-
Compressor.LoadDictionary(data);
29+
if (data.Length <= 0) return;
30+
_decompressor.LoadDictionary(data);
31+
_compressor.LoadDictionary(data);
3132
}
3233

3334
public Span<byte> Decompress(ReadOnlySpan<byte> data, int maxsize) {
34-
return Decompressor.Unwrap(data, maxsize);
35+
return _decompressor.Unwrap(data, maxsize);
3536
}
3637

3738
public void Decompress(ReadOnlySpan<byte> data, Span<byte> dest) {
38-
Decompressor.Unwrap(data, dest);
39+
_decompressor.Unwrap(data, dest);
3940
}
4041

4142
public Span<byte> Compress(ReadOnlySpan<byte> data) {
42-
return Compressor.Wrap(data);
43+
return _compressor.Wrap(data);
4344
}
4445

4546
public void Compress(ReadOnlySpan<byte> data, Span<byte> dest) {
46-
Compressor.Wrap(data, dest);
47+
_compressor.Wrap(data, dest);
4748
}
4849

4950
public void Dispose() {
50-
Decompressor.Dispose();
51-
Compressor.Dispose();
51+
_decompressor.Dispose();
52+
_compressor.Dispose();
5253
}
5354

54-
public bool SupportsTraining => true;
55+
public bool SupportsTraining => DictCapacity > 0;
5556

5657
public void Train(IEnumerable<byte[]> data) {
5758
var span = DictBuilder.TrainFromBufferFastCover(data, _compressionLevel, DictCapacity);
@@ -68,6 +69,6 @@ public byte[] GetTrainData() {
6869
public bool SupportsStreaming => true;
6970

7071
public Stream GetDecompressionStream(Stream stream) {
71-
return new DecompressionStream(stream, Decompressor);
72+
return new DecompressionStream(stream, _decompressor);
7273
}
7374
}

‎NekoLib.Archive/MiniCli.cs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
using System.Dynamic;
2+
using System.Reflection;
3+
using System.Text;
4+
5+
namespace NekoLib.Archive;
6+
7+
internal static class MiniCli {
8+
public class MiniNameAttribute(string name) : Attribute {
9+
public string Name = name;
10+
}
11+
public class SkipAttribute : Attribute { }
12+
13+
public class HelpAttribute(string text) : Attribute {
14+
public string Text = text;
15+
}
16+
public static T Parse<T>(IEnumerable<string> args) where T : new() {
17+
using var enumearator = args.GetEnumerator();
18+
var instance = new T();
19+
while (enumearator.MoveNext()) {
20+
var n = enumearator.Current;
21+
FieldInfo? f = null;
22+
if (n.StartsWith("--")) {
23+
f = typeof(T).GetField(n[2..], BindingFlags.IgnoreCase);
24+
if (f is null)
25+
continue;
26+
}
27+
else if (n.StartsWith("-")) {
28+
foreach (var field in typeof(T).GetFields()) {
29+
var mininame = field.GetCustomAttribute<MiniNameAttribute>()?.Name;
30+
if (mininame == n[1..]) {
31+
f = field;
32+
break;
33+
}
34+
}
35+
if (f is null)
36+
continue;
37+
//f = f ?? throw new ArgumentException($"Unknown arg {n}");
38+
}
39+
else {
40+
continue;
41+
throw new ArgumentException($"Unknown arg {n}");
42+
}
43+
enumearator.MoveNext();
44+
if (f.FieldType == typeof(bool)) {
45+
f.SetValue(instance, true);
46+
continue;
47+
}
48+
if (f.FieldType.IsArray) {
49+
var strs = enumearator.Current.Split(",");
50+
var t = f.FieldType.GetElementType() ?? throw new Exception();
51+
var arr = Array.CreateInstance(t, strs.Length);
52+
for (var i = 0; i < arr.Length; i++) {
53+
Console.WriteLine(arr.Length);
54+
arr.SetValue(Convert.ChangeType(enumearator.Current, t), i);
55+
}
56+
57+
f.SetValue(instance, arr);
58+
continue;
59+
}
60+
61+
f.SetValue(instance, Convert.ChangeType(enumearator.Current, f.FieldType));
62+
}
63+
64+
return instance;
65+
}
66+
67+
public static string GetHelpFor(Type type, string prepend = "") {
68+
var a = new StringBuilder();
69+
var val = Activator.CreateInstance(type);
70+
foreach (var field in type.GetFields()) {
71+
if ( field.GetCustomAttribute<SkipAttribute>() is not null)
72+
continue;
73+
74+
a.Append(prepend);
75+
a.Append("--");
76+
a.Append(field.Name);
77+
var mini = field.GetCustomAttribute<MiniNameAttribute>()?.Name;
78+
if (mini is not null) {
79+
a.Append("(-");
80+
a.Append(mini);
81+
a.Append(')');
82+
}
83+
var help = field.GetCustomAttribute<HelpAttribute>()?.Text;
84+
a.Append(":\t");
85+
a.Append(help);
86+
a.Append("\t[");
87+
a.Append(field.FieldType.Name);
88+
a.Append("] Default = ");
89+
a.Append(field.GetValue(val));
90+
a.Append('\n');
91+
}
92+
93+
return a.ToString();
94+
}
95+
96+
public static string GetHelpFor<T>(string prepend = "") => GetHelpFor(typeof(T), prepend);
97+
}

‎NekoLib.Archive/NekoArchiveCompressor.cs‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@
99
namespace NekoLib.Archive;
1010

1111
public class NekoArchiveCompressor {
12-
public string DirectoryPath = "";
12+
public List<string> DirectoriesPath = [];
1313
public string OutputDir = "";
1414
public int ArchiveCount = 1;
1515
public string ArchiveName = "";
1616
public ICompressor Compressor;
17+
public bool Force;
1718

1819
internal class ExtensionNode(string ext) {
1920
public string Extension = ext;
@@ -31,10 +32,10 @@ internal class FileNode(string name) {
3132
public byte[] Data;
3233
}
3334

34-
public NekoArchiveCompressor SetDirectoryPath(string path) {
35-
DirectoryPath = path;
35+
public NekoArchiveCompressor AddDirectoryPath(string path) {
36+
DirectoriesPath.Add(path);
3637
if (ArchiveName == "") ArchiveName = Path.GetFileName(Path.GetDirectoryName(path))??"data";
37-
if (OutputDir == "") OutputDir = Path.Combine(DirectoryPath, "..");
38+
if (OutputDir == "") OutputDir = Path.Combine(path, "..");
3839
return this;
3940
}
4041

@@ -57,25 +58,37 @@ public NekoArchiveCompressor SetCompressor(ICompressor compressor) {
5758
Compressor = compressor;
5859
return this;
5960
}
61+
62+
public NekoArchiveCompressor SetForce(bool force) {
63+
Force = force;
64+
return this;
65+
}
6066

67+
//TODO: support multiple dirs DirectoriesPath[0]
6168
public unsafe void Compress() {
62-
Directory.Exists(DirectoryPath);
69+
Console.WriteLine("Compressing started");
70+
Directory.Exists(DirectoriesPath[0]);
6371
var infos = new List<FileInfo>();
6472
var fileTree = new Dictionary<string, ExtensionNode>();
6573
ulong offset = 0;
66-
using var fileStream = File.Open(Path.Join(OutputDir, ArchiveName+".nla"), FileMode.CreateNew, FileAccess.Write);
74+
var archivePath = Path.Join(OutputDir, ArchiveName + ".nla");
75+
Console.WriteLine("Compressing to "+archivePath);
76+
if (Force && File.Exists(archivePath)) {
77+
File.Delete(archivePath);
78+
}
79+
using var fileStream = File.Open(archivePath, FileMode.CreateNew, FileAccess.Write);
6780
using var br = new BinaryWriter(fileStream);
6881
var datablob = Array.Empty<byte[]>();
6982
if (Compressor.SupportsTraining) {
7083
var uncomp = new List<byte[]>();
71-
foreach (var file in Directory.GetFiles(DirectoryPath, "*.*", SearchOption.AllDirectories)) {
84+
foreach (var file in Directory.GetFiles(DirectoriesPath[0], "*.*", SearchOption.AllDirectories)) {
7285
var f = File.ReadAllBytes(file);
7386
uncomp.Add(f);
7487
}
7588
Compressor.Train(uncomp);
7689
}
77-
foreach (var file in Directory.GetFiles(DirectoryPath, "*.*", SearchOption.AllDirectories)) {
78-
var rlPath = Path.GetRelativePath(DirectoryPath, file);
90+
foreach (var file in Directory.GetFiles(DirectoriesPath[0], "*.*", SearchOption.AllDirectories)) {
91+
var rlPath = Path.GetRelativePath(DirectoriesPath[0], file);
7992
var ep = EntryPath.FromString(rlPath);
8093
var fileInfo = new FileInfo(file);
8194
if (!fileTree.TryGetValue(ep.Extension, out var node))

‎NekoLib.Archive/Program.cs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
namespace NekoLib.Archive;
2+
3+
public class Program {
4+
class DecompressArgs {
5+
[MiniCli.MiniName("i")]
6+
[MiniCli.Help("Path to root (first) archive")]
7+
public required string Input = "";
8+
9+
[MiniCli.MiniName("o")]
10+
[MiniCli.Help("Output directory")]
11+
public required string Output = "";
12+
// [MiniCli.MiniName("D")]
13+
// public string[] LoadDlls;
14+
}
15+
16+
class CompressArgs {
17+
[MiniCli.MiniName("i")]
18+
[MiniCli.Help("Input directories to compress")]
19+
public string[] Input = [];
20+
[MiniCli.MiniName("o")]
21+
[MiniCli.Help("Output directory")]
22+
public string Output = "";
23+
[MiniCli.MiniName("r")]
24+
[MiniCli.Help("use directory as root")]
25+
public bool Root = false;
26+
[MiniCli.MiniName("m")]
27+
[MiniCli.Help("Max size of 1 archive data (in bytes)")]
28+
public int? MaxSize = null;
29+
[MiniCli.MiniName("n")]
30+
[MiniCli.Help("Name of the archives")]
31+
public string Name = "";
32+
[MiniCli.MiniName("t")]
33+
[MiniCli.Help("Compression type (ZSTD or NONE)")]
34+
public string CompressionType = "ZSTD";
35+
[MiniCli.MiniName("f")]
36+
[MiniCli.Help("Remove archives if exists")]
37+
public bool Force = false;
38+
// [MiniCli.MiniName("D")]
39+
// public string[] LoadDlls;
40+
}
41+
42+
class ZstdCompressArgs {
43+
[MiniCli.MiniName("l")]
44+
[MiniCli.Help("Compression level")]
45+
public int CompressionLevel = 3;
46+
[MiniCli.MiniName("s")]
47+
[MiniCli.Help("Dictionary size (recommended to be ~100x less than data), 0 to disable")]
48+
public int DictionarySize = 112640;
49+
}
50+
51+
public static void PrintHelp()
52+
{
53+
Console.WriteLine("Usage: dotnet NekoLib.Archive.dll -- command [ARGS]");
54+
Console.WriteLine("Available commands: decompress, compress");
55+
Console.WriteLine("decompress ARGS:");
56+
Console.WriteLine(MiniCli.GetHelpFor<DecompressArgs>("\t"));
57+
Console.WriteLine("Compress ARGS:");
58+
Console.WriteLine(MiniCli.GetHelpFor<CompressArgs>("\t"));
59+
Console.WriteLine("\tZSTD compression extra:");
60+
Console.WriteLine(MiniCli.GetHelpFor<ZstdCompressArgs>("\t\t"));
61+
}
62+
63+
public static void Main(string[] args) {
64+
if (args.Length < 1) {
65+
PrintHelp();
66+
return;
67+
}
68+
var arglist = args.ToList();
69+
int i = 0;
70+
var subcommand = arglist[0];
71+
arglist.RemoveAt(0);
72+
if (subcommand != "compress" && subcommand != "decompress") {
73+
Console.WriteLine("ERROR: Unknown verb "+subcommand);
74+
PrintHelp();
75+
return;
76+
}
77+
78+
if (subcommand == "compress") {
79+
var a = MiniCli.Parse<CompressArgs>(arglist);
80+
var comp = new NekoArchiveCompressor();
81+
82+
switch (a.Input.Length) {
83+
case <= 0:
84+
throw new ArgumentException("No input folders provided");
85+
case > 1:
86+
throw new NotImplementedException();
87+
}
88+
89+
foreach (var path in a.Input) {
90+
comp.AddDirectoryPath(path);
91+
}
92+
if (a.Output != "")
93+
comp.SetOutputDir(a.Output);
94+
if (a.Name != "")
95+
comp.SetArchiveName(a.Name);
96+
if (a.CompressionType.Equals("zstd", StringComparison.CurrentCultureIgnoreCase)) {
97+
var zstd = new ZstdProvider();
98+
var b = MiniCli.Parse<ZstdCompressArgs>(arglist);
99+
zstd.CompressionLevel = b.CompressionLevel;
100+
zstd.DictCapacity = b.DictionarySize;
101+
comp.SetCompressor(zstd);
102+
}
103+
else {
104+
comp.SetCompressor(new StoreProvider());
105+
}
106+
if (a.Force) {
107+
comp.SetForce(true);
108+
}
109+
comp.Compress();
110+
//.SetCompressor(new ZstdProvider(Compressor.MaxCompressionLevel));
111+
}
112+
}
113+
}

‎NekoLib.sln.DotSettings.user‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASpan_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fd15de8ed93064806a163b97c7155b882d19c00_003F1f_003Fb0fa23b1_003FSpan_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
99
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelpers_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fd15de8ed93064806a163b97c7155b882d19c00_003F62_003Feffdbe80_003FThrowHelpers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
1010
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc8fee2b6e43d4b47bd6d21ef51f1052060a00_003F01_003F17b5858c_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
11+
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc8fee2b6e43d4b47bd6d21ef51f1052060a00_003F01_003F17b5858c_003FThrowHelper_002Ecs_002Fz_003A2_002D0/@EntryIndexedValue">ForceIncluded</s:String>
1112
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fd15de8ed93064806a163b97c7155b882d19c00_003F2a_003F6080e81a_003FThrowHelper_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
1213
<s:String x:Key="/Default/Environment/AssemblyExplorer/XmlDocument/@EntryValue">&lt;AssemblyExplorer&gt;
1314
&lt;Assembly Path="/home/vandercat/RiderProjects/NekoLib/NekoLib/bin/Release/net8.0/NekoLib.dll" /&gt;

‎Test/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class Program {
77
static void Main(string[] args) {
88

99
var comp = new NekoArchiveCompressor()
10-
.SetDirectoryPath(args[0])
10+
.AddDirectoryPath(args[0])
1111
.SetOutputDir(args[1])
1212
.SetCompressor(new StoreProvider());
1313
//.SetCompressor(new ZstdProvider(Compressor.MaxCompressionLevel));

0 commit comments

Comments
 (0)