moparisthebest/cache-fs
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Implement zstd compression for attribute cache

Browse Source
This commit is contained in:
Travis Burtrum 2022-08-04 01:09:21 -04:00
parent c0f4a33e86
commit cd6c0716c6
3 changed files with 52 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Cargo.lock generated
View File

@ -47,6 +47,16 @@ dependencies = [
"libc", "libc",
"log", "log",
"serde", "serde",
"zstd",
]
[[package]]
name = "cc"
version = "1.0.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
dependencies = [
"jobserver",
] ]
[[package]] [[package]]
@ -102,6 +112,15 @@ dependencies = [
"quick-error", "quick-error",
] ]
[[package]]
name = "jobserver"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.126" version = "0.2.126"
@ -311,3 +330,32 @@ dependencies = [
"syn", "syn",
"synstructure", "synstructure",
] ]
[[package]]
name = "zstd"
version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "5.0.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.1+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
dependencies = [
"cc",
"libc",
]

1
Cargo.toml
View File

@ -27,3 +27,4 @@ log = "0.4"
serde = { version="1.0", features = [ "derive" ] } serde = { version="1.0", features = [ "derive" ] }
bincode = "1.3" bincode = "1.3"
zstd = "0.11"

4
src/main.rs
View File

@ -107,7 +107,7 @@ impl Debug for FileTree {
impl FileTree { impl FileTree {
fn load_or_build(root_path: &Path, cache_path: &Path) -> SerdeResult<Self> { fn load_or_build(root_path: &Path, cache_path: &Path) -> SerdeResult<Self> {
let path = cache_path.join("cache-fs.tree"); let path = cache_path.join("cache-fs.tree.zst");
match FileTree::load(&path) { match FileTree::load(&path) {
Ok(tree) => return Ok(tree), Ok(tree) => return Ok(tree),
Err(e) => warn!("error loading {:?}: {:?}", path, e), Err(e) => warn!("error loading {:?}: {:?}", path, e),
@ -120,6 +120,7 @@ impl FileTree {
fn load(path: &Path) -> SerdeResult<Self> { fn load(path: &Path) -> SerdeResult<Self> {
let file = File::open(path)?; let file = File::open(path)?;
let file = BufReader::new(file); let file = BufReader::new(file);
let file = zstd::stream::Decoder::new(file)?;
Ok(bincode::deserialize_from(file)?) Ok(bincode::deserialize_from(file)?)
} }
@ -127,6 +128,7 @@ impl FileTree {
fn save(&self, path: &Path) -> SerdeResult<()> { fn save(&self, path: &Path) -> SerdeResult<()> {
let file = File::create(path)?; let file = File::create(path)?;
let file = BufWriter::new(file); let file = BufWriter::new(file);
let file = zstd::stream::Encoder::new(file, 9)?.auto_finish();
Ok(bincode::serialize_into(file, self)?) Ok(bincode::serialize_into(file, self)?)
} }