yume/fedivet
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gzip compress audit file

Browse source 
Signed-off-by: eternal-flame-AD <yume@yumechi.jp>
This commit is contained in:
ゆめ 2024-10-16 18:34:45 -05:00
parent c67e647b11
commit 1af251420e
No known key found for this signature in database
1 changed files with 14 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/evaluate/chain/audit.rs
View file

@ -1,7 +1,10 @@
use axum::response::IntoResponse; use axum::response::IntoResponse;
use flate2::write::GzEncoder;
use flate2::{Compression, GzBuilder};
use serde::Serialize; use serde::Serialize;
use std::collections::HashSet; use std::collections::HashSet;
use std::fs::{File, OpenOptions}; use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter};
use std::sync::atomic::AtomicU32; use std::sync::atomic::AtomicU32;
use std::{collections::HashMap, fmt::Debug, ops::DerefMut, path::PathBuf, sync::Arc}; use std::{collections::HashMap, fmt::Debug, ops::DerefMut, path::PathBuf, sync::Arc};
use tokio::sync::{Mutex, RwLock}; use tokio::sync::{Mutex, RwLock};
@ -57,7 +60,7 @@ pub enum AuditError {
pub struct AuditState { pub struct AuditState {
options: AuditOptions, options: AuditOptions,
cur_file: Option<RwLock<HashMap<String, Mutex<File>>>>, cur_file: Option<RwLock<HashMap<String, Mutex<GzEncoder<File>>>>>,
vacuum_counter: AtomicU32, vacuum_counter: AtomicU32,
} }
@ -81,7 +84,7 @@ impl AuditState {
.filter_map(|f| f.ok()) .filter_map(|f| f.ok())
.filter(|f| { .filter(|f| {
f.file_type().map(|t| t.is_file()).unwrap_or(false) f.file_type().map(|t| t.is_file()).unwrap_or(false)
&& f.file_name().to_string_lossy().ends_with(".json") && f.file_name().to_string_lossy().ends_with(".json.gz")
}) })
.filter(|f| !in_use_files.contains(&f.file_name().to_string_lossy().to_string())); .filter(|f| !in_use_files.contains(&f.file_name().to_string_lossy().to_string()));
@ -108,7 +111,7 @@ impl AuditState {
let mut write = self.cur_file.as_ref().unwrap().write().await; let mut write = self.cur_file.as_ref().unwrap().write().await;
let full_name = format!("{}_{}.json", name, time_str); let full_name = format!("{}_{}.json.gz", name, time_str);
let file = OpenOptions::new() let file = OpenOptions::new()
.create(true) .create(true)
@ -116,7 +119,9 @@ impl AuditState {
.append(true) .append(true)
.open(&self.options.output.join(&full_name))?; .open(&self.options.output.join(&full_name))?;
write.insert(name.to_string(), Mutex::new(file)); let gz = GzEncoder::new(file, Compression::default());
write.insert(name.to_string(), Mutex::new(gz));
Ok(()) Ok(())
} }
@ -143,7 +148,7 @@ impl AuditState {
serde_json::to_writer(f.deref_mut(), &item)?; serde_json::to_writer(f.deref_mut(), &item)?;
let meta = f.metadata()?; let meta = f.get_ref().metadata()?;
if let Some(size) = self.options.rotate_size { if let Some(size) = self.options.rotate_size {
if meta.len() >= size { if meta.len() >= size {
@ -161,6 +166,10 @@ impl AuditState {
let file = File::create(&self.options.output.join(name))?; let file = File::create(&self.options.output.join(name))?;
let file = GzBuilder::new()
.filename(name)
.write(file, Compression::default());
write.insert(name.to_string(), Mutex::new(file)); write.insert(name.to_string(), Mutex::new(file));
// this is deliberately out of order to make sure we don't create endless files if serialization fails // this is deliberately out of order to make sure we don't create endless files if serialization fails