niko/oxkerclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert: Bollard update rolled back, closes #66

Browse Source 
Using Bollard 0.19.x meant that network stats weren't being recorded correctly. Updated local unhealthy container to use a speed test in order to check for network IO.
This commit is contained in:
Jack Wills
2025-06-19 20:36:54 +00:00
parent c48465f536
commit aac9c6b598
8 changed files with 256 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/docker_data/mod.rs
+244 -188
View File
@@ -1,7 +1,9 @@
use bollard::{
Docker,
query_parameters::{ListContainersOptions, LogsOptions, RemoveContainerOptions, StatsOptions},
secret::ContainerStatsResponse,
container::{
ListContainersOptions, LogsOptions, MemoryStatsStats, RemoveContainerOptions,
StartContainerOptions, Stats, StatsOptions,
},
service::ContainerSummary,
};
use futures_util::StreamExt;
@@ -73,44 +75,31 @@ pub struct DockerData {
impl DockerData {
/// Use docker stats to calculate current cpu usage
#[allow(clippy::cast_precision_loss)]
fn calculate_usage(stats: &ContainerStatsResponse) -> f64 {
fn calculate_usage(stats: &Stats) -> f64 {
let mut cpu_percentage = 0.0;
let cpu_delta = stats
.cpu_stats
.cpu_usage
.total_usage
.saturating_sub(stats.precpu_stats.cpu_usage.total_usage)
as f64;
let total_usage = stats.precpu_stats.as_ref().map_or(0, |i| {
i.cpu_usage
.as_ref()
.map_or(0, |i| i.total_usage.unwrap_or_default())
});
let cpu_delta = stats.cpu_stats.as_ref().map_or(0, |i| {
i.cpu_usage.as_ref().map_or(0, |i| {
i.total_usage
.unwrap_or_default()
.saturating_sub(total_usage)
})
}) as f64;
if let (Some(Some(cpu_stats_usage)), Some(Some(precpu_stats_usage))) = (
stats.cpu_stats.as_ref().map(|i| i.system_cpu_usage),
stats.precpu_stats.as_ref().map(|i| i.system_cpu_usage),
if let (Some(cpu_stats_usage), Some(precpu_stats_usage)) = (
stats.cpu_stats.system_cpu_usage,
stats.precpu_stats.system_cpu_usage,
) {
let system_delta = cpu_stats_usage.saturating_sub(precpu_stats_usage) as f64;
let online_cpus = f64::from(stats.cpu_stats.as_ref().map_or(0, |i| {
i.online_cpus.unwrap_or_else(|| {
u32::try_from(
stats
.cpu_stats
.clone()
.unwrap_or_default()
.cpu_usage
.unwrap_or_default()
.percpu_usage
.as_ref()
.map_or(0, std::vec::Vec::len),
)
.unwrap_or_default()
})
}));
let online_cpus = stats.cpu_stats.online_cpus.unwrap_or_else(|| {
u64::try_from(
stats
.cpu_stats
.cpu_usage
.percpu_usage
.as_ref()
.map_or(0, std::vec::Vec::len),
)
.unwrap_or_default()
}) as f64;
if system_delta > 0.0 && cpu_delta > 0.0 {
cpu_percentage = (cpu_delta / system_delta) * online_cpus * 100.0;
}
@@ -142,23 +131,20 @@ impl DockerData {
)
.take(1);
// some err here
while let Some(Ok(stats)) = stream.next().await {
// Memory stats are only collected if the container is alive - is this the behaviour we want?
let (mem_stat, cpu_stats) = if state.is_alive() {
let mem_cache = stats.memory_stats.as_ref().map_or(&0, |i| {
i.stats
.as_ref()
.map_or(&0, |i| i.get("inactive_file").unwrap_or(&0))
let mem_cache = stats.memory_stats.stats.map_or(0, |i| match i {
MemoryStatsStats::V1(x) => x.inactive_file,
MemoryStatsStats::V2(x) => x.inactive_file,
});
(
Some(
stats
.memory_stats
.as_ref()
.map_or(0, |i| i.usage.unwrap_or_default())
.saturating_sub(*mem_cache),
.usage
.unwrap_or_default()
.saturating_sub(mem_cache),
),
Some(Self::calculate_usage(&stats)),
)
@@ -166,22 +152,26 @@ impl DockerData {
(None, None)
};
let (rx, tx) = stats.networks.as_ref().map_or((0, 0), |i| {
(
i.rx_bytes.unwrap_or_default(),
i.tx_bytes.unwrap_or_default(),
)
});
let op_key = stats
.networks
.as_ref()
.and_then(|networks| networks.keys().next().cloned());
let (rx, tx) = if let Some(key) = op_key {
stats
.networks
.unwrap_or_default()
.get(&key)
.map_or((0, 0), |f| (f.rx_bytes, f.tx_bytes))
} else {
(0, 0)
};
app_data.lock().update_stats_by_id(
id,
cpu_stats,
mem_stat,
stats
.memory_stats
.unwrap_or_default()
.limit
.unwrap_or_default(),
stats.memory_stats.limit.unwrap_or_default(),
rx,
tx,
);
@@ -216,7 +206,7 @@ impl DockerData {
async fn update_all_containers(&self) {
let containers = self
.docker
.list_containers(Some(ListContainersOptions {
.list_containers(Some(ListContainersOptions::<String> {
all: true,
..Default::default()
}))
@@ -241,7 +231,6 @@ impl DockerData {
None => None,
})
.collect::<Vec<ContainerSummary>>();
self.app_data.lock().update_containers(output);
}
@@ -255,11 +244,11 @@ impl DockerData {
spawns: Arc<Mutex<HashMap<SpawnId, JoinHandle<()>>>>,
stderr: bool,
) {
let options = Some(LogsOptions {
let options = Some(LogsOptions::<String> {
stdout: true,
stderr,
timestamps: true,
since: i32::try_from(since).unwrap_or_default(),
since: i64::try_from(since).unwrap_or_default(),
..Default::default()
});
@@ -376,31 +365,14 @@ impl DockerData {
.await
}
DockerCommand::Pause => docker.pause_container(id.get()).await,
DockerCommand::Restart => {
docker
.restart_container(
id.get(),
None::<bollard::query_parameters::RestartContainerOptions>,
)
.await
}
DockerCommand::Restart => docker.restart_container(id.get(), None).await,
DockerCommand::Resume => docker.unpause_container(id.get()).await,
DockerCommand::Start => {
docker
.start_container(
id.get(),
None::<bollard::query_parameters::StartContainerOptions>,
)
.await
}
DockerCommand::Stop => {
docker
.stop_container(
id.get(),
None::<bollard::query_parameters::StopContainerOptions>,
)
.start_container(id.get(), None::<StartContainerOptions<String>>)
.await
}
DockerCommand::Stop => docker.stop_container(id.get(), None).await,
}
.is_err()
{
@@ -476,72 +448,119 @@ impl DockerData {
#[allow(clippy::float_cmp)]
mod tests {
use bollard::secret::{ContainerCpuStats, ContainerCpuUsage};
use bollard::container::{
BlkioStats, CPUStats, CPUUsage, MemoryStats, PidsStats, Stats, StorageStats, ThrottlingData,
};
use super::*;
fn gen_stats() -> ContainerStatsResponse {
ContainerStatsResponse {
read: None,
preread: None,
num_procs: Some(1),
pids_stats: None,
fn gen_stats() -> Stats {
Stats {
read: String::new(),
preread: String::new(),
num_procs: 1,
pids_stats: PidsStats {
current: None,
limit: None,
},
network: None,
networks: None,
memory_stats: None,
blkio_stats: None,
cpu_stats: Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
memory_stats: MemoryStats {
stats: None,
max_usage: None,
usage: None,
failcnt: None,
limit: None,
commit: None,
commit_peak: None,
commitbytes: None,
commitpeakbytes: None,
privateworkingset: None,
},
blkio_stats: BlkioStats {
io_service_bytes_recursive: None,
io_serviced_recursive: None,
io_queue_recursive: None,
io_service_time_recursive: None,
io_wait_time_recursive: None,
io_merged_recursive: None,
io_time_recursive: None,
sectors_recursive: None,
},
cpu_stats: CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
}),
precpu_stats: Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
},
precpu_stats: CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
}),
storage_stats: None,
name: None,
id: None,
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
},
storage_stats: StorageStats {
read_count_normalized: None,
read_size_bytes: None,
write_count_normalized: None,
write_size_bytes: None,
},
name: String::new(),
id: String::new(),
}
}
#[test]
fn test_calculate_usage_50() {
let mut stats = gen_stats();
stats.precpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
stats.precpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
});
stats.cpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
stats.cpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![150]),
usage_in_usermode: Some(20),
total_usage: Some(150),
usage_in_kernelmode: Some(30),
}),
usage_in_usermode: 20,
total_usage: 150,
usage_in_kernelmode: 30,
},
system_cpu_usage: Some(500),
online_cpus: Some(1),
throttling_data: None,
});
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
let cpu_percentage = DockerData::calculate_usage(&stats);
assert_eq!(50.0, cpu_percentage);
}
@@ -549,28 +568,37 @@ mod tests {
#[test]
fn test_calculate_usage_25() {
let mut stats = gen_stats();
stats.precpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
stats.precpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
});
stats.cpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
stats.cpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![75]),
usage_in_usermode: Some(20),
total_usage: Some(125),
usage_in_kernelmode: Some(30),
}),
usage_in_usermode: 20,
total_usage: 125,
usage_in_kernelmode: 30,
},
system_cpu_usage: Some(500),
online_cpus: Some(1),
throttling_data: None,
});
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
let cpu_percentage = DockerData::calculate_usage(&stats);
assert_eq!(25.0, cpu_percentage);
}
@@ -578,28 +606,38 @@ mod tests {
#[test]
fn test_calculate_usage_75() {
let mut stats = gen_stats();
stats.precpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
stats.precpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
});
stats.cpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
stats.cpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![175]),
usage_in_usermode: Some(20),
total_usage: Some(175),
usage_in_kernelmode: Some(30),
}),
usage_in_usermode: 20,
total_usage: 175,
usage_in_kernelmode: 30,
},
system_cpu_usage: Some(500),
online_cpus: Some(1),
throttling_data: None,
});
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
let cpu_percentage = DockerData::calculate_usage(&stats);
assert_eq!(75.0, cpu_percentage);
}
@@ -607,28 +645,36 @@ mod tests {
#[test]
fn test_calculate_usage_100() {
let mut stats = gen_stats();
stats.precpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
stats.precpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
});
stats.cpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
stats.cpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![200]),
usage_in_usermode: Some(20),
total_usage: Some(200),
usage_in_kernelmode: Some(30),
}),
usage_in_usermode: 20,
total_usage: 200,
usage_in_kernelmode: 30,
},
system_cpu_usage: Some(500),
online_cpus: Some(1),
throttling_data: None,
});
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
let cpu_percentage = DockerData::calculate_usage(&stats);
assert_eq!(100.0, cpu_percentage);
}
@@ -636,28 +682,38 @@ mod tests {
#[test]
fn test_calculate_usage_175() {
let mut stats = gen_stats();
stats.precpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
stats.precpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![50]),
usage_in_usermode: Some(10),
total_usage: Some(100),
usage_in_kernelmode: Some(20),
}),
usage_in_usermode: 10,
total_usage: 100,
usage_in_kernelmode: 20,
},
system_cpu_usage: Some(400),
online_cpus: Some(1),
throttling_data: None,
});
stats.cpu_stats = Some(ContainerCpuStats {
cpu_usage: Some(ContainerCpuUsage {
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
stats.cpu_stats = CPUStats {
cpu_usage: CPUUsage {
percpu_usage: Some(vec![275]),
usage_in_usermode: Some(20),
total_usage: Some(275),
usage_in_kernelmode: Some(30),
}),
usage_in_usermode: 20,
total_usage: 275,
usage_in_kernelmode: 30,
},
system_cpu_usage: Some(500),
online_cpus: Some(1),
throttling_data: None,
});
throttling_data: ThrottlingData {
periods: 0,
throttled_periods: 0,
throttled_time: 0,
},
};
let cpu_percentage = DockerData::calculate_usage(&stats);
assert_eq!(175.0, cpu_percentage);
}