Skip to content

Commit 604271a

Browse files
chesedochesedo
authored
bug: no networks (#541)
* refactor: connect to user network on startup * feat: revive containers not connected to the user network * refactor: attach network first * refactor: revive created containers * feat: attaching state * refactor: resetting some code
1 parent 8f71804 commit 604271a

File tree

2 files changed

+176
-33
lines changed

2 files changed

+176
-33
lines changed

‎common/src/models/project.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ pub struct Response {
1818
#[strum(serialize_all = "lowercase")]
1919
pub enum State {
2020
Creating,
21+
Attaching,
2122
Starting,
2223
Started,
2324
Ready,
@@ -42,10 +43,10 @@ impl Display for Response {
4243
impl State {
4344
pub fn get_color(&self) -> Color {
4445
match self {
45-
State::Creating | State::Starting | State::Started => Color::Cyan,
46-
State::Ready => Color::Green,
47-
State::Stopped | State::Stopping | State::Destroying | State::Destroyed => Color::Blue,
48-
State::Errored => Color::Red,
46+
Self::Creating | Self::Attaching | Self::Starting | Self::Started => Color::Cyan,
47+
Self::Ready => Color::Green,
48+
Self::Stopped | Self::Stopping | Self::Destroying | Self::Destroyed => Color::Blue,
49+
Self::Errored => Color::Red,
4950
}
5051
}
5152
}

‎gateway/src/project.rs

Lines changed: 171 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ use bollard::container::{
88
};
99
use bollard::errors::Error as DockerError;
1010
use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum};
11+
use bollard::network::{ConnectNetworkOptions, DisconnectNetworkOptions};
12+
use bollard::service::EndpointSettings;
1113
use bollard::system::EventsOptions;
1214
use fqdn::FQDN;
1315
use futures::prelude::*;
@@ -19,7 +21,7 @@ use once_cell::sync::Lazy;
1921
use rand::distributions::{Alphanumeric, DistString};
2022
use serde::{Deserialize, Serialize};
2123
use tokio::time::{self, timeout};
22-
use tracing::{debug, error, instrument};
24+
use tracing::{debug, error, info, instrument};
2325

2426
use crate::{
2527
ContainerSettings, DockerContext, EndState, Error, ErrorKind, IntoTryState, ProjectName,
@@ -146,6 +148,7 @@ impl From<DockerError> for Error {
146148
#[serde(rename_all = "lowercase")]
147149
pub enum Project {
148150
Creating(ProjectCreating),
151+
Attaching(ProjectAttaching),
149152
Starting(ProjectStarting),
150153
Started(ProjectStarted),
151154
Ready(ProjectReady),
@@ -158,6 +161,7 @@ pub enum Project {
158161

159162
impl_from_variant!(Project:
160163
ProjectCreating => Creating,
164+
ProjectAttaching => Attaching,
161165
ProjectStarting => Starting,
162166
ProjectStarted => Started,
163167
ProjectReady => Ready,
@@ -220,6 +224,7 @@ impl Project {
220224
Self::Starting(_) => "starting",
221225
Self::Stopping(_) => "stopping",
222226
Self::Creating(_) => "creating",
227+
Self::Attaching(_) => "attaching",
223228
Self::Destroying(_) => "destroying",
224229
Self::Destroyed(_) => "destroyed",
225230
Self::Errored(_) => "error",
@@ -230,6 +235,7 @@ impl Project {
230235
match self {
231236
Self::Starting(ProjectStarting { container, .. })
232237
| Self::Started(ProjectStarted { container, .. })
238+
| Self::Attaching(ProjectAttaching { container, .. })
233239
| Self::Ready(ProjectReady { container, .. })
234240
| Self::Stopping(ProjectStopping { container })
235241
| Self::Stopped(ProjectStopped { container })
@@ -256,6 +262,7 @@ impl From<Project> for shuttle_common::models::project::State {
256262
fn from(project: Project) -> Self {
257263
match project {
258264
Project::Creating(_) => Self::Creating,
265+
Project::Attaching(_) => Self::Attaching,
259266
Project::Starting(_) => Self::Starting,
260267
Project::Started(_) => Self::Started,
261268
Project::Ready(_) => Self::Ready,
@@ -283,6 +290,17 @@ where
283290

284291
let mut new = match self {
285292
Self::Creating(creating) => creating.next(ctx).await.into_try_state(),
293+
Self::Attaching(attaching) => match attaching.next(ctx).await {
294+
Err(ProjectError {
295+
kind: ProjectErrorKind::NoNetwork,
296+
ctx,
297+
..
298+
}) => {
299+
// Restart the container to try and connect to the network again
300+
Ok(ctx.unwrap().stop().unwrap())
301+
}
302+
attaching => attaching.into_try_state(),
303+
},
286304
Self::Starting(ready) => ready.next(ctx).await.into_try_state(),
287305
Self::Started(started) => match started.next(ctx).await {
288306
Ok(ProjectReadying::Ready(ready)) => Ok(ready.into()),
@@ -350,6 +368,7 @@ where
350368
async fn refresh(self, ctx: &Ctx) -> Result<Self, Self::Error> {
351369
let refreshed = match self {
352370
Self::Creating(creating) => Self::Creating(creating),
371+
Self::Attaching(attaching) => Self::Attaching(attaching),
353372
Self::Starting(ProjectStarting { container })
354373
| Self::Started(ProjectStarted { container, .. })
355374
| Self::Ready(ProjectReady { container, .. })
@@ -463,8 +482,6 @@ impl ProjectCreating {
463482
image: default_image,
464483
prefix,
465484
provisioner_host,
466-
network_name,
467-
network_id,
468485
fqdn: public,
469486
..
470487
} = ctx.container_settings();
@@ -521,14 +538,6 @@ impl ProjectCreating {
521538

522539
let mut config = Config::<String>::from(container_config);
523540

524-
config.networking_config = deserialize_json!({
525-
"EndpointsConfig": {
526-
network_name: {
527-
"NetworkID": network_id
528-
}
529-
}
530-
});
531-
532541
config.host_config = deserialize_json!({
533542
"Mounts": [{
534543
"Target": "/opt/shuttle",
@@ -559,7 +568,7 @@ impl<Ctx> State<Ctx> for ProjectCreating
559568
where
560569
Ctx: DockerContext,
561570
{
562-
type Next = ProjectStarting;
571+
type Next = ProjectAttaching;
563572
type Error = ProjectError;
564573

565574
#[instrument(skip_all)]
@@ -582,6 +591,85 @@ where
582591
}
583592
})
584593
.await?;
594+
Ok(ProjectAttaching { container })
595+
}
596+
}
597+
598+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
599+
pub struct ProjectAttaching {
600+
container: ContainerInspectResponse,
601+
}
602+
603+
#[async_trait]
604+
impl<Ctx> State<Ctx> for ProjectAttaching
605+
where
606+
Ctx: DockerContext,
607+
{
608+
type Next = ProjectStarting;
609+
type Error = ProjectError;
610+
611+
#[instrument(skip_all)]
612+
async fn next(self, ctx: &Ctx) -> Result<Self::Next, Self::Error> {
613+
let Self { container } = self;
614+
615+
let container_id = container.id.as_ref().unwrap();
616+
let ContainerSettings {
617+
network_name,
618+
network_id,
619+
..
620+
} = ctx.container_settings();
621+
622+
// Disconnect the bridge network before trying to start up
623+
// For docker bug https://github.com/docker/cli/issues/1891
624+
//
625+
// Also disconnecting from all network because docker just losses track of their IDs sometimes when restarting
626+
for network in safe_unwrap!(container.network_settings.networks).keys() {
627+
ctx.docker().disconnect_network(network, DisconnectNetworkOptions{
628+
container: container_id,
629+
force: true,
630+
})
631+
.await
632+
.or_else(|err| {
633+
if matches!(err, DockerError::DockerResponseServerError { status_code, .. } if status_code == 500) {
634+
info!("already disconnected from the {network} network");
635+
Ok(())
636+
} else {
637+
Err(err)
638+
}
639+
})?;
640+
}
641+
642+
// Make sure the container is connected to the user network
643+
let network_config = ConnectNetworkOptions {
644+
container: container_id,
645+
endpoint_config: EndpointSettings {
646+
network_id: Some(network_id.to_string()),
647+
..Default::default()
648+
},
649+
};
650+
ctx.docker()
651+
.connect_network(network_name, network_config)
652+
.await
653+
.or_else(|err| {
654+
if matches!(
655+
err,
656+
DockerError::DockerResponseServerError { status_code, .. } if status_code == 409
657+
) {
658+
info!("already connected to the shuttle network");
659+
Ok(())
660+
} else {
661+
error!(
662+
error = &err as &dyn std::error::Error,
663+
"failed to connect to shuttle network"
664+
);
665+
Err(ProjectError::no_network(
666+
"failed to connect to shuttle network",
667+
))
668+
}
669+
})?;
670+
671+
let container = container.refresh(ctx).await?;
672+
585673
Ok(ProjectStarting { container })
586674
}
587675
}
@@ -602,6 +690,7 @@ where
602690
#[instrument(skip_all)]
603691
async fn next(self, ctx: &Ctx) -> Result<Self::Next, Self::Error> {
604692
let container_id = self.container.id.as_ref().unwrap();
693+
605694
ctx.docker()
606695
.start_container::<String>(container_id, None)
607696
.await
@@ -916,6 +1005,7 @@ where
9161005
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
9171006
pub enum ProjectErrorKind {
9181007
Internal,
1008+
NoNetwork,
9191009
}
9201010

9211011
/// A runtime error coming from inside a project
@@ -934,6 +1024,14 @@ impl ProjectError {
9341024
ctx: None,
9351025
}
9361026
}
1027+
1028+
pub fn no_network<S: AsRef<str>>(message: S) -> Self {
1029+
Self {
1030+
kind: ProjectErrorKind::NoNetwork,
1031+
message: message.as_ref().to_string(),
1032+
ctx: None,
1033+
}
1034+
}
9371035
}
9381036

9391037
impl std::fmt::Display for ProjectError {
@@ -1032,22 +1130,47 @@ pub mod exec {
10321130
.inspect_container(safe_unwrap!(container.id), None)
10331131
.await
10341132
{
1035-
if let Some(ContainerState {
1036-
status: Some(ContainerStateStatusEnum::EXITED),
1037-
..
1038-
}) = container.state
1039-
{
1040-
debug!("{} will be revived", project_name.clone());
1041-
_ = gateway
1042-
.new_task()
1043-
.project(project_name)
1044-
.and_then(task::run(|ctx| async move {
1045-
TaskResult::Done(Project::Stopped(ProjectStopped {
1046-
container: ctx.state.container().unwrap(),
1133+
match container.state {
1134+
Some(ContainerState {
1135+
status: Some(ContainerStateStatusEnum::EXITED),
1136+
..
1137+
}) => {
1138+
debug!("{} will be revived", project_name.clone());
1139+
_ = gateway
1140+
.new_task()
1141+
.project(project_name)
1142+
.and_then(task::run(|ctx| async move {
1143+
TaskResult::Done(Project::Stopped(ProjectStopped {
1144+
container: ctx.state.container().unwrap(),
1145+
}))
1146+
}))
1147+
.send(&sender)
1148+
.await;
1149+
}
1150+
Some(ContainerState {
1151+
status: Some(ContainerStateStatusEnum::RUNNING),
1152+
..
1153+
})
1154+
| Some(ContainerState {
1155+
status: Some(ContainerStateStatusEnum::CREATED),
1156+
..
1157+
}) => {
1158+
debug!(
1159+
"{} is errored but ready according to docker. So restarting it",
1160+
project_name.clone()
1161+
);
1162+
_ = gateway
1163+
.new_task()
1164+
.project(project_name)
1165+
.and_then(task::run(|ctx| async move {
1166+
TaskResult::Done(Project::Stopping(ProjectStopping {
1167+
container: ctx.state.container().unwrap(),
1168+
}))
10471169
}))
1048-
}))
1049-
.send(&sender)
1050-
.await;
1170+
.send(&sender)
1171+
.await;
1172+
}
1173+
_ => {}
10511174
}
10521175
}
10531176
}
@@ -1062,6 +1185,7 @@ pub mod exec {
10621185
pub mod tests {
10631186

10641187
use bollard::models::ContainerState;
1188+
use bollard::service::NetworkSettings;
10651189
use futures::prelude::*;
10661190
use hyper::{Body, Request, StatusCode};
10671191

@@ -1084,17 +1208,35 @@ pub mod tests {
10841208
image: None,
10851209
from: None,
10861210
}),
1087-
#[assertion = "Container created, assigned an `id`"]
1211+
#[assertion = "Container created, attach network"]
1212+
Ok(Project::Attaching(ProjectAttaching {
1213+
container: ContainerInspectResponse {
1214+
state: Some(ContainerState {
1215+
status: Some(ContainerStateStatusEnum::CREATED),
1216+
..
1217+
}),
1218+
network_settings: Some(NetworkSettings {
1219+
networks: Some(networks),
1220+
..
1221+
}),
1222+
..
1223+
}
1224+
})) if networks.keys().collect::<Vec<_>>() == vec!["bridge"],
1225+
#[assertion = "Container attached, assigned an `id`"]
10881226
Ok(Project::Starting(ProjectStarting {
10891227
container: ContainerInspectResponse {
10901228
id: Some(container_id),
10911229
state: Some(ContainerState {
10921230
status: Some(ContainerStateStatusEnum::CREATED),
10931231
..
10941232
}),
1233+
network_settings: Some(NetworkSettings {
1234+
networks: Some(networks),
1235+
..
1236+
}),
10951237
..
10961238
}
1097-
})),
1239+
})) if networks.keys().collect::<Vec<_>>() == vec![&ctx.container_settings.network_name],
10981240
#[assertion = "Container started, in a running state"]
10991241
Ok(Project::Started(ProjectStarted {
11001242
container: ContainerInspectResponse {

0 commit comments

Comments
 (0)