Repository navigation
Add new multirack CRDB tables for unadopted racks and sleds #11548
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
39498f0
eb2ebaf
bdbe9dd
0b0ba42
b1bbed4
d3844c6
eede59c
42cf02d
324d548
8b69ba5
3989c3e
2a6ffe6
cefd279
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| -- Tracks other racks that have joined the underlay network but haven't been | ||
| -- adopted into the multirack cluster. | ||
| -- | ||
| -- If/when a rack is adopted, its row in this table should be marked deleted, | ||
| -- and a corresponding row should be inserted into the `rack` table. | ||
| CREATE TABLE IF NOT EXISTS omicron.public.unadopted_rack ( | ||
| -- This should be the same id that gets used for the `rack` table, as well | ||
| -- as the `rack_id` column in `unadopted_sled`: | ||
| id UUID PRIMARY KEY, | ||
|
|
||
| time_created TIMESTAMPTZ NOT NULL, | ||
| time_modified TIMESTAMPTZ NOT NULL, | ||
| time_deleted TIMESTAMPTZ, | ||
|
|
||
| -- The IPv6 underlay /56 prefix for the rack: | ||
| rack_subnet INET NOT NULL, | ||
|
|
||
| -- Underlay address of the particular sled-agent instance on the unadopted | ||
| -- rack that's running the multirack join service: | ||
| join_service_ip INET NOT NULL | ||
| ); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| -- Tracks sleds from other racks that have joined the underlay network but | ||
| -- haven't been adopted into the multirack cluster. | ||
| -- | ||
| -- If/when these sleds are adopted, these rows should be marked deleted, and | ||
| -- there will subsequently be corresponding rows created in the `sled` and | ||
| -- `inv_sled_agent` tables via the normal mechanisms. | ||
| CREATE TABLE IF NOT EXISTS omicron.public.unadopted_sled ( | ||
| id UUID PRIMARY KEY, | ||
| -- Foreign key into the `unadopted_rack` table (though, also, the rack ID | ||
| -- that gets used in the `rack` table should be the same if/when the rack | ||
| -- is adopted) | ||
| rack_id UUID NOT NULL, | ||
|
|
||
| time_created TIMESTAMPTZ NOT NULL, | ||
| time_modified TIMESTAMPTZ NOT NULL, | ||
| time_deleted TIMESTAMPTZ, | ||
|
|
||
| -- Baseboard information about the sled | ||
| serial_number STRING(63) NOT NULL, | ||
| part_number STRING(63) NOT NULL, | ||
|
|
||
| sled_agent_ip INET NOT NULL, | ||
|
|
||
| -- This tracks whether this particular sled's sled-agent has been | ||
| -- successfully started by the multirack join service: | ||
| sled_agent_started BOOL NOT NULL, | ||
| -- This nullable column should be set to the last error we've seen for this | ||
| -- particular sled's sled-agent startup process, if any: | ||
| last_start_error_seen TEXT, | ||
|
|
||
| CONSTRAINT start_error_is_null_if_started CHECK ( | ||
| (NOT sled_agent_started) OR (last_start_error_seen IS NULL) | ||
| ) | ||
| ); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| CREATE UNIQUE INDEX IF NOT EXISTS lookup_unadopted_rack_by_id ON omicron.public.unadopted_rack ( | ||
| id | ||
| ) WHERE time_deleted IS NULL; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| -- DO NOT EDIT. Generated by test_migration_verification_files. | ||
| SELECT CAST(IF((SELECT true WHERE EXISTS (SELECT index_name FROM omicron.crdb_internal.table_indexes WHERE descriptor_name = 'unadopted_rack' AND index_name = 'lookup_unadopted_rack_by_id')),'true','Schema change verification failed: index lookup_unadopted_rack_by_id on table unadopted_rack does not exist') AS BOOL); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| CREATE UNIQUE INDEX IF NOT EXISTS lookup_unadopted_sled_by_id ON omicron.public.unadopted_sled ( | ||
| id | ||
| ) WHERE time_deleted IS NULL; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| -- DO NOT EDIT. Generated by test_migration_verification_files. | ||
| SELECT CAST(IF((SELECT true WHERE EXISTS (SELECT index_name FROM omicron.crdb_internal.table_indexes WHERE descriptor_name = 'unadopted_sled' AND index_name = 'lookup_unadopted_sled_by_id')),'true','Schema change verification failed: index lookup_unadopted_sled_by_id on table unadopted_sled does not exist') AS BOOL); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9730,6 +9730,70 @@ CREATE TABLE IF NOT EXISTS omicron.public.fm_config ( | |
| ) | ||
| ); | ||
|
|
||
| -- Tracks other racks that have joined the underlay network but haven't been | ||
| -- adopted into the multirack cluster. | ||
| -- | ||
| -- If/when a rack is adopted, its row in this table should be marked deleted, | ||
| -- and a corresponding row should be inserted into the `rack` table. | ||
| CREATE TABLE IF NOT EXISTS omicron.public.unadopted_rack ( | ||
| -- This should be the same id that gets used for the `rack` table, as well | ||
| -- as the `rack_id` column in `unadopted_sled`: | ||
| id UUID PRIMARY KEY, | ||
|
|
||
| time_created TIMESTAMPTZ NOT NULL, | ||
| time_modified TIMESTAMPTZ NOT NULL, | ||
| time_deleted TIMESTAMPTZ, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's fine to defer this for now, but just a heads-up that when you start writing queries against this and want to filter out deleted rows, you'll probably nee a unique partial index (lots of other examples in here though).
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah cool, thanks for the pointer, I hadn't encountered/noticed that yet. Will add, either in this PR or the next one in the stack, but probably this one just to save the boilerplate of doing another migration later on. |
||
|
|
||
| -- The IPv6 underlay /56 prefix for the rack: | ||
| rack_subnet INET NOT NULL, | ||
|
|
||
| -- Underlay address of the particular sled-agent instance on the unadopted | ||
| -- rack that's running the multirack join service: | ||
| join_service_ip INET NOT NULL | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need a way to represent "the multirack join service failed"? I think in practice that means its rack will need to be clean-slated, which presumably means when it comes back it'll have a different ID and a new row in this table. But then how does the old one ever get deleted?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, we need to be able to do this. We should discuss at multirack huddle. |
||
| ); | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS lookup_unadopted_rack_by_id ON omicron.public.unadopted_rack ( | ||
| id | ||
| ) WHERE time_deleted IS NULL; | ||
|
|
||
| -- Tracks sleds from other racks that have joined the underlay network but | ||
| -- haven't been adopted into the multirack cluster. | ||
| -- | ||
| -- If/when these sleds are adopted, these rows should be marked deleted, and | ||
| -- there will subsequently be corresponding rows created in the `sled` and | ||
| -- `inv_sled_agent` tables via the normal mechanisms. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't believe that sled-agents will be added to the sled-table via any normal process. We'll need to put them there in the background task after receiving the completion response that sets the bootstore generation. I opened an issue for this earlier today.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I take it back. The sled-agent always upserts itself once it can locate a nexus. That might be fine. What I'm worried about is the sled getting added to the sled-table before we are ready - mainly before the bootstore generation for this rack is updated and the rack is present in the rack table.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah right, yeah, I remember originally we were thinking about having a separate parallel insertion path but I figured if we're not inserting I'm less familiar with the bootstore stuff, so not sure if that throws a wrench in things, but if we just needed to wait for a record to be inserted in the This is also getting pretty tangential to this PR though so we can probably discuss this later, maybe in the sync meeting tonight.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Or, well, I don't know. I published that comment and then immediately started questioning this. Maybe the initial inserts into
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it depends somewhat on how they find Nexus, which is DNS. Can they find DNS via the hardcoded addresses for the AZ? In any event you are right, we should discuss at huddle. I already added a note last night 😎 . |
||
| CREATE TABLE IF NOT EXISTS omicron.public.unadopted_sled ( | ||
| id UUID PRIMARY KEY, | ||
| -- Foreign key into the `unadopted_rack` table (though, also, the rack ID | ||
| -- that gets used in the `rack` table should be the same if/when the rack | ||
| -- is adopted) | ||
| rack_id UUID NOT NULL, | ||
|
|
||
| time_created TIMESTAMPTZ NOT NULL, | ||
| time_modified TIMESTAMPTZ NOT NULL, | ||
| time_deleted TIMESTAMPTZ, | ||
|
|
||
| -- Baseboard information about the sled | ||
| serial_number STRING(63) NOT NULL, | ||
| part_number STRING(63) NOT NULL, | ||
|
|
||
| sled_agent_ip INET NOT NULL, | ||
|
|
||
| -- This tracks whether this particular sled's sled-agent has been | ||
| -- successfully started by the multirack join service: | ||
| sled_agent_started BOOL NOT NULL, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I assume this is a "once it becomes true, we never set it to false again" column? (Wondering if there's any reason to guard against concurrent Nexuses operating on different views of a given sled-agent, but if we're going to "fuse" this column it's probably unnecessary. If we do end up wanting to record more varied states, we might need something?)
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah, once it's true we're done. |
||
| -- This nullable column should be set to the last error we've seen for this | ||
| -- particular sled's sled-agent startup process, if any: | ||
| last_start_error_seen TEXT, | ||
|
|
||
| CONSTRAINT start_error_is_null_if_started CHECK ( | ||
| (NOT sled_agent_started) OR (last_start_error_seen IS NULL) | ||
| ) | ||
| ); | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS lookup_unadopted_sled_by_id ON omicron.public.unadopted_sled ( | ||
| id | ||
| ) WHERE time_deleted IS NULL; | ||
|
|
||
| -- Keep this at the end of file so that the database does not contain a version | ||
| -- until it is fully populated. | ||
|
|
@@ -9740,7 +9804,7 @@ INSERT INTO omicron.public.db_metadata ( | |
| version, | ||
| target_version | ||
| ) VALUES | ||
| (TRUE, NOW(), NOW(), '309.0.0', NULL) | ||
| (TRUE, NOW(), NOW(), '310.0.0', NULL) | ||
| ON CONFLICT DO NOTHING; | ||
|
|
||
| COMMIT; | ||
Uh oh!
There was an error while loading. Please reload this page.