From 9073d00792ad119264b515cf5f2edfc62050177e Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 24 Apr 2024 10:28:17 -0700 Subject: [PATCH 01/15] refactor finding resources and adapted resources, update list to use --- dsc/src/subcommand.rs | 14 +- dsc_lib/src/configure/mod.rs | 8 +- dsc_lib/src/discovery/command_discovery.rs | 268 +++++++++++++-------- dsc_lib/src/discovery/discovery_trait.rs | 6 +- dsc_lib/src/discovery/mod.rs | 12 +- dsc_lib/src/dscerror.rs | 3 + dsc_lib/src/lib.rs | 4 +- 7 files changed, 200 insertions(+), 115 deletions(-) diff --git a/dsc/src/subcommand.rs b/dsc/src/subcommand.rs index 2d153c39..df75c403 100644 --- a/dsc/src/subcommand.rs +++ b/dsc/src/subcommand.rs @@ -337,7 +337,7 @@ pub fn validate_config(config: &str) -> Result<(), DscError> { resource_types.push(type_name.to_lowercase().to_string()); } - dsc.discover_resources(&resource_types); + dsc.find_resources(&resource_types); for resource_block in resources { let Some(type_name) = resource_block["type"].as_str() else { @@ -402,15 +402,15 @@ pub fn resource(subcommand: &ResourceSubCommand, stdin: &Option) { list_resources(&mut dsc, resource_name, adapter_name, description, tags, format); }, ResourceSubCommand::Schema { resource , format } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); resource_command::schema(&dsc, resource, format); }, ResourceSubCommand::Export { resource, format } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); resource_command::export(&mut dsc, resource, format); }, ResourceSubCommand::Get { resource, input, path, all, format } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); if *all { resource_command::get_all(&dsc, resource, format); } else { let parsed_input = get_input(input, stdin, path); @@ -418,17 +418,17 @@ pub fn resource(subcommand: &ResourceSubCommand, stdin: &Option) { } }, ResourceSubCommand::Set { resource, input, path, format } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); let parsed_input = get_input(input, stdin, path); resource_command::set(&dsc, resource, parsed_input, format); }, ResourceSubCommand::Test { resource, input, path, format } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); let parsed_input = get_input(input, stdin, path); resource_command::test(&dsc, resource, parsed_input, format); }, ResourceSubCommand::Delete { resource, input, path } => { - dsc.discover_resources(&[resource.to_lowercase().to_string()]); + dsc.find_resources(&[resource.to_lowercase().to_string()]); let parsed_input = get_input(input, stdin, path); resource_command::delete(&dsc, resource, parsed_input); }, diff --git a/dsc_lib/src/configure/mod.rs b/dsc_lib/src/configure/mod.rs index c2132faf..d4ee0d9c 100644 --- a/dsc_lib/src/configure/mod.rs +++ b/dsc_lib/src/configure/mod.rs @@ -335,7 +335,7 @@ impl Configurator { resource_type: resource.resource_type.clone(), result: set_result, }; - result.results.push(resource_result); + result.results.push(resource_result); } else if dsc_resource.capabilities.contains(&Capability::Delete) { debug!("Resource implements delete and _exist is false"); let before_result = dsc_resource.get(&desired)?; @@ -343,10 +343,10 @@ impl Configurator { dsc_resource.delete(&desired)?; let end_datetime = chrono::Local::now(); let after_result = dsc_resource.get(&desired)?; - // convert get result to set result + // convert get result to set result let set_result = match before_result { GetResult::Resource(before_response) => { - let GetResult::Resource(after_result) = after_result else { + let GetResult::Resource(after_result) = after_result else { return Err(DscError::NotSupported("Group resources not supported for delete".to_string())) }; let before_value = serde_json::to_value(&before_response.actual_state)?; @@ -614,7 +614,7 @@ impl Configurator { let mut required_resources = config.resources.iter().map(|p| p.resource_type.to_lowercase()).collect::>(); required_resources.sort_unstable(); required_resources.dedup(); - self.discovery.discover_resources(&required_resources); + self.discovery.find_resources(&required_resources); Ok(config) } diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 5adf163a..3069d4dc 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -10,21 +10,29 @@ use crate::dscresources::command_resource::log_resource_traces; use crate::dscerror::DscError; use indicatif::ProgressStyle; use regex::RegexBuilder; +use semver::Version; use std::collections::{BTreeMap, HashSet}; use std::env; use std::ffi::OsStr; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; -use tracing::{debug, error, trace, warn, warn_span, Span}; +use tracing::{debug, error, info, trace, warn, warn_span, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; pub struct CommandDiscovery { + // use BTreeMap so that the results are sorted + resources: BTreeMap>, + adapters: BTreeMap>, + adapted_resources: BTreeMap>, } impl CommandDiscovery { pub fn new() -> CommandDiscovery { CommandDiscovery { + resources: BTreeMap::new(), + adapters: BTreeMap::new(), + adapted_resources: BTreeMap::new(), } } @@ -76,64 +84,74 @@ impl Default for CommandDiscovery { impl ResourceDiscovery for CommandDiscovery { - #[allow(clippy::too_many_lines)] - fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result, DscError> { + fn discover_resources(&mut self, filter: &str) -> Result<(), DscError> { + if !self.resources.is_empty() || !self.adapters.is_empty() { + return Ok(()); + } + + info!("Discovering resources using filter: {filter}"); - debug!("Listing resources with type_name_filter/adapter_name_filter: {type_name_filter}/{adapter_name_filter}"); + let regex_str = convert_wildcard_to_regex(filter); + debug!("Using regex {regex_str} as filter for adapter name"); + let mut regex_builder = RegexBuilder::new(®ex_str); + regex_builder.case_insensitive(true); + let Ok(regex) = regex_builder.build() else { + return Err(DscError::Operation("Could not build Regex filter for adapter name".to_string())); + }; let pb_span = warn_span!(""); pb_span.pb_set_style(&ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise:.cyan}] {msg:.yellow}" + "{spinner:.green} [{elapsed_precise:.cyan}] [{bar:40.cyan/blue}] {pos:>7}/{len:7} {msg:.yellow}" )?); pb_span.pb_set_message("Searching for resources"); let _ = pb_span.enter(); - let mut resources: BTreeMap = BTreeMap::new(); - let mut adapter_resources: BTreeMap = BTreeMap::new(); - - let regex_str = convert_wildcard_to_regex(type_name_filter); - debug!("Using regex {regex_str} as filter for resource type"); - let mut regex_builder = RegexBuilder::new(®ex_str); - regex_builder.case_insensitive(true); - let Ok(type_regex) = regex_builder.build() else { - let err_str = "Could not build Regex filter for resource type"; - error!(err_str); - return Err(DscError::Operation(err_str.to_string())); - }; + let mut resources = BTreeMap::>::new(); + let mut adapters = BTreeMap::>::new(); if let Ok(paths) = CommandDiscovery::get_resource_paths() { for path in paths { - debug!("Searching in {:?}", path); + trace!("Searching in {:?}", path); if path.exists() && path.is_dir() { for entry in path.read_dir().unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.is_file() { - let file_name = path.file_name().unwrap().to_str().unwrap(); + let Some(os_file_name) = path.file_name() else { + // skip if not a file + continue; + }; + let Some(file_name) = os_file_name.to_str() else { + // skip if not a valid file name + continue; + }; let file_name_lowercase = file_name.to_lowercase(); if file_name_lowercase.ends_with(".dsc.resource.json") || - file_name_lowercase.ends_with(".dsc.resource.yaml") || - file_name_lowercase.ends_with(".dsc.resource.yml") { + file_name_lowercase.ends_with(".dsc.resource.yaml") || + file_name_lowercase.ends_with(".dsc.resource.yml") { let resource = match load_manifest(&path) { Ok(r) => r, Err(e) => { - // In case of "resource list" operation - print all failures to read manifests as warnings - warn!("{}", e); + // At this point we can't determine whether or not the bad manifest contains + // resource that is requested by resource/config operation + // if it is, then "ResouceNotFound" error will be issued later + // and here we just record the error into debug stream. + debug!("{e}"); continue; }, }; - if let Some(ref manifest) = resource.manifest { - let manifest = import_manifest(manifest.clone())?; - if manifest.kind == Some(Kind::Adapter) { - adapter_resources.insert(resource.type_name.to_lowercase(),resource.clone()); + if regex.is_match(filter) { + if let Some(ref manifest) = resource.manifest { + let manifest = import_manifest(manifest.clone())?; + if manifest.kind == Some(Kind::Adapter) { + insert_resource(&mut adapters, &resource)?; + } else { + insert_resource(&mut resources, &resource)?; + } } } - - if adapter_name_filter.is_empty() && type_regex.is_match(&resource.type_name) { - resources.insert(resource.type_name.to_lowercase(), resource); - } } } } @@ -141,86 +159,122 @@ impl ResourceDiscovery for CommandDiscovery { } } debug!("Found {} matching non-adapter-based resources", resources.len()); + self.resources = resources; + self.adapters = adapters; + Ok(()) + } - if !adapter_name_filter.is_empty() { - let regex_str = convert_wildcard_to_regex(adapter_name_filter); - debug!("Using regex {regex_str} as filter for adapter name"); - let mut regex_builder = RegexBuilder::new(®ex_str); - regex_builder.case_insensitive(true); - let Ok(adapter_regex) = regex_builder.build() else { - let err_str = "Could not build Regex filter for adapter name"; - error!(err_str); - return Err(DscError::Operation(err_str.to_string())); - }; + fn discover_adapted_resources(&mut self, filter: &str) -> Result<(), DscError> { + if self.resources.is_empty() && self.adapters.is_empty() { + self.discover_resources("*")?; + } - // now go through the adapter resources and add them to the list of resources - for adapter in adapter_resources { - if adapter_regex.is_match(&adapter.1.type_name) { - debug!("Enumerating resources for adapter '{}'", adapter.1.type_name); - let pb_adapter_span = warn_span!(""); - pb_adapter_span.pb_set_style(&ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise:.cyan}] {msg:.white}" - )?); - pb_adapter_span.pb_set_message(format!("Enumerating resources for adapter '{}'", adapter.1.type_name).as_str()); - let _ = pb_adapter_span.enter(); - let adapter_resource = adapter.1; - let adapter_type_name = adapter_resource.type_name.clone(); - let manifest = if let Some(manifest) = adapter_resource.manifest { - if let Ok(manifest) = import_manifest(manifest) { - manifest - } else { - return Err(DscError::Operation(format!("Failed to import manifest for '{}'", adapter_resource.type_name.clone()))); - } - } else { - return Err(DscError::MissingManifest(adapter_resource.type_name.clone())); - }; - let mut adapter_resources_count = 0; - // invoke the list command - let list_command = manifest.adapter.unwrap().list; - let (exit_code, stdout, stderr) = match invoke_command(&list_command.executable, list_command.args, None, Some(&adapter_resource.directory), None) - { - Ok((exit_code, stdout, stderr)) => (exit_code, stdout, stderr), - Err(e) => { - // In case of "resource list" operation - print failure from adapter as warning - warn!("Could not start {}: {}", list_command.executable, e); - continue; - }, - }; - log_resource_traces(&stderr); + if self.adapters.is_empty() { + return Ok(()); + } + + let regex_str = convert_wildcard_to_regex(filter); + debug!("Using regex {regex_str} as filter for adapter name"); + let mut regex_builder = RegexBuilder::new(®ex_str); + regex_builder.case_insensitive(true); + let Ok(regex) = regex_builder.build() else { + return Err(DscError::Operation("Could not build Regex filter for adapter name".to_string())); + }; - if exit_code != 0 { - // In case of "resource list" operation - print failure from adapter as warning - warn!("Adapter failed to list resources with exit code {exit_code}: {stderr}"); + let pb_span = warn_span!(""); + pb_span.pb_set_style(&ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise:.cyan}] [{bar:40.cyan/blue}] {pos:>7}/{len:7} {msg:.yellow}" + )?); + pb_span.pb_set_message("Searching for adapted resources"); + let _ = pb_span.enter(); + + let mut adapted_resources = BTreeMap::>::new(); + + for (adapter_name, adapters) in self.adapters.iter() { + for adapter in adapters { + info!("Enumerating resources for adapter '{}'", adapter_name); + let pb_adapter_span = warn_span!(""); + pb_adapter_span.pb_set_style(&ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise:.cyan}] {msg:.white}" + )?); + pb_adapter_span.pb_set_message(format!("Enumerating resources for adapter '{}'", adapter_name).as_str()); + let _ = pb_adapter_span.enter(); + let manifest = if let Some(manifest) = &adapter.manifest { + if let Ok(manifest) = import_manifest(manifest.clone()) { + manifest + } else { + return Err(DscError::Operation(format!("Failed to import manifest for '{}'", adapter_name.clone()))); } + } else { + return Err(DscError::MissingManifest(adapter_name.clone())); + }; - for line in stdout.lines() { - match serde_json::from_str::(line){ - Result::Ok(resource) => { - if resource.require_adapter.is_none() { - warn!("{}", DscError::MissingRequires(adapter.0.clone(), resource.type_name.clone()).to_string()); - continue; - } - if type_regex.is_match(&resource.type_name) { - resources.insert(resource.type_name.to_lowercase(), resource); - adapter_resources_count += 1; - } - }, - Result::Err(err) => { - warn!("Failed to parse resource: {line} -> {err}"); + let mut adapter_resources_count = 0; + // invoke the list command + let list_command = manifest.adapter.unwrap().list; + let (exit_code, stdout, stderr) = match invoke_command(&list_command.executable, list_command.args, None, Some(&adapter.directory), None) + { + Ok((exit_code, stdout, stderr)) => (exit_code, stdout, stderr), + Err(e) => { + // In case of error, log and continue + warn!("Could not start {}: {}", list_command.executable, e); + continue; + }, + }; + log_resource_traces(&stderr); + + if exit_code != 0 { + // in case of failure, log and continue + warn!("Adapter failed to list resources with exit code {exit_code}: {stderr}"); + continue; + } + + for line in stdout.lines() { + match serde_json::from_str::(line){ + Result::Ok(resource) => { + if resource.require_adapter.is_none() { + warn!("{}", DscError::MissingRequires(adapter_name.clone(), resource.type_name.clone()).to_string()); continue; } - }; - } - debug!("Adapter '{}' listed {} matching resources", adapter_type_name, adapter_resources_count); + if regex.is_match(&resource.type_name) { + insert_resource(&mut adapted_resources, &resource)?; + adapter_resources_count += 1; + } + }, + Result::Err(err) => { + warn!("Failed to parse resource: {line} -> {err}"); + continue; + } + }; } + + debug!("Adapter '{}' listed {} resources", adapter_name, adapter_resources_count); } } + + self.adapted_resources = adapted_resources; + Ok(()) + } + + fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { + + trace!("Listing resources with type_name_filter/adapter_name_filter: {type_name_filter}/{adapter_name_filter}"); + + if !adapter_name_filter.is_empty() { + self.discover_adapted_resources(adapter_name_filter)?; + } + + let mut resources = BTreeMap::>::new(); + + resources.append(&mut self.resources); + resources.append(&mut self.adapted_resources); + Ok(resources) } #[allow(clippy::too_many_lines)] - fn discover_resources(&mut self, required_resource_types: &[String]) -> Result, DscError> + fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError> { debug!("Searching for resources: {:?}", required_resource_types); @@ -360,6 +414,28 @@ impl ResourceDiscovery for CommandDiscovery { } } +// helper to insert a resource into a vector of resources in order of newest to oldest +fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource) -> Result<(), DscError> { + if resources.contains_key(&resource.type_name) { + let Some(resource_versions) = resources.get_mut(&resource.type_name) else { + resources.insert(resource.type_name.clone(), vec![resource.clone()]); + return Ok(()); + }; + // compare the resource versions and insert newest to oldest using semver + let mut insert_index = resource_versions.len(); + for (index, resource_version) in resource_versions.iter().enumerate() { + if Version::parse(&resource_version.version)? < Version::parse(&resource.version)? { + insert_index = index; + break; + } + } + resource_versions.insert(insert_index, resource.clone()); + } else { + resources.insert(resource.type_name.clone(), vec![resource.clone()]); + } + Ok(()) +} + fn load_manifest(path: &Path) -> Result { let file = File::open(path)?; let reader = BufReader::new(file); diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index 4050d335..7681ade5 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -5,6 +5,8 @@ use crate::{dscresources::dscresource::DscResource, dscerror::DscError}; use std::collections::BTreeMap; pub trait ResourceDiscovery { - fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result, DscError>; - fn discover_resources(&mut self, required_resource_types: &[String]) -> Result, DscError>; + fn discover_resources(&mut self, filter: &str) -> Result<(), DscError>; + fn discover_adapted_resources(&mut self, filter: &str) -> Result<(), DscError>; + fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; + fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError>; } diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index 9a960d7c..0dd17da2 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -45,8 +45,12 @@ impl Discovery { } }; - for resource in discovered_resources { - resources.push(resource.1); + for (_resource_name, resource) in discovered_resources { + let Some(resource) = resource.first() else { + continue; + }; + + resources.push(resource.clone()); }; } @@ -58,7 +62,7 @@ impl Discovery { self.resources.get(type_name) } - pub fn discover_resources(&mut self, required_resource_types: &[String]) { + pub fn find_resources(&mut self, required_resource_types: &[String]) { let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new()), @@ -67,7 +71,7 @@ impl Discovery { let mut remaining_required_resource_types = required_resource_types.to_owned(); for mut discovery_type in discovery_types { - let discovered_resources = match discovery_type.discover_resources(&remaining_required_resource_types) { + let discovered_resources = match discovery_type.find_resources(&remaining_required_resource_types) { Ok(value) => value, Err(err) => { error!("{err}"); diff --git a/dsc_lib/src/dscerror.rs b/dsc_lib/src/dscerror.rs index 5e5ae66f..3cbcbffd 100644 --- a/dsc_lib/src/dscerror.rs +++ b/dsc_lib/src/dscerror.rs @@ -98,6 +98,9 @@ pub enum DscError { #[error("Security context: {0}")] SecurityContext(String), + #[error("Semver: {0}")] + Semver(#[from] semver::Error), + #[error("Utf-8 conversion error: {0}")] Utf8Conversion(#[from] Utf8Error), diff --git a/dsc_lib/src/lib.rs b/dsc_lib/src/lib.rs index d871952f..743e6b1c 100644 --- a/dsc_lib/src/lib.rs +++ b/dsc_lib/src/lib.rs @@ -43,8 +43,8 @@ impl DscManager { self.discovery.list_available_resources(type_name_filter, adapter_name_filter) } - pub fn discover_resources(&mut self, required_resource_types: &[String]) { - self.discovery.discover_resources(required_resource_types); + pub fn find_resources(&mut self, required_resource_types: &[String]) { + self.discovery.find_resources(required_resource_types); } /// Invoke the get operation on a resource. /// From d86df6d8d84acdacb7e5626a70c20c0bb6e1958e Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 24 Apr 2024 15:13:11 -0700 Subject: [PATCH 02/15] refactor resource discovery code to use new helper functions --- dsc_lib/src/discovery/command_discovery.rs | 165 +++++---------------- dsc_lib/src/discovery/mod.rs | 11 +- 2 files changed, 44 insertions(+), 132 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 3069d4dc..c5737411 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -17,7 +17,7 @@ use std::ffi::OsStr; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; -use tracing::{debug, error, info, trace, warn, warn_span, Span}; +use tracing::{debug, info, trace, warn, warn_span}; use tracing_indicatif::span_ext::IndicatifSpanExt; pub struct CommandDiscovery { @@ -273,144 +273,52 @@ impl ResourceDiscovery for CommandDiscovery { Ok(resources) } - #[allow(clippy::too_many_lines)] + // TODO: handle version requirements fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError> { debug!("Searching for resources: {:?}", required_resource_types); + self.discover_resources("*")?; - let pb_span = warn_span!(""); - pb_span.pb_set_style(&ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise:.cyan}] [{bar:40.cyan/blue}] {pos:>7}/{len:7} {msg:.yellow}" - )?); - pb_span.pb_set_message("Searching for resources"); - let _ = pb_span.enter(); - - let mut resources: BTreeMap = BTreeMap::new(); - let mut adapter_resources: BTreeMap = BTreeMap::new(); + let mut found_resources = BTreeMap::::new(); let mut remaining_required_resource_types = required_resource_types.to_owned(); - if let Ok(paths) = CommandDiscovery::get_resource_paths() { - for path in paths { - debug!("Searching in {:?}", path); - if path.exists() && path.is_dir() { - for entry in path.read_dir().unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if path.is_file() { - let file_name = path.file_name().unwrap().to_str().unwrap(); - let file_name_lowercase = file_name.to_lowercase(); - if file_name_lowercase.ends_with(".dsc.resource.json") || - file_name_lowercase.ends_with(".dsc.resource.yaml") || - file_name_lowercase.ends_with(".dsc.resource.yml") { - let resource = match load_manifest(&path) - { - Ok(r) => r, - Err(e) => { - /* In case of non-list resource/config operations: - At this point we can't determine whether or not the bad manifest contains resource that is requested by resource/config operation - if it is, then "ResouceNotFound" error will be issued later - and here we just record the error into debug stream.*/ - debug!("{}", e); - continue; - }, - }; + for (resource_name, resources) in self.resources.iter() { + let Some(resource ) = resources.first() else { + // skip if no resources + continue; + }; - if let Some(ref manifest) = resource.manifest { - let manifest = import_manifest(manifest.clone())?; - if manifest.kind == Some(Kind::Adapter) { - adapter_resources.insert(resource.type_name.to_lowercase(), resource.clone()); - resources.insert(resource.type_name.to_lowercase(), resource.clone()); - } - } - if remaining_required_resource_types.contains(&resource.type_name.to_lowercase()) - { - remaining_required_resource_types.retain(|x| *x != resource.type_name.to_lowercase()); - debug!("Found {} in {}", &resource.type_name, path.display()); - Span::current().pb_inc(1); - resources.insert(resource.type_name.to_lowercase(), resource); - if remaining_required_resource_types.is_empty() - { - return Ok(resources); - } - } - } - } - } + if remaining_required_resource_types.contains(&resource_name.to_lowercase()) + { + // remove the resource from the list of required resources + remaining_required_resource_types.retain(|x| *x != resource_name.to_lowercase()); + found_resources.insert(resource_name.to_lowercase(), resource.clone()); + if remaining_required_resource_types.is_empty() + { + return Ok(found_resources); } } } - debug!("Found {} matching non-adapter-based resources", resources.len()); + debug!("Found {} matching non-adapter-based resources", found_resources.len()); // now go through the adapter resources and add them to the list of resources - for adapter in adapter_resources { - debug!("Enumerating resources for adapter '{}'", adapter.1.type_name); - let pb_adapter_span = warn_span!(""); - pb_adapter_span.pb_set_style(&ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise:.cyan}] {msg:.white}" - )?); - pb_adapter_span.pb_set_message(format!("Enumerating resources for adapter '{}'", adapter.1.type_name).as_str()); - let _ = pb_adapter_span.enter(); - let adapter_resource = adapter.1; - let adapter_type_name = adapter_resource.type_name.clone(); - let manifest = if let Some(manifest) = adapter_resource.manifest { - if let Ok(manifest) = import_manifest(manifest) { - manifest - } else { - return Err(DscError::Operation(format!("Failed to import manifest for '{}'", adapter_resource.type_name.clone()))); - } - } else { - return Err(DscError::MissingManifest(adapter_resource.type_name.clone())); + for (adapted_name, adapted_resource) in self.adapted_resources.iter() { + let Some(adapted_resource) = adapted_resource.first() else { + // skip if no resources + continue; }; - let mut adapter_resources_count = 0; - // invoke the list command - let list_command = manifest.adapter.unwrap().list; - let (exit_code, stdout, stderr) = match invoke_command(&list_command.executable, list_command.args, None, Some(&adapter_resource.directory), None) - { - Ok((exit_code, stdout, stderr)) => (exit_code, stdout, stderr), - Err(e) => { - /* In case of non-list resource/config operations: - print failure from adapter as error because this adapter was specifically requested by current resource/config operation*/ - error!("Could not start {}: {}", list_command.executable, e); - continue; - }, - }; - log_resource_traces(&stderr); - if exit_code != 0 { - /* In case of non-list resource/config operations: - print failure from adapter as error because this adapter was specifically requested by current resource/config operation*/ - error!("Adapter failed to list resources with exit code {exit_code}: {stderr}"); - } - - for line in stdout.lines() { - match serde_json::from_str::(line){ - Result::Ok(resource) => { - if resource.require_adapter.is_none() { - error!("{}", DscError::MissingRequires(adapter.0.clone(), resource.type_name.clone()).to_string()); - continue; - } - if remaining_required_resource_types.contains(&resource.type_name.to_lowercase()) - { - remaining_required_resource_types.retain(|x| *x != resource.type_name.to_lowercase()); - debug!("Found {} in {}", &resource.type_name, &resource.path); - resources.insert(resource.type_name.to_lowercase(), resource); - adapter_resources_count += 1; - if remaining_required_resource_types.is_empty() - { - return Ok(resources); - } - } - }, - Result::Err(err) => { - error!("Failed to parse resource: {line} -> {err}"); - continue; - } - }; + if remaining_required_resource_types.contains(&adapted_name.to_lowercase()) + { + remaining_required_resource_types.retain(|x| *x != adapted_name.to_lowercase()); + found_resources.insert(adapted_name.to_lowercase(), adapted_resource.clone()); + if remaining_required_resource_types.is_empty() + { + return Ok(found_resources); + } } - - debug!("Adapter '{}' listed {} matching resources", adapter_type_name, adapter_resources_count); } - Ok(resources) + Ok(found_resources) } } @@ -423,8 +331,15 @@ fn insert_resource(resources: &mut BTreeMap>, resource: }; // compare the resource versions and insert newest to oldest using semver let mut insert_index = resource_versions.len(); - for (index, resource_version) in resource_versions.iter().enumerate() { - if Version::parse(&resource_version.version)? < Version::parse(&resource.version)? { + for (index, resource_instance) in resource_versions.iter().enumerate() { + let resource_instance_version = Version::parse(&resource_instance.version)?; + let resource_version = Version::parse(&resource.version)?; + // if the version already exists, we skip + if resource_instance_version == resource_version { + return Ok(()); + } + + if resource_instance_version < resource_version { insert_index = index; break; } diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index 0dd17da2..ce4d417f 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -27,7 +27,6 @@ impl Discovery { } /// List operation. - #[allow(clippy::missing_panics_doc)] // false positive in clippy; this function will never panic pub fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Vec { let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new()), @@ -45,12 +44,10 @@ impl Discovery { } }; - for (_resource_name, resource) in discovered_resources { - let Some(resource) = resource.first() else { - continue; - }; - - resources.push(resource.clone()); + for (_resource_name, found_resources) in discovered_resources { + for resource in found_resources { + resources.push(resource.clone()); + } }; } From 526d62b2bf8f4184b16cf67a6ad17562a97d0822 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 24 Apr 2024 15:32:02 -0700 Subject: [PATCH 03/15] fix clippy --- dsc_lib/src/discovery/command_discovery.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index c5737411..8892d767 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -190,14 +190,14 @@ impl ResourceDiscovery for CommandDiscovery { let mut adapted_resources = BTreeMap::>::new(); - for (adapter_name, adapters) in self.adapters.iter() { + for (adapter_name, adapters) in &self.adapters { for adapter in adapters { info!("Enumerating resources for adapter '{}'", adapter_name); let pb_adapter_span = warn_span!(""); pb_adapter_span.pb_set_style(&ProgressStyle::with_template( "{spinner:.green} [{elapsed_precise:.cyan}] {msg:.white}" )?); - pb_adapter_span.pb_set_message(format!("Enumerating resources for adapter '{}'", adapter_name).as_str()); + pb_adapter_span.pb_set_message(format!("Enumerating resources for adapter '{adapter_name}'").as_str()); let _ = pb_adapter_span.enter(); let manifest = if let Some(manifest) = &adapter.manifest { if let Ok(manifest) = import_manifest(manifest.clone()) { @@ -282,7 +282,7 @@ impl ResourceDiscovery for CommandDiscovery { let mut found_resources = BTreeMap::::new(); let mut remaining_required_resource_types = required_resource_types.to_owned(); - for (resource_name, resources) in self.resources.iter() { + for (resource_name, resources) in &self.resources { let Some(resource ) = resources.first() else { // skip if no resources continue; @@ -302,7 +302,7 @@ impl ResourceDiscovery for CommandDiscovery { debug!("Found {} matching non-adapter-based resources", found_resources.len()); // now go through the adapter resources and add them to the list of resources - for (adapted_name, adapted_resource) in self.adapted_resources.iter() { + for (adapted_name, adapted_resource) in &self.adapted_resources { let Some(adapted_resource) = adapted_resource.first() else { // skip if no resources continue; From afe994c419b25c1d3b08ec6441edf2be999bf1b2 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 16:45:32 -0700 Subject: [PATCH 04/15] force discover for list --- dsc_lib/src/discovery/command_discovery.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 8892d767..d152ea68 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -261,6 +261,8 @@ impl ResourceDiscovery for CommandDiscovery { trace!("Listing resources with type_name_filter/adapter_name_filter: {type_name_filter}/{adapter_name_filter}"); + self.discover_resources(type_name_filter)?; + if !adapter_name_filter.is_empty() { self.discover_adapted_resources(adapter_name_filter)?; } From 7ee09eff64604a7e999a4e63bf91002657c58af1 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 17:47:33 -0700 Subject: [PATCH 05/15] fix listing adapted resources --- build.ps1 | 7 ++++- dsc_lib/src/discovery/command_discovery.rs | 31 +++++++++++++++++----- dsc_lib/src/discovery/discovery_trait.rs | 2 +- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/build.ps1 b/build.ps1 index f186ed7f..268e727f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -161,6 +161,8 @@ if (!$SkipBuild) { # make sure dependencies are built first so clippy runs correctly $windows_projects = @("pal", "registry", "reboot_pending", "wmi-adapter") + $macOS_projects = @("resources/brew") + # projects are in dependency order $projects = @( "tree-sitter-dscexpression", @@ -170,7 +172,6 @@ if (!$SkipBuild) { "osinfo", "powershell-adapter", "process", - "resources/brew", "runcommandonset", "tools/dsctest", "tools/test_group_resource", @@ -187,6 +188,10 @@ if (!$SkipBuild) { Get-ChildItem -Path $target -Recurse -Hidden | ForEach-Object { $_.Attributes = 'Normal' } } + if ($IsMacOS) { + $projects += $macOS_projects + } + $failed = $false foreach ($project in $projects) { ## Build format_json diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index d152ea68..a533d587 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -129,6 +129,7 @@ impl ResourceDiscovery for CommandDiscovery { if file_name_lowercase.ends_with(".dsc.resource.json") || file_name_lowercase.ends_with(".dsc.resource.yaml") || file_name_lowercase.ends_with(".dsc.resource.yml") { + trace!("Found resource manifest: {path:?}"); let resource = match load_manifest(&path) { Ok(r) => r, @@ -146,8 +147,10 @@ impl ResourceDiscovery for CommandDiscovery { if let Some(ref manifest) = resource.manifest { let manifest = import_manifest(manifest.clone())?; if manifest.kind == Some(Kind::Adapter) { + trace!("Resource adapter {} found", resource.type_name); insert_resource(&mut adapters, &resource)?; } else { + trace!("Resource {} found", resource.type_name); insert_resource(&mut resources, &resource)?; } } @@ -164,7 +167,7 @@ impl ResourceDiscovery for CommandDiscovery { Ok(()) } - fn discover_adapted_resources(&mut self, filter: &str) -> Result<(), DscError> { + fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError> { if self.resources.is_empty() && self.adapters.is_empty() { self.discover_resources("*")?; } @@ -173,7 +176,7 @@ impl ResourceDiscovery for CommandDiscovery { return Ok(()); } - let regex_str = convert_wildcard_to_regex(filter); + let regex_str = convert_wildcard_to_regex(adapter_filter); debug!("Using regex {regex_str} as filter for adapter name"); let mut regex_builder = RegexBuilder::new(®ex_str); regex_builder.case_insensitive(true); @@ -181,6 +184,14 @@ impl ResourceDiscovery for CommandDiscovery { return Err(DscError::Operation("Could not build Regex filter for adapter name".to_string())); }; + let name_regex_str = convert_wildcard_to_regex(name_filter); + debug!("Using regex {name_regex_str} as filter for resource name"); + let mut name_regex_builder = RegexBuilder::new(&name_regex_str); + name_regex_builder.case_insensitive(true); + let Ok(name_regex) = name_regex_builder.build() else { + return Err(DscError::Operation("Could not build Regex filter for resource name".to_string())); + }; + let pb_span = warn_span!(""); pb_span.pb_set_style(&ProgressStyle::with_template( "{spinner:.green} [{elapsed_precise:.cyan}] [{bar:40.cyan/blue}] {pos:>7}/{len:7} {msg:.yellow}" @@ -192,6 +203,10 @@ impl ResourceDiscovery for CommandDiscovery { for (adapter_name, adapters) in &self.adapters { for adapter in adapters { + if !regex.is_match(&adapter_name) { + continue; + } + info!("Enumerating resources for adapter '{}'", adapter_name); let pb_adapter_span = warn_span!(""); pb_adapter_span.pb_set_style(&ProgressStyle::with_template( @@ -237,7 +252,7 @@ impl ResourceDiscovery for CommandDiscovery { continue; } - if regex.is_match(&resource.type_name) { + if name_regex.is_match(&resource.type_name) { insert_resource(&mut adapted_resources, &resource)?; adapter_resources_count += 1; } @@ -264,13 +279,17 @@ impl ResourceDiscovery for CommandDiscovery { self.discover_resources(type_name_filter)?; if !adapter_name_filter.is_empty() { - self.discover_adapted_resources(adapter_name_filter)?; + self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; } let mut resources = BTreeMap::>::new(); - resources.append(&mut self.resources); - resources.append(&mut self.adapted_resources); + if adapter_name_filter.is_empty() { + resources.append(&mut self.resources); + resources.append(&mut self.adapters); + } else { + resources.append(&mut self.adapted_resources); + } Ok(resources) } diff --git a/dsc_lib/src/discovery/discovery_trait.rs b/dsc_lib/src/discovery/discovery_trait.rs index 7681ade5..a6eb226f 100644 --- a/dsc_lib/src/discovery/discovery_trait.rs +++ b/dsc_lib/src/discovery/discovery_trait.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; pub trait ResourceDiscovery { fn discover_resources(&mut self, filter: &str) -> Result<(), DscError>; - fn discover_adapted_resources(&mut self, filter: &str) -> Result<(), DscError>; + fn discover_adapted_resources(&mut self, name_filter: &str, adapter_filter: &str) -> Result<(), DscError>; fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError>; fn find_resources(&mut self, required_resource_types: &[String]) -> Result, DscError>; } From 18c925494a9647dd96f2de531a56c9d4e003b263 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Wed, 24 Apr 2024 17:58:49 -0700 Subject: [PATCH 06/15] fix clippy --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index a533d587..961b708f 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -203,7 +203,7 @@ impl ResourceDiscovery for CommandDiscovery { for (adapter_name, adapters) in &self.adapters { for adapter in adapters { - if !regex.is_match(&adapter_name) { + if !regex.is_match(adapter_name) { continue; } From 12020025b7d51e97d6e79eedf377e009a6a91ca0 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 19:40:13 -0700 Subject: [PATCH 07/15] fix use of filtering on type --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 961b708f..eb3a495f 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -143,7 +143,7 @@ impl ResourceDiscovery for CommandDiscovery { }, }; - if regex.is_match(filter) { + if regex.is_match(&resource.type_name) { if let Some(ref manifest) = resource.manifest { let manifest = import_manifest(manifest.clone())?; if manifest.kind == Some(Kind::Adapter) { From 23799e841914eaea06322de8614528a229d076ef Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 20:11:59 -0700 Subject: [PATCH 08/15] fix writing out warning --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index eb3a495f..03c06bcc 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -138,7 +138,7 @@ impl ResourceDiscovery for CommandDiscovery { // resource that is requested by resource/config operation // if it is, then "ResouceNotFound" error will be issued later // and here we just record the error into debug stream. - debug!("{e}"); + warn!("{e}"); continue; }, }; From e67dfc0c527ad11d4ac78cf3b3195ff34fc2a0f3 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 20:12:45 -0700 Subject: [PATCH 09/15] update comment --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 03c06bcc..894319f3 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -137,7 +137,7 @@ impl ResourceDiscovery for CommandDiscovery { // At this point we can't determine whether or not the bad manifest contains // resource that is requested by resource/config operation // if it is, then "ResouceNotFound" error will be issued later - // and here we just record the error into debug stream. + // and here we just write as warning warn!("{e}"); continue; }, From 2071b203beda151a9abae2cb1ff959cb35b7c1a9 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 21:03:49 -0700 Subject: [PATCH 10/15] fix logic for applying filter and make invalid version an info --- dsc_lib/src/discovery/command_discovery.rs | 30 ++++++++++++++-------- dsc_lib/src/dscerror.rs | 3 --- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 894319f3..b12b3eea 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -274,20 +274,16 @@ impl ResourceDiscovery for CommandDiscovery { fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Result>, DscError> { - trace!("Listing resources with type_name_filter/adapter_name_filter: {type_name_filter}/{adapter_name_filter}"); - - self.discover_resources(type_name_filter)?; - - if !adapter_name_filter.is_empty() { - self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; - } - + trace!("Listing resources with type_name_filter '{type_name_filter}' and adapter_name_filter '{adapter_name_filter}'"); let mut resources = BTreeMap::>::new(); if adapter_name_filter.is_empty() { + self.discover_resources(type_name_filter)?; resources.append(&mut self.resources); resources.append(&mut self.adapters); } else { + self.discover_resources("*")?; + self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; resources.append(&mut self.adapted_resources); } @@ -353,8 +349,22 @@ fn insert_resource(resources: &mut BTreeMap>, resource: // compare the resource versions and insert newest to oldest using semver let mut insert_index = resource_versions.len(); for (index, resource_instance) in resource_versions.iter().enumerate() { - let resource_instance_version = Version::parse(&resource_instance.version)?; - let resource_version = Version::parse(&resource.version)?; + let resource_instance_version = match Version::parse(&resource_instance.version) { + Ok(v) => v, + Err(err) => { + // write as info since PowerShell resources tend to have invalid semver + info!("Resource '{}' has invalid version: {err}", resource_instance.type_name); + continue; + }, + }; + let resource_version = match Version::parse(&resource.version) { + Ok(v) => v, + Err(err) => { + // write as info since PowerShell resources tend to have invalid semver + info!("Resource '{}' has invalid version: {err}", resource.type_name); + continue; + }, + }; // if the version already exists, we skip if resource_instance_version == resource_version { return Ok(()); diff --git a/dsc_lib/src/dscerror.rs b/dsc_lib/src/dscerror.rs index 3cbcbffd..5e5ae66f 100644 --- a/dsc_lib/src/dscerror.rs +++ b/dsc_lib/src/dscerror.rs @@ -98,9 +98,6 @@ pub enum DscError { #[error("Security context: {0}")] SecurityContext(String), - #[error("Semver: {0}")] - Semver(#[from] semver::Error), - #[error("Utf-8 conversion error: {0}")] Utf8Conversion(#[from] Utf8Error), From f8a4eb1f09ec648c3d397df8929c5a48e366bd09 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Wed, 24 Apr 2024 21:13:13 -0700 Subject: [PATCH 11/15] fix clippy --- dsc_lib/src/discovery/command_discovery.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index b12b3eea..7c9de2c1 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -148,10 +148,10 @@ impl ResourceDiscovery for CommandDiscovery { let manifest = import_manifest(manifest.clone())?; if manifest.kind == Some(Kind::Adapter) { trace!("Resource adapter {} found", resource.type_name); - insert_resource(&mut adapters, &resource)?; + insert_resource(&mut adapters, &resource); } else { trace!("Resource {} found", resource.type_name); - insert_resource(&mut resources, &resource)?; + insert_resource(&mut resources, &resource); } } } @@ -253,7 +253,7 @@ impl ResourceDiscovery for CommandDiscovery { } if name_regex.is_match(&resource.type_name) { - insert_resource(&mut adapted_resources, &resource)?; + insert_resource(&mut adapted_resources, &resource); adapter_resources_count += 1; } }, @@ -340,11 +340,11 @@ impl ResourceDiscovery for CommandDiscovery { } // helper to insert a resource into a vector of resources in order of newest to oldest -fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource) -> Result<(), DscError> { +fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource) { if resources.contains_key(&resource.type_name) { let Some(resource_versions) = resources.get_mut(&resource.type_name) else { resources.insert(resource.type_name.clone(), vec![resource.clone()]); - return Ok(()); + return; }; // compare the resource versions and insert newest to oldest using semver let mut insert_index = resource_versions.len(); @@ -367,7 +367,7 @@ fn insert_resource(resources: &mut BTreeMap>, resource: }; // if the version already exists, we skip if resource_instance_version == resource_version { - return Ok(()); + return; } if resource_instance_version < resource_version { @@ -379,7 +379,6 @@ fn insert_resource(resources: &mut BTreeMap>, resource: } else { resources.insert(resource.type_name.clone(), vec![resource.clone()]); } - Ok(()) } fn load_manifest(path: &Path) -> Result { From d947a8c13f0bb381c9a64858a6a5d60380d974e4 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Thu, 25 Apr 2024 13:30:29 -0700 Subject: [PATCH 12/15] add comments, fix formatting --- dsc_lib/src/discovery/command_discovery.rs | 52 ++++++++++++++++------ dsc_lib/src/discovery/mod.rs | 26 ++++++++--- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 7c9de2c1..8c5b49d4 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -148,10 +148,10 @@ impl ResourceDiscovery for CommandDiscovery { let manifest = import_manifest(manifest.clone())?; if manifest.kind == Some(Kind::Adapter) { trace!("Resource adapter {} found", resource.type_name); - insert_resource(&mut adapters, &resource); + insert_resource(&mut adapters, &resource, true); } else { trace!("Resource {} found", resource.type_name); - insert_resource(&mut resources, &resource); + insert_resource(&mut resources, &resource, true); } } } @@ -253,7 +253,9 @@ impl ResourceDiscovery for CommandDiscovery { } if name_regex.is_match(&resource.type_name) { - insert_resource(&mut adapted_resources, &resource); + // we allow duplicate versions since it can come from different adapters + // like PowerShell vs WindowsPowerShell + insert_resource(&mut adapted_resources, &resource, false); adapter_resources_count += 1; } }, @@ -300,6 +302,7 @@ impl ResourceDiscovery for CommandDiscovery { let mut remaining_required_resource_types = required_resource_types.to_owned(); for (resource_name, resources) in &self.resources { + // TODO: handle version requirements let Some(resource ) = resources.first() else { // skip if no resources continue; @@ -318,21 +321,44 @@ impl ResourceDiscovery for CommandDiscovery { } debug!("Found {} matching non-adapter-based resources", found_resources.len()); - // now go through the adapter resources and add them to the list of resources - for (adapted_name, adapted_resource) in &self.adapted_resources { - let Some(adapted_resource) = adapted_resource.first() else { - // skip if no resources + // now go through the adapters + for (adapter_name, adapters) in self.adapters.clone() { + // TODO: handle version requirements + let Some(adapter) = adapters.first() else { + // skip if no adapters continue; }; - if remaining_required_resource_types.contains(&adapted_name.to_lowercase()) + if remaining_required_resource_types.contains(&adapter_name.to_lowercase()) { - remaining_required_resource_types.retain(|x| *x != adapted_name.to_lowercase()); - found_resources.insert(adapted_name.to_lowercase(), adapted_resource.clone()); + // if an adapter is required, we need to find the resources it adapts + self.discover_adapted_resources("*", &adapter_name)?; + + // remove the adapter from the list of required resources + remaining_required_resource_types.retain(|x| *x != adapter_name.to_lowercase()); + found_resources.insert(adapter_name.to_lowercase(), adapter.clone()); if remaining_required_resource_types.is_empty() { return Ok(found_resources); } + + // now go through the adapter resources and add them to the list of resources + for (adapted_name, adapted_resource) in &self.adapted_resources { + let Some(adapted_resource) = adapted_resource.first() else { + // skip if no resources + continue; + }; + + if remaining_required_resource_types.contains(&adapted_name.to_lowercase()) + { + remaining_required_resource_types.retain(|x| *x != adapted_name.to_lowercase()); + found_resources.insert(adapted_name.to_lowercase(), adapted_resource.clone()); + if remaining_required_resource_types.is_empty() + { + return Ok(found_resources); + } + } + } } } Ok(found_resources) @@ -340,7 +366,7 @@ impl ResourceDiscovery for CommandDiscovery { } // helper to insert a resource into a vector of resources in order of newest to oldest -fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource) { +fn insert_resource(resources: &mut BTreeMap>, resource: &DscResource, skip_duplicate_version: bool) { if resources.contains_key(&resource.type_name) { let Some(resource_versions) = resources.get_mut(&resource.type_name) else { resources.insert(resource.type_name.clone(), vec![resource.clone()]); @@ -365,8 +391,8 @@ fn insert_resource(resources: &mut BTreeMap>, resource: continue; }, }; - // if the version already exists, we skip - if resource_instance_version == resource_version { + // if the version already exists, we might skip it + if !skip_duplicate_version && resource_instance_version == resource_version { return; } diff --git a/dsc_lib/src/discovery/mod.rs b/dsc_lib/src/discovery/mod.rs index ce4d417f..4a45664a 100644 --- a/dsc_lib/src/discovery/mod.rs +++ b/dsc_lib/src/discovery/mod.rs @@ -26,7 +26,16 @@ impl Discovery { }) } - /// List operation. + /// List operation for getting available resources based on the filters. + /// + /// # Arguments + /// + /// * `type_name_filter` - The filter for the resource type name. + /// * `adapter_name_filter` - The filter for the adapter name. + /// + /// # Returns + /// + /// A vector of `DscResource` instances. pub fn list_available_resources(&mut self, type_name_filter: &str, adapter_name_filter: &str) -> Vec { let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new()), @@ -59,22 +68,25 @@ impl Discovery { self.resources.get(type_name) } + /// Find resources based on the required resource types. + /// + /// # Arguments + /// + /// * `required_resource_types` - The required resource types. pub fn find_resources(&mut self, required_resource_types: &[String]) { - let discovery_types: Vec> = vec![ Box::new(command_discovery::CommandDiscovery::new()), ]; - let mut remaining_required_resource_types = required_resource_types.to_owned(); for mut discovery_type in discovery_types { let discovered_resources = match discovery_type.find_resources(&remaining_required_resource_types) { Ok(value) => value, Err(err) => { - error!("{err}"); - continue; - } - }; + error!("{err}"); + continue; + } + }; for resource in discovered_resources { self.resources.insert(resource.0.clone(), resource.1); From bc98145b66bbf0595e175d559f38d6c503732f5a Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Thu, 25 Apr 2024 16:22:34 -0700 Subject: [PATCH 13/15] cleanup traces and add adapter to list of found resources when enumerating adapted resources --- dsc/src/resource_command.rs | 8 +-- dsc_lib/src/discovery/command_discovery.rs | 44 +++++++------- dsc_lib/src/dscerror.rs | 3 + dsc_lib/src/dscresources/command_resource.rs | 62 ++++++-------------- 4 files changed, 44 insertions(+), 73 deletions(-) diff --git a/dsc/src/resource_command.rs b/dsc/src/resource_command.rs index ba40fcac..94d59bb3 100644 --- a/dsc/src/resource_command.rs +++ b/dsc/src/resource_command.rs @@ -27,7 +27,7 @@ pub fn get(dsc: &DscManager, resource_type: &str, mut input: String, format: &Op if let Some(pr) = get_resource(dsc, requires) { resource = pr; } else { - error!("Adapter {} not found", requires); + error!("Adapter '{}' not found", requires); return; }; } @@ -112,7 +112,7 @@ pub fn set(dsc: &DscManager, resource_type: &str, mut input: String, format: &Op if let Some(pr) = get_resource(dsc, requires) { resource = pr; } else { - error!("Adapter {} not found", requires); + error!("Adapter '{}' not found", requires); return; }; } @@ -149,7 +149,7 @@ pub fn test(dsc: &DscManager, resource_type: &str, mut input: String, format: &O if let Some(pr) = get_resource(dsc, requires) { resource = pr; } else { - error!("Adapter {} not found", requires); + error!("Adapter '{}' not found", requires); return; }; } @@ -186,7 +186,7 @@ pub fn delete(dsc: &DscManager, resource_type: &str, mut input: String) { if let Some(pr) = get_resource(dsc, requires) { resource = pr; } else { - error!("Adapter {} not found", requires); + error!("Adapter '{}' not found", requires); return; }; } diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 8c5b49d4..8e83bb0c 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -6,7 +6,6 @@ use crate::discovery::convert_wildcard_to_regex; use crate::dscresources::dscresource::{Capability, DscResource, ImplementedAs}; use crate::dscresources::resource_manifest::{import_manifest, validate_semver, Kind, ResourceManifest}; use crate::dscresources::command_resource::invoke_command; -use crate::dscresources::command_resource::log_resource_traces; use crate::dscerror::DscError; use indicatif::ProgressStyle; use regex::RegexBuilder; @@ -49,7 +48,7 @@ impl CommandDiscovery { trace!("DSC_RESOURCE_PATH not set, trying PATH"); match env::var_os("PATH") { Some(value) => { - debug!("Using PATH: {:?}", value.to_string_lossy()); + trace!("Using PATH: {:?}", value.to_string_lossy()); value }, None => { @@ -85,10 +84,6 @@ impl Default for CommandDiscovery { impl ResourceDiscovery for CommandDiscovery { fn discover_resources(&mut self, filter: &str) -> Result<(), DscError> { - if !self.resources.is_empty() || !self.adapters.is_empty() { - return Ok(()); - } - info!("Discovering resources using filter: {filter}"); let regex_str = convert_wildcard_to_regex(filter); @@ -232,11 +227,10 @@ impl ResourceDiscovery for CommandDiscovery { Ok((exit_code, stdout, stderr)) => (exit_code, stdout, stderr), Err(e) => { // In case of error, log and continue - warn!("Could not start {}: {}", list_command.executable, e); + warn!("{e}"); continue; }, }; - log_resource_traces(&stderr); if exit_code != 0 { // in case of failure, log and continue @@ -331,9 +325,6 @@ impl ResourceDiscovery for CommandDiscovery { if remaining_required_resource_types.contains(&adapter_name.to_lowercase()) { - // if an adapter is required, we need to find the resources it adapts - self.discover_adapted_resources("*", &adapter_name)?; - // remove the adapter from the list of required resources remaining_required_resource_types.retain(|x| *x != adapter_name.to_lowercase()); found_resources.insert(adapter_name.to_lowercase(), adapter.clone()); @@ -341,22 +332,27 @@ impl ResourceDiscovery for CommandDiscovery { { return Ok(found_resources); } + } - // now go through the adapter resources and add them to the list of resources - for (adapted_name, adapted_resource) in &self.adapted_resources { - let Some(adapted_resource) = adapted_resource.first() else { - // skip if no resources - continue; - }; + self.discover_adapted_resources("*", &adapter_name)?; - if remaining_required_resource_types.contains(&adapted_name.to_lowercase()) + // now go through the adapter resources and add them to the list of resources + for (adapted_name, adapted_resource) in &self.adapted_resources { + let Some(adapted_resource) = adapted_resource.first() else { + // skip if no resources + continue; + }; + + if remaining_required_resource_types.contains(&adapted_name.to_lowercase()) + { + remaining_required_resource_types.retain(|x| *x != adapted_name.to_lowercase()); + found_resources.insert(adapted_name.to_lowercase(), adapted_resource.clone()); + + // also insert the adapter + found_resources.insert(adapter_name.to_lowercase(), adapter.clone()); + if remaining_required_resource_types.is_empty() { - remaining_required_resource_types.retain(|x| *x != adapted_name.to_lowercase()); - found_resources.insert(adapted_name.to_lowercase(), adapted_resource.clone()); - if remaining_required_resource_types.is_empty() - { - return Ok(found_resources); - } + return Ok(found_resources); } } } diff --git a/dsc_lib/src/dscerror.rs b/dsc_lib/src/dscerror.rs index 5e5ae66f..fd401eb7 100644 --- a/dsc_lib/src/dscerror.rs +++ b/dsc_lib/src/dscerror.rs @@ -17,6 +17,9 @@ pub enum DscError { #[error("Command: Resource '{0}' [Exit code {1}] {2}")] Command(String, i32, String), + #[error("Command: Executable '{0}' [Exit code {1}] {2}")] + CommandExit(String, i32, String), + #[error("CommandOperation: {0} for executable '{1}'")] CommandOperation(String, String), diff --git a/dsc_lib/src/dscresources/command_resource.rs b/dsc_lib/src/dscresources/command_resource.rs index c0718bc0..5f9a5b4d 100644 --- a/dsc_lib/src/dscresources/command_resource.rs +++ b/dsc_lib/src/dscresources/command_resource.rs @@ -12,22 +12,22 @@ use tracing::{error, warn, info, debug, trace}; pub const EXIT_PROCESS_TERMINATED: i32 = 0x102; -pub fn log_resource_traces(stderr: &str) +pub fn log_resource_traces(process_name: &str, stderr: &str) { if !stderr.is_empty() { for trace_line in stderr.lines() { if let Result::Ok(json_obj) = serde_json::from_str::(trace_line) { if let Some(msg) = json_obj.get("Error") { - error!("{}", msg.as_str().unwrap_or_default()); + error!("Process {process_name}: {}", msg.as_str().unwrap_or_default()); } else if let Some(msg) = json_obj.get("Warning") { - warn!("{}", msg.as_str().unwrap_or_default()); + warn!("Process {process_name}: {}", msg.as_str().unwrap_or_default()); } else if let Some(msg) = json_obj.get("Info") { - info!("{}", msg.as_str().unwrap_or_default()); + info!("Process {process_name}: {}", msg.as_str().unwrap_or_default()); } else if let Some(msg) = json_obj.get("Debug") { - debug!("{}", msg.as_str().unwrap_or_default()); + debug!("Process {process_name}: {}", msg.as_str().unwrap_or_default()); } else if let Some(msg) = json_obj.get("Trace") { - trace!("{}", msg.as_str().unwrap_or_default()); + trace!("Process {process_name}: {}", msg.as_str().unwrap_or_default()); }; }; } @@ -54,12 +54,7 @@ pub fn invoke_get(resource: &ResourceManifest, cwd: &str, filter: &str) -> Resul } info!("Invoking get '{}' using '{}'", &resource.resource_type, &resource.get.executable); - let (exit_code, stdout, stderr) = invoke_command(&resource.get.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } - + let (_exit_code, stdout, stderr) = invoke_command(&resource.get.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; if resource.kind == Some(Kind::Resource) { debug!("Verifying output of get '{}' using '{}'", &resource.resource_type, &resource.get.executable); verify_json(resource, cwd, &stdout)?; @@ -131,10 +126,6 @@ pub fn invoke_set(resource: &ResourceManifest, cwd: &str, desired: &str, skip_te info!("Getting current state for set by invoking get {} using {}", &resource.resource_type, &resource.get.executable); let (exit_code, stdout, stderr) = invoke_command(&resource.get.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } if resource.kind == Some(Kind::Resource) { debug!("Verifying output of get '{}' using '{}'", &resource.resource_type, &resource.get.executable); @@ -165,10 +156,6 @@ pub fn invoke_set(resource: &ResourceManifest, cwd: &str, desired: &str, skip_te info!("Invoking set '{}' using '{}'", &resource.resource_type, &set.executable); let (exit_code, stdout, stderr) = invoke_command(&set.executable, args, input_desired, Some(cwd), env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } match set.returns { Some(ReturnKind::State) => { @@ -260,10 +247,6 @@ pub fn invoke_test(resource: &ResourceManifest, cwd: &str, expected: &str) -> Re info!("Invoking test '{}' using '{}'", &resource.resource_type, &test.executable); let (exit_code, stdout, stderr) = invoke_command(&test.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } if resource.kind == Some(Kind::Resource) { debug!("Verifying output of test '{}' using '{}'", &resource.resource_type, &test.executable); @@ -377,11 +360,7 @@ pub fn invoke_delete(resource: &ResourceManifest, cwd: &str, filter: &str) -> Re let command_input = get_command_input(&delete.input, filter)?; info!("Invoking delete '{}' using '{}'", &resource.resource_type, &delete.executable); - let (exit_code, _stdout, stderr) = invoke_command(&delete.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } + let (_exit_code, _stdout, _stderr) = invoke_command(&delete.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; Ok(()) } @@ -412,12 +391,7 @@ pub fn invoke_validate(resource: &ResourceManifest, cwd: &str, config: &str) -> let command_input = get_command_input(&validate.input, config)?; info!("Invoking validate '{}' using '{}'", &resource.resource_type, &validate.executable); - let (exit_code, stdout, stderr) = invoke_command(&validate.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } - + let (_exit_code, stdout, _stderr) = invoke_command(&validate.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; let result: ValidateResult = serde_json::from_str(&stdout)?; Ok(result) } @@ -438,11 +412,7 @@ pub fn get_schema(resource: &ResourceManifest, cwd: &str) -> Result { - let (exit_code, stdout, stderr) = invoke_command(&command.executable, command.args.clone(), None, Some(cwd), None)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } + let (_exit_code, stdout, _stderr) = invoke_command(&command.executable, command.args.clone(), None, Some(cwd), None)?; Ok(stdout) }, SchemaKind::Embedded(ref schema) => { @@ -499,11 +469,7 @@ pub fn invoke_export(resource: &ResourceManifest, cwd: &str, input: Option<&str> args = process_args(&export.args, ""); } - let (exit_code, stdout, stderr) = invoke_command(&export.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; - log_resource_traces(&stderr); - if exit_code != 0 { - return Err(DscError::Command(resource.resource_type.clone(), exit_code, stderr)); - } + let (_exit_code, stdout, stderr) = invoke_command(&export.executable, args, command_input.stdin.as_deref(), Some(cwd), command_input.env)?; let mut instances: Vec = Vec::new(); for line in stdout.lines() { @@ -594,7 +560,13 @@ pub fn invoke_command(executable: &str, args: Option>, input: Option } if !stderr.is_empty() { trace!("STDERR returned: {}", &stderr); + log_resource_traces(executable, &stderr); } + + if exit_code != 0 { + return Err(DscError::Command(executable.to_string(), exit_code, stderr)); + } + Ok((exit_code, stdout, stderr)) } From 3360cb64520c6058883babfb31d2ede4727e369c Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Thu, 25 Apr 2024 17:17:40 -0700 Subject: [PATCH 14/15] update tracing tests --- dsc/tests/dsc_tracing.tests.ps1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dsc/tests/dsc_tracing.tests.ps1 b/dsc/tests/dsc_tracing.tests.ps1 index 9ce01fe7..24ae2edb 100644 --- a/dsc/tests/dsc_tracing.tests.ps1 +++ b/dsc/tests/dsc_tracing.tests.ps1 @@ -19,7 +19,7 @@ Describe 'tracing tests' { It 'trace level error does not emit other levels' { $logPath = "$TestDrive/dsc_trace.log" - $null = '{}' | dsc --trace-level error resource get -r 'DoesNotExist' 2> $logPath + $null = '{}' | dsc --trace-level error resource list 'DoesNotExist' 2> $logPath $log = Get-Content $logPath -Raw $log | Should -Not -BeLikeExactly "* WARNING *" $log | Should -Not -BeLikeExactly "* INFO *" @@ -29,18 +29,18 @@ Describe 'tracing tests' { It 'trace format plaintext does not emit ANSI' { $logPath = "$TestDrive/dsc_trace.log" - $null = '{}' | dsc --trace-format plaintext resource get -r 'DoesNotExist' 2> $logPath + $null = '{}' | dsc --trace-format plaintext resource list 'DoesNotExist' 2> $logPath $log = Get-Content $logPath -Raw $log | Should -Not -BeLikeExactly "*``[0m*" } It 'trace format json emits json' { $logPath = "$TestDrive/dsc_trace.log" - $null = '{}' | dsc --trace-format json resource get -r 'DoesNotExist' 2> $logPath + $null = '{}' | dsc --trace-format json resource list 'DoesNotExist' 2> $logPath foreach ($line in (Get-Content $logPath)) { $trace = $line | ConvertFrom-Json -Depth 10 $trace.timestamp | Should -Not -BeNullOrEmpty - $trace.level | Should -BeIn 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'TRACE' + $trace.level | Should -BeIn 'ERROR', 'WARN', 'INFO', 'DEBUG', 'TRACE' $trace.fields.message | Should -Not -BeNullOrEmpty } } @@ -55,12 +55,12 @@ Describe 'tracing tests' { param($level, $sourceExpected) $logPath = "$TestDrive/dsc_trace.log" - $null = '{}' | dsc -l $level resource get -r 'DoesNotExist' 2> $logPath + $null = '{}' | dsc -l $level resource list 'DoesNotExist' 2> $logPath $log = Get-Content $logPath -Raw if ($sourceExpected) { - $log | Should -BeLike "*dsc*: *" + $log | Should -BeLike "*dsc_lib*: *" } else { - $log | Should -Not -BeLike "*dsc*: *" + $log | Should -Not -BeLike "*dsc_lib*: *" } } } From 2374d6b779e6d291bcac38153253b0bd2008c300 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL HE/HIM) (from Dev Box)" Date: Fri, 26 Apr 2024 17:11:13 -0700 Subject: [PATCH 15/15] update comment based on Andrew's feedback --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 8e83bb0c..fb536db2 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -20,7 +20,7 @@ use tracing::{debug, info, trace, warn, warn_span}; use tracing_indicatif::span_ext::IndicatifSpanExt; pub struct CommandDiscovery { - // use BTreeMap so that the results are sorted + // use BTreeMap so that the results are sorted by the typename, the Vec is sorted by version resources: BTreeMap>, adapters: BTreeMap>, adapted_resources: BTreeMap>,