From 25147155117ef7bc37b50223abc9355608f17c8f Mon Sep 17 00:00:00 2001 From: stuioco Date: Fri, 23 Aug 2024 06:19:31 +0100 Subject: [PATCH 1/9] changed return types and added datasource_query --- core/templating/datasource.go | 4 +- core/templating/datasource_query.go | 162 ++++++++++++++++++++++++++++ core/templating/template_helpers.go | 44 +++++--- core/templating/templating.go | 1 + 4 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 core/templating/datasource_query.go diff --git a/core/templating/datasource.go b/core/templating/datasource.go index 0dcaf9f0e..e1e3abc83 100644 --- a/core/templating/datasource.go +++ b/core/templating/datasource.go @@ -27,7 +27,7 @@ func NewCsvDataSource(fileName, fileContent string) (*DataSource, error) { return &DataSource{"csv", fileName, records, sync.Mutex{}}, nil } -func (dataSource DataSource) GetDataSourceView() (v2.CSVDataSourceView, error) { +func (dataSource *DataSource) GetDataSourceView() (v2.CSVDataSourceView, error) { content, err := getData(dataSource) if err != nil { @@ -36,7 +36,7 @@ func (dataSource DataSource) GetDataSourceView() (v2.CSVDataSourceView, error) { return v2.CSVDataSourceView{Name: dataSource.Name, Data: content}, nil } -func getData(source DataSource) (string, error) { +func getData(source *DataSource) (string, error) { var csvData strings.Builder csvWriter := csv.NewWriter(&csvData) diff --git a/core/templating/datasource_query.go b/core/templating/datasource_query.go new file mode 100644 index 000000000..4d14eb20b --- /dev/null +++ b/core/templating/datasource_query.go @@ -0,0 +1,162 @@ +package templating + +import ( + "errors" + "regexp" + "strings" +) + +// A mock structure representing your CSV data +// var data = [][]string{ +// {"id", "name", "age", "city"}, +// {"1", "John", "30", "New York"}, +// {"2", "Jane", "25", "Los Angeles"}, +// {"3", "Doe", "40", "New York"}, +// } + +// RowMap represents a single row in the result set +type RowMap map[string]string + +// Condition represents a single condition in the WHERE clause +type Condition struct { + Column string + Operator string + Value string +} + +// SelectQuery represents a simple SQL-like SELECT query +type SelectQuery struct { + Columns []string + Conditions []Condition +} + +// ParseQuery parses a query string into a SelectQuery struct +func ParseQuery(query string) (SelectQuery, error) { + query = strings.TrimSpace(query) + + selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+WHERE\s+(.+)$`) + matches := selectRegex.FindStringSubmatch(query) + if len(matches) != 3 { + return SelectQuery{}, errors.New("invalid query format") + } + + columnsPart := matches[1] + wherePart := matches[2] + + columns := strings.Split(columnsPart, ",") + for i, column := range columns { + columns[i] = strings.TrimSpace(column) + } + + conditions, err := parseConditions(wherePart) + if err != nil { + return SelectQuery{}, err + } + + return SelectQuery{ + Columns: columns, + Conditions: conditions, + }, nil +} + +// parseConditions parses the WHERE part of the query into a slice of Conditions +func parseConditions(wherePart string) ([]Condition, error) { + conditions := []Condition{} + + conditionRegex := regexp.MustCompile(`(\w+)\s*(==|!=)\s*'([^']*)'`) + conditionMatches := conditionRegex.FindAllStringSubmatch(wherePart, -1) + + if len(conditionMatches) == 0 { + return nil, errors.New("no valid conditions found") + } + + for _, match := range conditionMatches { + if len(match) != 4 { + return nil, errors.New("invalid condition format") + } + conditions = append(conditions, Condition{ + Column: match[1], + Operator: match[2], + Value: match[3], + }) + } + + return conditions, nil +} + +// ExecuteQuery runs the query against the in-memory data +func ExecuteQuery(data [][]string, query SelectQuery) []RowMap { + headers := data[0] // first row as header + results := []RowMap{} + + for _, row := range data[1:] { + rowMap := mapRow(headers, row) + if matchesConditions(rowMap, query.Conditions) { + results = append(results, projectRow(rowMap, query.Columns)) + } + } + return results +} + +// mapRow converts a CSV row into a map with column names as keys +func mapRow(headers, row []string) RowMap { + rowMap := make(RowMap) + for i, header := range headers { + rowMap[header] = row[i] + } + return rowMap +} + +// projectRow filters the row based on the selected columns +func projectRow(row RowMap, columns []string) RowMap { + if len(columns) == 0 { + return row + } + projected := make(RowMap) + for _, col := range columns { + if val, ok := row[col]; ok { + projected[col] = val + } + } + return projected +} + +// matchesConditions checks if a row matches all given conditions +func matchesConditions(row RowMap, conditions []Condition) bool { + for _, condition := range conditions { + val, ok := row[condition.Column] + if !ok { + return false // Column doesn't exist + } + + switch condition.Operator { + case "==": + if val != condition.Value { + return false + } + case "!=": + if val == condition.Value { + return false + } + default: + return false // Unsupported operator + } + } + return true // All conditions matched +} + +// Example usage +// func main() { +// queryStr := "SELECT age, city WHERE age!='30' AND city=='New York'" + +// query, err := ParseQuery(queryStr) +// if err != nil { +// fmt.Println("Error:", err) +// return +// } + +// results := ExecuteQuery(data, query) +// for _, row := range results { +// fmt.Println(row) +// } +// } diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index 28df03453..c57e75c8b 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -247,19 +247,20 @@ func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, se return getEvaluationString("csv", options) } -func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchFieldName string, searchFieldValue string) []map[string]string { +func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchFieldName string, searchFieldValue string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { log.Debug("could not find datasource " + dataSourceName) - return []map[string]string{} + return []RowMap{} } if len(source.Data) < 1 { log.Debug("no data available in datasource " + dataSourceName) - return []map[string]string{} + return []RowMap{} } source.mu.Lock() defer source.mu.Unlock() + headers := source.Data[0] fieldIndex := -1 for i, header := range headers { @@ -270,12 +271,13 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField } if fieldIndex == -1 { log.Debug("could not find search field name " + searchFieldName) - return []map[string]string{} + return []RowMap{} } - var result []map[string]string + + var result []RowMap for _, row := range source.Data[1:] { if fieldIndex < len(row) && row[fieldIndex] == searchFieldValue { - rowMap := make(map[string]string) + rowMap := make(RowMap) for i, cell := range row { if i < len(headers) { rowMap[headers[i]] = cell @@ -284,7 +286,6 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField result = append(result, rowMap) } } - return result } @@ -301,23 +302,23 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string { } } -func (t templateHelpers) csvAsMap(dataSourceName string) []map[string]string { +func (t templateHelpers) csvAsMap(dataSourceName string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { log.Debug("could not find datasource " + dataSourceName) - return []map[string]string{} + return []RowMap{} } source.mu.Lock() defer source.mu.Unlock() if len(source.Data) < 1 { log.Debug("no data available in datasource " + dataSourceName) - return []map[string]string{} + return []RowMap{} } headers := source.Data[0] - var result []map[string]string + var result []RowMap for _, row := range source.Data[1:] { - rowMap := make(map[string]string) + rowMap := make(RowMap) for i, cell := range row { if i < len(headers) { rowMap[headers[i]] = cell @@ -398,6 +399,25 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { return fmt.Sprintf("%d", numRows) } +func (t templateHelpers) csvSQL(dataSourceName, queryString string) []RowMap { + //queryString := "SELECT age, city WHERE age!='30' AND city=='New York'" + templateDataSources := t.TemplateDataSource.DataSources + source, exists := templateDataSources[dataSourceName] + if !exists { + log.Debug("could not find datasource " + dataSourceName) + return []RowMap{} + } + query, err := ParseQuery(queryString) + if err != nil { + fmt.Println("Error:", err) + return []RowMap{} + } + source.mu.Lock() + defer source.mu.Unlock() + results := ExecuteQuery(source.Data, query) + return results +} + func (t templateHelpers) parseJournalBasedOnIndex(indexName, keyValue, dataSource, queryType, lookupQuery string, options *raymond.Options) interface{} { journalDetails := options.Value("Journal").(Journal) if journalEntry, err := getIndexEntry(journalDetails, indexName, keyValue); err == nil { diff --git a/core/templating/templating.go b/core/templating/templating.go index 0e8e4abe8..58dde209e 100644 --- a/core/templating/templating.go +++ b/core/templating/templating.go @@ -107,6 +107,7 @@ func NewTemplator() *Templator { helperMethodMap["csvAddRow"] = t.csvAddRow helperMethodMap["csvDeleteRows"] = t.csvDeleteRows helperMethodMap["csvCountRows"] = t.csvCountRows + helperMethodMap["csvSQL"] = t.csvSQL helperMethodMap["journal"] = t.parseJournalBasedOnIndex helperMethodMap["hasJournalKey"] = t.hasJournalKey helperMethodMap["setStatusCode"] = t.setStatusCode From edfe0f7f4baa302d30e530905acf176c552eb59b Mon Sep 17 00:00:00 2001 From: stuioco Date: Fri, 23 Aug 2024 17:58:18 +0100 Subject: [PATCH 2/9] SQL select over csv --- core/templating/datasource_query.go | 132 +++++++++++++++++++--------- core/templating/template_helpers.go | 83 ++++++++++++++--- core/templating/templating.go | 2 + 3 files changed, 166 insertions(+), 51 deletions(-) diff --git a/core/templating/datasource_query.go b/core/templating/datasource_query.go index 4d14eb20b..05b4d1917 100644 --- a/core/templating/datasource_query.go +++ b/core/templating/datasource_query.go @@ -4,15 +4,9 @@ import ( "errors" "regexp" "strings" -) -// A mock structure representing your CSV data -// var data = [][]string{ -// {"id", "name", "age", "city"}, -// {"1", "John", "30", "New York"}, -// {"2", "Jane", "25", "Los Angeles"}, -// {"3", "Doe", "40", "New York"}, -// } + log "github.com/sirupsen/logrus" +) // RowMap represents a single row in the result set type RowMap map[string]string @@ -25,45 +19,102 @@ type Condition struct { } // SelectQuery represents a simple SQL-like SELECT query +// +// type SelectQuery struct { +// Columns []string +// Conditions []Condition +// } type SelectQuery struct { - Columns []string - Conditions []Condition + Columns []string + Conditions []Condition + DataSourceName string } // ParseQuery parses a query string into a SelectQuery struct -func ParseQuery(query string) (SelectQuery, error) { +// func ParseQuery(query string, headers []string) (SelectQuery, error) { +// query = strings.TrimSpace(query) + +// selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+WHERE\s+(.+)$`) +// matches := selectRegex.FindStringSubmatch(query) +// if len(matches) != 3 { +// return SelectQuery{}, errors.New("invalid query format") +// } + +// columnsPart := matches[1] +// wherePart := matches[2] + +// columns := parseColumns(columnsPart, headers) +// conditions, err := parseConditions(wherePart) +// if err != nil { +// return SelectQuery{}, err +// } + +// return SelectQuery{ +// Columns: columns, +// Conditions: conditions, +// }, nil +// } +func ParseQuery(query string, datasource map[string]*DataSource) (SelectQuery, error) { query = strings.TrimSpace(query) - selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+WHERE\s+(.+)$`) + // Regular expression to match SELECT, FROM, and WHERE clauses + selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) matches := selectRegex.FindStringSubmatch(query) - if len(matches) != 3 { + if len(matches) < 3 { return SelectQuery{}, errors.New("invalid query format") } columnsPart := matches[1] - wherePart := matches[2] - - columns := strings.Split(columnsPart, ",") - for i, column := range columns { - columns[i] = strings.TrimSpace(column) + dataSourceName := matches[2] + wherePart := "" + if len(matches) == 4 { + wherePart = matches[3] } - conditions, err := parseConditions(wherePart) - if err != nil { - return SelectQuery{}, err + // Assuming headers can be retrieved from the data source at a later stage + //headers := []string{} // This will be populated once the data source is retrieved + headers := datasource[dataSourceName].Data[0] + log.Debug("###################") + log.Debug(headers) + log.Debug("###################") + // Determine the columns to select + columns := parseColumns(columnsPart, headers) + + // Parse the WHERE clause if present + var conditions []Condition + var err error + if wherePart != "" { + conditions, err = parseConditions(wherePart) + if err != nil { + return SelectQuery{}, err + } } return SelectQuery{ - Columns: columns, - Conditions: conditions, + Columns: columns, + Conditions: conditions, + DataSourceName: dataSourceName, }, nil } +// parseColumns determines the columns to select based on the query part and headers +func parseColumns(columnsPart string, headers []string) []string { + columnsPart = strings.TrimSpace(columnsPart) + if columnsPart == "*" { + return headers + } + columns := strings.Split(columnsPart, ",") + for i, column := range columns { + columns[i] = strings.TrimSpace(column) + } + return columns +} + // parseConditions parses the WHERE part of the query into a slice of Conditions func parseConditions(wherePart string) ([]Condition, error) { conditions := []Condition{} - conditionRegex := regexp.MustCompile(`(\w+)\s*(==|!=)\s*'([^']*)'`) + conditionRegex := regexp.MustCompile(`(\w+)\s*(==|!=|<=|>=|<|>)\s*'([^']*)'`) conditionMatches := conditionRegex.FindAllStringSubmatch(wherePart, -1) if len(conditionMatches) == 0 { @@ -88,7 +139,6 @@ func parseConditions(wherePart string) ([]Condition, error) { func ExecuteQuery(data [][]string, query SelectQuery) []RowMap { headers := data[0] // first row as header results := []RowMap{} - for _, row := range data[1:] { rowMap := mapRow(headers, row) if matchesConditions(rowMap, query.Conditions) { @@ -138,25 +188,25 @@ func matchesConditions(row RowMap, conditions []Condition) bool { if val == condition.Value { return false } + case "<": + if isGreaterThanOrEqual(val, condition.Value) { + return false + } + case "<=": + if isGreaterThan(val, condition.Value) { + return false + } + case ">": + if isLessThanOrEqual(val, condition.Value) { + return false + } + case ">=": + if isLessThan(val, condition.Value) { + return false + } default: return false // Unsupported operator } } return true // All conditions matched } - -// Example usage -// func main() { -// queryStr := "SELECT age, city WHERE age!='30' AND city=='New York'" - -// query, err := ParseQuery(queryStr) -// if err != nil { -// fmt.Println("Error:", err) -// return -// } - -// results := ExecuteQuery(data, query) -// for _, row := range results { -// fmt.Println(row) -// } -// } diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index c57e75c8b..585efc2e7 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -135,7 +135,7 @@ func (t templateHelpers) isBool(s string) bool { return err == nil } -func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { +func isGreaterThan(valueToCheck, minimumValue string) bool { num1, err := strconv.ParseFloat(valueToCheck, 64) if err != nil { return false @@ -147,7 +147,27 @@ func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { return num1 > num2 } -func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { +func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { + return isGreaterThan(valueToCheck, minimumValue) +} + +func isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { + num1, err := strconv.ParseFloat(valueToCheck, 64) + if err != nil { + return false + } + num2, err := strconv.ParseFloat(minimumValue, 64) + if err != nil { + return false + } + return num1 >= num2 +} + +func (t templateHelpers) isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { + return isGreaterThan(valueToCheck, minimumValue) +} + +func isLessThan(valueToCheck, maximumValue string) bool { num1, err := strconv.ParseFloat(valueToCheck, 64) if err != nil { return false @@ -159,6 +179,26 @@ func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { return num1 < num2 } +func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { + return isLessThan(valueToCheck, maximumValue) +} + +func isLessThanOrEqual(valueToCheck, maximumValue string) bool { + num1, err := strconv.ParseFloat(valueToCheck, 64) + if err != nil { + return false + } + num2, err := strconv.ParseFloat(maximumValue, 64) + if err != nil { + return false + } + return num1 <= num2 +} + +func (t templateHelpers) isLessThanOrEqual(valueToCheck, maximumValue string) bool { + return isLessThan(valueToCheck, maximumValue) +} + func (t templateHelpers) isBetween(valueToCheck, minimumValue, maximumValue string) bool { return t.isGreaterThan(valueToCheck, minimumValue) && t.isLessThan(valueToCheck, maximumValue) } @@ -399,21 +439,44 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { return fmt.Sprintf("%d", numRows) } -func (t templateHelpers) csvSQL(dataSourceName, queryString string) []RowMap { - //queryString := "SELECT age, city WHERE age!='30' AND city=='New York'" +// func (t templateHelpers) csvSQL(dataSourceName, queryString string) []RowMap { +// templateDataSources := t.TemplateDataSource.DataSources +// source, exists := templateDataSources[dataSourceName] +// if !exists { +// log.Debug("could not find datasource " + dataSourceName) +// return []RowMap{} +// } +// source.mu.Lock() +// defer source.mu.Unlock() +// query, err := ParseQuery(queryString, source.Data[0]) +// if err != nil { +// log.Error("Error:", err) +// return []RowMap{} +// } +// results := ExecuteQuery(source.Data, query) +// return results +// } +func (t templateHelpers) csvSQL(queryString string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources - source, exists := templateDataSources[dataSourceName] - if !exists { - log.Debug("could not find datasource " + dataSourceName) + + // Parse the query string to get the SelectQuery + query, err := ParseQuery(queryString, templateDataSources) + if err != nil { + log.Debug("Error parsing query:", err) return []RowMap{} } - query, err := ParseQuery(queryString) - if err != nil { - fmt.Println("Error:", err) + + // Find the data source by name + source, exists := templateDataSources[query.DataSourceName] + if !exists { + log.Debug("Could not find datasource " + query.DataSourceName) return []RowMap{} } + source.mu.Lock() defer source.mu.Unlock() + + // Execute the query against the data source results := ExecuteQuery(source.Data, query) return results } diff --git a/core/templating/templating.go b/core/templating/templating.go index 58dde209e..65c00cf24 100644 --- a/core/templating/templating.go +++ b/core/templating/templating.go @@ -95,7 +95,9 @@ func NewTemplator() *Templator { helperMethodMap["isAlphanumeric"] = t.isAlphanumeric helperMethodMap["isBool"] = t.isBool helperMethodMap["isGreaterThan"] = t.isGreaterThan + helperMethodMap["isGreaterThanOrEqual"] = t.isGreaterThanOrEqual helperMethodMap["isLessThan"] = t.isLessThan + helperMethodMap["isLessThanOrEqual"] = t.isLessThanOrEqual helperMethodMap["isBetween"] = t.isBetween helperMethodMap["matchesRegex"] = t.matchesRegex helperMethodMap["faker"] = t.faker From 302831d22f1b39dbfc3ac144d70a3a135a999433 Mon Sep 17 00:00:00 2001 From: stuioco Date: Sat, 24 Aug 2024 19:14:05 +0100 Subject: [PATCH 3/9] Datasource SQL Support for UPDATE and DELETE --- core/templating/datasource_query.go | 268 +++++++++++++++++++++------- core/templating/template_helpers.go | 193 +++++++++++++------- 2 files changed, 332 insertions(+), 129 deletions(-) diff --git a/core/templating/datasource_query.go b/core/templating/datasource_query.go index 05b4d1917..3aaffd174 100644 --- a/core/templating/datasource_query.go +++ b/core/templating/datasource_query.go @@ -4,8 +4,6 @@ import ( "errors" "regexp" "strings" - - log "github.com/sirupsen/logrus" ) // RowMap represents a single row in the result set @@ -19,78 +17,109 @@ type Condition struct { } // SelectQuery represents a simple SQL-like SELECT query -// -// type SelectQuery struct { -// Columns []string -// Conditions []Condition -// } type SelectQuery struct { + Type string // "SELECT", "UPDATE", or "DELETE" Columns []string Conditions []Condition + SetClauses map[string]string // For UPDATE queries DataSourceName string } -// ParseQuery parses a query string into a SelectQuery struct -// func ParseQuery(query string, headers []string) (SelectQuery, error) { -// query = strings.TrimSpace(query) - -// selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+WHERE\s+(.+)$`) -// matches := selectRegex.FindStringSubmatch(query) -// if len(matches) != 3 { -// return SelectQuery{}, errors.New("invalid query format") -// } +func parseQuery(query string, datasource map[string]*DataSource) (SelectQuery, error) { + query = strings.TrimSpace(query) -// columnsPart := matches[1] -// wherePart := matches[2] + var queryType string + if strings.HasPrefix(strings.ToUpper(query), "SELECT") { + queryType = "SELECT" + } else if strings.HasPrefix(strings.ToUpper(query), "UPDATE") { + queryType = "UPDATE" + } else if strings.HasPrefix(strings.ToUpper(query), "DELETE") { + queryType = "DELETE" + } else { + return SelectQuery{}, errors.New("invalid query type") + } -// columns := parseColumns(columnsPart, headers) -// conditions, err := parseConditions(wherePart) -// if err != nil { -// return SelectQuery{}, err -// } + var selectRegex *regexp.Regexp + var matches []string + var columnsPart, dataSourceName, wherePart string -// return SelectQuery{ -// Columns: columns, -// Conditions: conditions, -// }, nil -// } -func ParseQuery(query string, datasource map[string]*DataSource) (SelectQuery, error) { - query = strings.TrimSpace(query) + switch queryType { + case "SELECT": + selectRegex = regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) + matches = selectRegex.FindStringSubmatch(query) + if len(matches) < 3 { + return SelectQuery{}, errors.New("invalid query format") + } + columnsPart = matches[1] + dataSourceName = matches[2] + if len(matches) == 4 { + wherePart = matches[3] + } + case "UPDATE": + //selectRegex = regexp.MustCompile(`(?i)^UPDATE\s+(\w+)\s+SET\s+(.+)\s*(?:WHERE\s+(.+))?$`) + selectRegex = regexp.MustCompile(`(?i)^UPDATE\s+(\w+)\s+SET\s+(.+?)(?:\s+WHERE\s+(.+))?$`) - // Regular expression to match SELECT, FROM, and WHERE clauses - selectRegex := regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) - matches := selectRegex.FindStringSubmatch(query) - if len(matches) < 3 { - return SelectQuery{}, errors.New("invalid query format") - } + matches = selectRegex.FindStringSubmatch(query) + if len(matches) < 3 { + return SelectQuery{}, errors.New("invalid UPDATE query format") + } + dataSourceName = matches[1] + setPart := matches[2] + if len(matches) == 4 { + wherePart = matches[3] + } + // log.Debug("########## ParseQuery ############") + // logMessage := fmt.Sprintf("len(matches)=%d", len(matches)) + // log.Debug(logMessage) + // logMessage = fmt.Sprintf("matches[3]=" + matches[3]) + // log.Debug(logMessage) + // formattedMatches := strings.Join(matches, " --- ") + // logMessage = fmt.Sprintf("matches=%v", formattedMatches) + // log.Debug(logMessage) + // log.Debug("######################") + setClauses := parseSetClauses(setPart) + conditions, err := parseConditions(wherePart) + if err != nil { + return SelectQuery{}, err + } + return SelectQuery{ + Type: "UPDATE", + Conditions: conditions, + SetClauses: setClauses, + DataSourceName: dataSourceName, + }, nil - columnsPart := matches[1] - dataSourceName := matches[2] - wherePart := "" - if len(matches) == 4 { - wherePart = matches[3] + case "DELETE": + selectRegex = regexp.MustCompile(`(?i)^DELETE\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) + matches = selectRegex.FindStringSubmatch(query) + if len(matches) < 2 { + return SelectQuery{}, errors.New("invalid DELETE query format") + } + dataSourceName = matches[1] + if len(matches) == 3 { + wherePart = matches[2] + } + conditions, err := parseConditions(wherePart) + if err != nil { + return SelectQuery{}, err + } + return SelectQuery{ + Type: "DELETE", + Conditions: conditions, + DataSourceName: dataSourceName, + }, nil } - // Assuming headers can be retrieved from the data source at a later stage - //headers := []string{} // This will be populated once the data source is retrieved headers := datasource[dataSourceName].Data[0] - log.Debug("###################") - log.Debug(headers) - log.Debug("###################") - // Determine the columns to select columns := parseColumns(columnsPart, headers) - // Parse the WHERE clause if present - var conditions []Condition - var err error - if wherePart != "" { - conditions, err = parseConditions(wherePart) - if err != nil { - return SelectQuery{}, err - } + conditions, err := parseConditions(wherePart) + if err != nil { + return SelectQuery{}, err } return SelectQuery{ + Type: queryType, Columns: columns, Conditions: conditions, DataSourceName: dataSourceName, @@ -110,7 +139,40 @@ func parseColumns(columnsPart string, headers []string) []string { return columns } -// parseConditions parses the WHERE part of the query into a slice of Conditions +// TrimQuotes trims matching single or double quotes from the outer edges of a string. +func trimQuotes(s string) string { + // Check if the string length is less than 2, in which case there's nothing to trim. + if len(s) < 2 { + return s + } + + // Get the first and last characters of the string. + firstChar := s[0] + lastChar := s[len(s)-1] + + // Check if both the first and last characters are matching quotes. + if (firstChar == '\'' || firstChar == '"') && firstChar == lastChar { + return s[1 : len(s)-1] + } + + // If no trimming is necessary, return the original string. + return s +} + +// parseSetClauses parses the SET part of an UPDATE query +func parseSetClauses(setPart string) map[string]string { + setClauses := make(map[string]string) + parts := strings.Split(setPart, ",") + for _, part := range parts { + keyValue := strings.Split(strings.TrimSpace(part), "=") + if len(keyValue) == 2 { + setClauses[strings.TrimSpace(keyValue[0])] = trimQuotes(strings.TrimSpace(keyValue[1])) + } + } + return setClauses +} + +// parseConditions parses the WHERE part of the query into a slice of Conditions and returns an error if any issues are found. func parseConditions(wherePart string) ([]Condition, error) { conditions := []Condition{} @@ -135,9 +197,29 @@ func parseConditions(wherePart string) ([]Condition, error) { return conditions, nil } -// ExecuteQuery runs the query against the in-memory data -func ExecuteQuery(data [][]string, query SelectQuery) []RowMap { - headers := data[0] // first row as header +// ExecuteQuery processes different types of SQL-like queries based on the query type +// func executeQuery(data *[][]string, query SelectQuery) ([]RowMap, error) { +// switch query.Type { +// case "SELECT": +// return executeSelectQuery(*data, query), nil +// case "UPDATE": +// if err := executeUpdateQuery(data, query); err != nil { +// return nil, err +// } +// return nil, nil // UPDATE doesn't return rows +// case "DELETE": +// if err := executeDeleteQuery(data, query); err != nil { +// return nil, err +// } +// return nil, nil // DELETE doesn't return rows +// default: +// return nil, errors.New("unsupported query type") +// } +// } + +// ExecuteSelectQuery executes a SELECT query and returns the results as a slice of RowMaps +func executeSelectQuery(data [][]string, query SelectQuery) []RowMap { + headers := data[0] // First row as header results := []RowMap{} for _, row := range data[1:] { rowMap := mapRow(headers, row) @@ -148,6 +230,58 @@ func ExecuteQuery(data [][]string, query SelectQuery) []RowMap { return results } +// ExecuteUpdateQuery executes an UPDATE query and modifies the data in-place +func executeUpdateQuery(data *[][]string, query SelectQuery) error { + if len(*data) < 2 { + return errors.New("no data available to update") + } + + headers := (*data)[0] + conditions := query.Conditions + setClauses := query.SetClauses + for i, row := range (*data)[1:] { + rowMap := mapRow(headers, row) + if matchesConditions(rowMap, conditions) { + for column, newValue := range setClauses { + colIndex := indexOf(headers, column) + if colIndex != -1 { + (*data)[i+1][colIndex] = newValue + } + } + } + } + return nil +} + +// ExecuteDeleteQuery executes a DELETE query and modifies the data in-place +func executeDeleteQuery(data *[][]string, query SelectQuery) error { + if len(*data) < 2 { + return errors.New("no data available to delete") + } + + headers := (*data)[0] + conditions := query.Conditions + filteredData := [][]string{headers} + for _, row := range (*data)[1:] { + rowMap := mapRow(headers, row) + if !matchesConditions(rowMap, conditions) { + filteredData = append(filteredData, row) + } + } + *data = filteredData + return nil +} + +// Helper function to find the index of a column header +func indexOf(headers []string, column string) int { + for i, header := range headers { + if header == column { + return i + } + } + return -1 +} + // mapRow converts a CSV row into a map with column names as keys func mapRow(headers, row []string) RowMap { rowMap := make(RowMap) @@ -157,6 +291,15 @@ func mapRow(headers, row []string) RowMap { return rowMap } +// mapToRow converts a map back to a CSV row +// func mapToRow(headers []string, rowMap RowMap) []string { +// row := make([]string, len(headers)) +// for i, header := range headers { +// row[i] = rowMap[header] +// } +// return row +// } + // projectRow filters the row based on the selected columns func projectRow(row RowMap, columns []string) RowMap { if len(columns) == 0 { @@ -176,9 +319,8 @@ func matchesConditions(row RowMap, conditions []Condition) bool { for _, condition := range conditions { val, ok := row[condition.Column] if !ok { - return false // Column doesn't exist + return false } - switch condition.Operator { case "==": if val != condition.Value { @@ -204,9 +346,7 @@ func matchesConditions(row RowMap, conditions []Condition) bool { if isLessThan(val, condition.Value) { return false } - default: - return false // Unsupported operator } } - return true // All conditions matched + return true } diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index 585efc2e7..aae9c4a2d 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -135,74 +135,130 @@ func (t templateHelpers) isBool(s string) bool { return err == nil } -func isGreaterThan(valueToCheck, minimumValue string) bool { - num1, err := strconv.ParseFloat(valueToCheck, 64) +// func isGreaterThan(valueToCheck, minimumValue string) bool { +// num1, err := strconv.ParseFloat(valueToCheck, 64) +// if err != nil { +// return false +// } +// num2, err := strconv.ParseFloat(minimumValue, 64) +// if err != nil { +// return false +// } +// return num1 > num2 +// } + +// func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { +// return isGreaterThan(valueToCheck, minimumValue) +// } + +// func isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { +// num1, err := strconv.ParseFloat(valueToCheck, 64) +// if err != nil { +// return false +// } +// num2, err := strconv.ParseFloat(minimumValue, 64) +// if err != nil { +// return false +// } +// return num1 >= num2 +// } + +// func (t templateHelpers) isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { +// return isGreaterThan(valueToCheck, minimumValue) +// } + +// func isLessThan(valueToCheck, maximumValue string) bool { +// num1, err := strconv.ParseFloat(valueToCheck, 64) +// if err != nil { +// return false +// } +// num2, err := strconv.ParseFloat(maximumValue, 64) +// if err != nil { +// return false +// } +// return num1 < num2 +// } + +// func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { +// return isLessThan(valueToCheck, maximumValue) +// } + +// func isLessThanOrEqual(valueToCheck, maximumValue string) bool { +// num1, err := strconv.ParseFloat(valueToCheck, 64) +// if err != nil { +// return false +// } +// num2, err := strconv.ParseFloat(maximumValue, 64) +// if err != nil { +// return false +// } +// return num1 <= num2 +// } + +// func (t templateHelpers) isLessThanOrEqual(valueToCheck, maximumValue string) bool { +// return isLessThan(valueToCheck, maximumValue) +// } + +// func (t templateHelpers) isBetween(valueToCheck, minimumValue, maximumValue string) bool { +// return t.isGreaterThan(valueToCheck, minimumValue) && t.isLessThan(valueToCheck, maximumValue) +// } + +//######################## + +// Comparison function type that takes two float64 values and returns a boolean. +type comparator func(float64, float64) bool + +// Generic comparison function that parses strings to floats and applies the given comparison function. +func compareValues(value1, value2 string, comp comparator) bool { + num1, err := strconv.ParseFloat(value1, 64) if err != nil { return false } - num2, err := strconv.ParseFloat(minimumValue, 64) + num2, err := strconv.ParseFloat(value2, 64) if err != nil { return false } - return num1 > num2 + return comp(num1, num2) } -func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { - return isGreaterThan(valueToCheck, minimumValue) +// Specific comparison functions using the generic compareValues function. +func isGreaterThan(value1, value2 string) bool { + return compareValues(value1, value2, func(a, b float64) bool { return a > b }) } -func isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { - num1, err := strconv.ParseFloat(valueToCheck, 64) - if err != nil { - return false - } - num2, err := strconv.ParseFloat(minimumValue, 64) - if err != nil { - return false - } - return num1 >= num2 +func isGreaterThanOrEqual(value1, value2 string) bool { + return compareValues(value1, value2, func(a, b float64) bool { return a >= b }) } -func (t templateHelpers) isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { - return isGreaterThan(valueToCheck, minimumValue) +func isLessThan(value1, value2 string) bool { + return compareValues(value1, value2, func(a, b float64) bool { return a < b }) } -func isLessThan(valueToCheck, maximumValue string) bool { - num1, err := strconv.ParseFloat(valueToCheck, 64) - if err != nil { - return false - } - num2, err := strconv.ParseFloat(maximumValue, 64) - if err != nil { - return false - } - return num1 < num2 +func isLessThanOrEqual(value1, value2 string) bool { + return compareValues(value1, value2, func(a, b float64) bool { return a <= b }) } -func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { - return isLessThan(valueToCheck, maximumValue) +func (t templateHelpers) isGreaterThan(value1, value2 string) bool { + return isGreaterThan(value1, value2) } -func isLessThanOrEqual(valueToCheck, maximumValue string) bool { - num1, err := strconv.ParseFloat(valueToCheck, 64) - if err != nil { - return false - } - num2, err := strconv.ParseFloat(maximumValue, 64) - if err != nil { - return false - } - return num1 <= num2 +func (t templateHelpers) isGreaterThanOrEqual(value1, value2 string) bool { + return isGreaterThanOrEqual(value1, value2) } -func (t templateHelpers) isLessThanOrEqual(valueToCheck, maximumValue string) bool { - return isLessThan(valueToCheck, maximumValue) +func (t templateHelpers) isLessThan(value1, value2 string) bool { + return isLessThan(value1, value2) } -func (t templateHelpers) isBetween(valueToCheck, minimumValue, maximumValue string) bool { - return t.isGreaterThan(valueToCheck, minimumValue) && t.isLessThan(valueToCheck, maximumValue) +func (t templateHelpers) isLessThanOrEqual(value1, value2 string) bool { + return isLessThanOrEqual(value1, value2) } +func (t templateHelpers) isBetween(value, min, max string) bool { + return t.isGreaterThan(value, min) && t.isLessThan(value, max) +} + +// ######################## func (t templateHelpers) matchesRegex(valueToCheck, pattern string) bool { re, err := regexp.Compile(pattern) if err != nil { @@ -439,28 +495,11 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { return fmt.Sprintf("%d", numRows) } -// func (t templateHelpers) csvSQL(dataSourceName, queryString string) []RowMap { -// templateDataSources := t.TemplateDataSource.DataSources -// source, exists := templateDataSources[dataSourceName] -// if !exists { -// log.Debug("could not find datasource " + dataSourceName) -// return []RowMap{} -// } -// source.mu.Lock() -// defer source.mu.Unlock() -// query, err := ParseQuery(queryString, source.Data[0]) -// if err != nil { -// log.Error("Error:", err) -// return []RowMap{} -// } -// results := ExecuteQuery(source.Data, query) -// return results -// } func (t templateHelpers) csvSQL(queryString string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources // Parse the query string to get the SelectQuery - query, err := ParseQuery(queryString, templateDataSources) + query, err := parseQuery(strings.TrimSpace(queryString), templateDataSources) if err != nil { log.Debug("Error parsing query:", err) return []RowMap{} @@ -476,9 +515,33 @@ func (t templateHelpers) csvSQL(queryString string) []RowMap { source.mu.Lock() defer source.mu.Unlock() - // Execute the query against the data source - results := ExecuteQuery(source.Data, query) - return results + var results []RowMap + var execErr error + + switch query.Type { + case "SELECT": + results = executeSelectQuery(source.Data, query) //This is not by reference as we are not changing the data + case "UPDATE": + execErr = executeUpdateQuery(&source.Data, query) //This must be by reference as we are not changing the data + case "DELETE": + execErr = executeDeleteQuery(&source.Data, query) //This must be by reference as we are not changing the data + default: + log.Debug(fmt.Errorf("unsupported query type %s", query.Type)) + return nil + } + + if execErr != nil { + log.Debug(fmt.Errorf("error executing query: %w", execErr)) + return nil + } + + // For SELECT queries, return results + if query.Type == "SELECT" { + return results + } + + // No results to return for UPDATE and DELETE queries + return nil } func (t templateHelpers) parseJournalBasedOnIndex(indexName, keyValue, dataSource, queryType, lookupQuery string, options *raymond.Options) interface{} { From ba4ab8e9138a47b130145b876ddef27518b6142f Mon Sep 17 00:00:00 2001 From: stuioco Date: Sun, 25 Aug 2024 16:37:56 +0100 Subject: [PATCH 4/9] SQL over CSV with tests and docs --- core/templating/datasource.go | 9 + core/templating/datasource_query.go | 127 +-- core/templating/datasource_query_test.go | 256 +++++ core/templating/template_helpers.go | 4 +- core/templating/templating_test backup.txt | 934 ++++++++++++++++++ core/templating/templating_test.go | 103 +- .../keyconcepts/templating/templating.rst | 234 +++-- .../github.com/SpectoLabs/raymond/helper.go | 52 + 8 files changed, 1544 insertions(+), 175 deletions(-) create mode 100644 core/templating/datasource_query_test.go create mode 100644 core/templating/templating_test backup.txt diff --git a/core/templating/datasource.go b/core/templating/datasource.go index e1e3abc83..cf5281e1d 100644 --- a/core/templating/datasource.go +++ b/core/templating/datasource.go @@ -53,3 +53,12 @@ func getData(source *DataSource) (string, error) { } return csvData.String(), nil } + +func dataSourceExists(dataSources map[string]*DataSource, name string) bool { + for _, dataSource := range dataSources { + if dataSource.Name == name { + return true + } + } + return false +} diff --git a/core/templating/datasource_query.go b/core/templating/datasource_query.go index 3aaffd174..fd18bb1b1 100644 --- a/core/templating/datasource_query.go +++ b/core/templating/datasource_query.go @@ -2,8 +2,11 @@ package templating import ( "errors" + "fmt" "regexp" "strings" + + log "github.com/sirupsen/logrus" ) // RowMap represents a single row in the result set @@ -16,8 +19,8 @@ type Condition struct { Value string } -// SelectQuery represents a simple SQL-like SELECT query -type SelectQuery struct { +// SQLStatement represents a simple SQL-like query +type SQLStatement struct { Type string // "SELECT", "UPDATE", or "DELETE" Columns []string Conditions []Condition @@ -25,9 +28,8 @@ type SelectQuery struct { DataSourceName string } -func parseQuery(query string, datasource map[string]*DataSource) (SelectQuery, error) { +func parseQuery(query string, datasource map[string]*DataSource) (SQLStatement, error) { query = strings.TrimSpace(query) - var queryType string if strings.HasPrefix(strings.ToUpper(query), "SELECT") { queryType = "SELECT" @@ -36,7 +38,7 @@ func parseQuery(query string, datasource map[string]*DataSource) (SelectQuery, e } else if strings.HasPrefix(strings.ToUpper(query), "DELETE") { queryType = "DELETE" } else { - return SelectQuery{}, errors.New("invalid query type") + return SQLStatement{}, errors.New("invalid query type") } var selectRegex *regexp.Regexp @@ -45,44 +47,61 @@ func parseQuery(query string, datasource map[string]*DataSource) (SelectQuery, e switch queryType { case "SELECT": - selectRegex = regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) + selectRegex = regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+([\w-]+)\s*(?:WHERE\s+(.+))?$`) matches = selectRegex.FindStringSubmatch(query) + for i, _ := range matches { + log.Debug(fmt.Sprint(":", matches[i])) + } + if len(matches) < 3 { - return SelectQuery{}, errors.New("invalid query format") + + return SQLStatement{}, errors.New("invalid query format") + } columnsPart = matches[1] dataSourceName = matches[2] + if !dataSourceExists(datasource, dataSourceName) { + return SQLStatement{}, errors.New("data source does not exist") + } + //if len(matches) == 4 && len(matches[3]) > 0 { if len(matches) == 4 { wherePart = matches[3] } - case "UPDATE": - //selectRegex = regexp.MustCompile(`(?i)^UPDATE\s+(\w+)\s+SET\s+(.+)\s*(?:WHERE\s+(.+))?$`) - selectRegex = regexp.MustCompile(`(?i)^UPDATE\s+(\w+)\s+SET\s+(.+?)(?:\s+WHERE\s+(.+))?$`) + headers := datasource[dataSourceName].Data[0] + columns := parseColumns(columnsPart, headers) + + conditions, err := parseConditions(wherePart) + if err != nil { + return SQLStatement{}, err + } + return SQLStatement{ + Type: queryType, + Columns: columns, + Conditions: conditions, + DataSourceName: dataSourceName, + }, nil + case "UPDATE": + selectRegex = regexp.MustCompile(`(?i)^UPDATE\s+([\w-]+)\s+SET\s+(.+?)(?:\s+WHERE\s+(.+))?$`) matches = selectRegex.FindStringSubmatch(query) if len(matches) < 3 { - return SelectQuery{}, errors.New("invalid UPDATE query format") + return SQLStatement{}, errors.New("invalid UPDATE query format") } dataSourceName = matches[1] + if !dataSourceExists(datasource, dataSourceName) { + return SQLStatement{}, errors.New("data source does not exist") + } setPart := matches[2] if len(matches) == 4 { wherePart = matches[3] } - // log.Debug("########## ParseQuery ############") - // logMessage := fmt.Sprintf("len(matches)=%d", len(matches)) - // log.Debug(logMessage) - // logMessage = fmt.Sprintf("matches[3]=" + matches[3]) - // log.Debug(logMessage) - // formattedMatches := strings.Join(matches, " --- ") - // logMessage = fmt.Sprintf("matches=%v", formattedMatches) - // log.Debug(logMessage) - // log.Debug("######################") + setClauses := parseSetClauses(setPart) conditions, err := parseConditions(wherePart) if err != nil { - return SelectQuery{}, err + return SQLStatement{}, err } - return SelectQuery{ + return SQLStatement{ Type: "UPDATE", Conditions: conditions, SetClauses: setClauses, @@ -90,40 +109,29 @@ func parseQuery(query string, datasource map[string]*DataSource) (SelectQuery, e }, nil case "DELETE": - selectRegex = regexp.MustCompile(`(?i)^DELETE\s+FROM\s+(\w+)\s*(?:WHERE\s+(.+))?$`) + selectRegex = regexp.MustCompile(`(?i)^DELETE\s+FROM\s+([\w-]+)\s*(?:WHERE\s+(.+))?$`) matches = selectRegex.FindStringSubmatch(query) if len(matches) < 2 { - return SelectQuery{}, errors.New("invalid DELETE query format") + return SQLStatement{}, errors.New("invalid DELETE query format") } dataSourceName = matches[1] + if !dataSourceExists(datasource, dataSourceName) { + return SQLStatement{}, errors.New("data source does not exist") + } if len(matches) == 3 { wherePart = matches[2] } conditions, err := parseConditions(wherePart) if err != nil { - return SelectQuery{}, err + return SQLStatement{}, err } - return SelectQuery{ + return SQLStatement{ Type: "DELETE", Conditions: conditions, DataSourceName: dataSourceName, }, nil } - - headers := datasource[dataSourceName].Data[0] - columns := parseColumns(columnsPart, headers) - - conditions, err := parseConditions(wherePart) - if err != nil { - return SelectQuery{}, err - } - - return SelectQuery{ - Type: queryType, - Columns: columns, - Conditions: conditions, - DataSourceName: dataSourceName, - }, nil + return SQLStatement{}, errors.New("invalid query format") } // parseColumns determines the columns to select based on the query part and headers @@ -180,7 +188,7 @@ func parseConditions(wherePart string) ([]Condition, error) { conditionMatches := conditionRegex.FindAllStringSubmatch(wherePart, -1) if len(conditionMatches) == 0 { - return nil, errors.New("no valid conditions found") + return conditions, nil } for _, match := range conditionMatches { @@ -197,28 +205,8 @@ func parseConditions(wherePart string) ([]Condition, error) { return conditions, nil } -// ExecuteQuery processes different types of SQL-like queries based on the query type -// func executeQuery(data *[][]string, query SelectQuery) ([]RowMap, error) { -// switch query.Type { -// case "SELECT": -// return executeSelectQuery(*data, query), nil -// case "UPDATE": -// if err := executeUpdateQuery(data, query); err != nil { -// return nil, err -// } -// return nil, nil // UPDATE doesn't return rows -// case "DELETE": -// if err := executeDeleteQuery(data, query); err != nil { -// return nil, err -// } -// return nil, nil // DELETE doesn't return rows -// default: -// return nil, errors.New("unsupported query type") -// } -// } - // ExecuteSelectQuery executes a SELECT query and returns the results as a slice of RowMaps -func executeSelectQuery(data [][]string, query SelectQuery) []RowMap { +func executeSelectQuery(data [][]string, query SQLStatement) []RowMap { headers := data[0] // First row as header results := []RowMap{} for _, row := range data[1:] { @@ -231,7 +219,7 @@ func executeSelectQuery(data [][]string, query SelectQuery) []RowMap { } // ExecuteUpdateQuery executes an UPDATE query and modifies the data in-place -func executeUpdateQuery(data *[][]string, query SelectQuery) error { +func executeUpdateQuery(data *[][]string, query SQLStatement) error { if len(*data) < 2 { return errors.New("no data available to update") } @@ -254,7 +242,7 @@ func executeUpdateQuery(data *[][]string, query SelectQuery) error { } // ExecuteDeleteQuery executes a DELETE query and modifies the data in-place -func executeDeleteQuery(data *[][]string, query SelectQuery) error { +func executeDeleteQuery(data *[][]string, query SQLStatement) error { if len(*data) < 2 { return errors.New("no data available to delete") } @@ -291,15 +279,6 @@ func mapRow(headers, row []string) RowMap { return rowMap } -// mapToRow converts a map back to a CSV row -// func mapToRow(headers []string, rowMap RowMap) []string { -// row := make([]string, len(headers)) -// for i, header := range headers { -// row[i] = rowMap[header] -// } -// return row -// } - // projectRow filters the row based on the selected columns func projectRow(row RowMap, columns []string) RowMap { if len(columns) == 0 { diff --git a/core/templating/datasource_query_test.go b/core/templating/datasource_query_test.go new file mode 100644 index 000000000..3f03f32ed --- /dev/null +++ b/core/templating/datasource_query_test.go @@ -0,0 +1,256 @@ +package templating + +import ( + "reflect" + "sync" + "testing" +) + +func TestParseQuery(t *testing.T) { + dataSource := map[string]*DataSource{ + "employees": { + SourceType: "csv", + Name: "employees", + Data: [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + }, + mu: sync.Mutex{}, + }, + } + + tests := []struct { + query string + expected SQLStatement + expectError bool + }{ + { + query: "SELECT name, age FROM employees WHERE age >= '30'", + expected: SQLStatement{ + Type: "SELECT", + Columns: []string{"name", "age"}, + Conditions: []Condition{{Column: "age", Operator: ">=", Value: "30"}}, + DataSourceName: "employees", + }, + expectError: false, + }, + { + query: "SELECT * FROM employees WHERE department == 'Engineering'", + expected: SQLStatement{Type: "SELECT", Columns: []string{"id", "name", "age", "department"}, Conditions: []Condition{{Column: "department", Operator: "==", Value: "Engineering"}}, DataSourceName: "employees"}, + expectError: false, + }, + { + query: "UPDATE employees SET age = '35' WHERE name == 'John Doe'", + expected: SQLStatement{Type: "UPDATE", Conditions: []Condition{{Column: "name", Operator: "==", Value: "John Doe"}}, SetClauses: map[string]string{"age": "35"}, DataSourceName: "employees"}, + expectError: false, + }, + { + query: "DELETE FROM employees WHERE age < '30'", + expected: SQLStatement{Type: "DELETE", Conditions: []Condition{{Column: "age", Operator: "<", Value: "30"}}, DataSourceName: "employees"}, + expectError: false, + }, + { + query: "INVALID QUERY", + expectError: true, + }, + } + + for _, test := range tests { + result, err := parseQuery(test.query, dataSource) + if test.expectError { + if err == nil { + t.Errorf("expected error but got none for query: %s", test.query) + } + } else { + if err != nil { + t.Errorf("unexpected error for query %s: %v", test.query, err) + } + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("expected %v, got %v for query: %s", test.expected, result, test.query) + } + } + } +} + +func TestTrimQuotes(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {`"hello"`, "hello"}, + {`'world'`, "world"}, + {`"12345`, `"12345`}, + {`hello`, "hello"}, + {`''`, ""}, + } + + for _, test := range tests { + result := trimQuotes(test.input) + if result != test.expected { + t.Errorf("expected %v, got %v for input %s", test.expected, result, test.input) + } + } +} + +func TestParseSetClauses(t *testing.T) { + input := "age = '35', department = 'Engineering'" + expected := map[string]string{ + "age": "35", + "department": "Engineering", + } + + result := parseSetClauses(input) + if !reflect.DeepEqual(result, expected) { + t.Errorf("expected %v, got %v", expected, result) + } +} + +func TestParseConditions_ValidInput(t *testing.T) { + wherePart := "id == '1' AND name != 'John' AND age >= '30'" + expected := []Condition{ + {"id", "==", "1"}, + {"name", "!=", "John"}, + {"age", ">=", "30"}, + } + + conditions, err := parseConditions(wherePart) + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !reflect.DeepEqual(conditions, expected) { + t.Errorf("expected %v, got %v", expected, conditions) + } +} + +func TestParseConditions_EmptyInput(t *testing.T) { + wherePart := "" + expected := []Condition{} + + conditions, err := parseConditions(wherePart) + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !reflect.DeepEqual(conditions, expected) { + t.Errorf("expected %v, got %v", expected, conditions) + } +} + +func TestParseConditions_SingleCondition(t *testing.T) { + wherePart := "name == 'Alice'" + expected := []Condition{ + {"name", "==", "Alice"}, + } + + conditions, err := parseConditions(wherePart) + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !reflect.DeepEqual(conditions, expected) { + t.Errorf("expected %v, got %v", expected, conditions) + } +} + +func TestParseConditions_MultipleConditions(t *testing.T) { + wherePart := "id == '1' AND age < '40'" + expected := []Condition{ + {"id", "==", "1"}, + {"age", "<", "40"}, + } + + conditions, err := parseConditions(wherePart) + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if !reflect.DeepEqual(conditions, expected) { + t.Errorf("expected %v, got %v", expected, conditions) + } +} + +func TestExecuteSelectQuery(t *testing.T) { + data := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + query := SQLStatement{ + Type: "SELECT", + Columns: []string{"name", "age"}, + Conditions: []Condition{{Column: "department", Operator: "==", Value: "Engineering"}}, + DataSourceName: "employees", + } + + expected := []RowMap{ + {"name": "John Doe", "age": "30"}, + } + + result := executeSelectQuery(data, query) + if !reflect.DeepEqual(result, expected) { + t.Errorf("expected %v, got %v", expected, result) + } +} + +func TestExecuteUpdateQuery(t *testing.T) { + data := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + query := SQLStatement{ + Type: "UPDATE", + Conditions: []Condition{{Column: "name", Operator: "==", Value: "John Doe"}}, + SetClauses: map[string]string{"age": "35"}, + DataSourceName: "employees", + } + + expected := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "35", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + err := executeUpdateQuery(&data, query) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(data, expected) { + t.Errorf("expected %v, got %v", expected, data) + } +} + +func TestExecuteDeleteQuery(t *testing.T) { + data := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + query := SQLStatement{ + Type: "DELETE", + Conditions: []Condition{{Column: "age", Operator: "==", Value: "30"}}, + DataSourceName: "employees", + } + + expected := [][]string{ + {"id", "name", "age", "department"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + err := executeDeleteQuery(&data, query) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(data, expected) { + t.Errorf("expected %v, got %v", expected, data) + } +} diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index aae9c4a2d..c295d3d9b 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -399,6 +399,7 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string { } func (t templateHelpers) csvAsMap(dataSourceName string) []RowMap { + templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { @@ -497,10 +498,11 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { func (t templateHelpers) csvSQL(queryString string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources - // Parse the query string to get the SelectQuery query, err := parseQuery(strings.TrimSpace(queryString), templateDataSources) if err != nil { + // Debugging: Print the parsed query + fmt.Printf("Error parsing query:: %+v\n", err) log.Debug("Error parsing query:", err) return []RowMap{} } diff --git a/core/templating/templating_test backup.txt b/core/templating/templating_test backup.txt new file mode 100644 index 000000000..414277391 --- /dev/null +++ b/core/templating/templating_test backup.txt @@ -0,0 +1,934 @@ +package templating_test + +import ( + "testing" + + "github.com/SpectoLabs/hoverfly/core/journal" + + "github.com/SpectoLabs/hoverfly/core/models" + "github.com/SpectoLabs/hoverfly/core/templating" + . "github.com/onsi/gomega" +) + +// We need to run the csv templating tests first because the datasource is part of the templatingHelper which can be registered only once +// for each runtime, raymond unfortunately doesn't provide a way to unregister helpers for us to isolate the test data. +func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '2' 'Marks'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`56`)) +} + +func Test_ApplyTemplate_ParseCsvAndReturnFallbackStringIfNoMatchFound(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '51' 'Marks'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`ABSENT`)) +} + +func Test_ApplyTemplate_ParseCsvAndReturnQueryStringIfNoMatchFound(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv2' 'Id' '51' 'Marks'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`{{ csv test-csv2 Id 51 Marks }}`)) +} + +func Test_ApplyTemplate_ParseCsvByPassingRequestParamAndReturnMatchValue(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Query: map[string][]string{"Id": {"1"}}, + }, make(map[string]string), `{{csv 'test-csv2' 'Id' 'Request.QueryParam.Id.[0]' 'Marks'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`55`)) +} + +func Test_ApplyTemplate_ParseCsv_WithMissingDataSource(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Query: map[string][]string{"Id": {"1"}}, + }, make(map[string]string), `{{csv 'test-csv3' 'Id' 55 'Marks'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`{{ csv test-csv3 Id 55 Marks }}`)) +} + +func Test_ApplyTemplate_ParseCsv_WithEachBlockAndMissingDataSource(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{"products": [1, 2]}`, + }, make(map[string]string), `{{#each (Request.Body 'jsonpath' '$.products')}} {{@index}} : Product Name with productId {{this}} is {{csv 'products' 'productId' this 'productName'}} {{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(` 0 : Product Name with productId 1 is {{ csv products productId 1 productName }} 1 : Product Name with productId 2 is {{ csv products productId 2 productName }} `)) +} + +// -------------------------------------- +func Test_ApplyTemplate_MatchingRowsCsvAndReturnMatchedString(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '2')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test2`)) +} + +func Test_ApplyTemplate_MatchingRowsCsvMissingDataSource(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv99' 'id' '2')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '99')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +// ------------------------------- +func Test_ApplyTemplate_CsvAsArrayAndReturnMatchedString(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv')}}{{#each this}}{{this}}{{/each}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`idnamemarks1Test1552Test256*DummyABSENT`)) +} + +func Test_ApplyTemplate_CsvAsArrayMissingDataSource(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv99')}}{{#each this}}{{this}}{{/each}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +// ------------------------------- +func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test1Test2Dummy`)) +} + +func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv99')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +// ------------------------------- + +func Test_ApplyTemplate_CsvAddRow(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}{{csv 'test-csv' 'id' '99' 'name'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Violet`)) +} + +func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1`)) +} + +func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv99' 'id' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'identity' '2' true}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_CsvCountRows(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`3`)) +} + +func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv99'}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) +} + +func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test1`)) +} + +// func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { +// RegisterTestingT(t) + +// template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL 'SELECT * FROM test-csv')}}{{this.name}}{{/each}}`) + +// Expect(err).To(BeNil()) +// Expect(template).To(Equal(`Test1Test2Dummy`)) +// } + +func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString2(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test1Test2Dummy`)) +} + +func Test_ApplyTemplate_CsvSQL_SelectAll(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1,Test1,55;2,Test2,56;*,Dummy,ABSENT;`)) +} + +func Test_ApplyTemplate_CsvSQL_Update(t *testing.T) { + RegisterTestingT(t) + + // First, update the data + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) + Expect(err).To(BeNil()) + + // Then, check the result + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`60`)) +} + +func Test_ApplyTemplate_CsvSQL_Delete(t *testing.T) { + RegisterTestingT(t) + + // First, delete the row + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "DELETE FROM test-csv2 WHERE id == '5553686208582'"}}`) + Expect(err).To(BeNil()) + + // Then, check the result + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv2")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1,Test1,55;2,Test2,56;`)) // Test3 entry should be deleted +} + +func Test_ApplyTemplate_CsvSQL_InvalidQuery(t *testing.T) { + RegisterTestingT(t) + + // An invalid query that should fail + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "SELECT foo FROM bar WHERE id == '1'"}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) // No valid data should be returned +} + +func Test_ApplyTemplate_EachBlockWithSplitTemplatingFunction(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), + `{{#each (split 'one,two,three' ',') }}{{this}} | {{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`one | two | three | `)) +} + +func Test_ApplyTemplate_EachBlockWithCsvTemplatingFunction(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Query: map[string][]string{"Ids": {"1", "2"}}, + }, make(map[string]string), `{{#each (Request.QueryParam.Ids) }}{{csv 'test-csv2' 'Id' this 'Marks'}} | {{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`55 | 56 | `)) +} + +func Test_ApplyTemplate_EachBlockWithCsvTemplatingFunctionAndLargeInteger(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{"ids": [1, 5553686208582]}`, + }, make(map[string]string), `{{#each (Request.Body 'jsonpath' '$.ids') }}{{csv 'test-csv2' 'Id' this 'Marks'}} | {{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`55 | 66 | `)) +} + +func Test_ShouldCreateTemplatingDataPathsFromRequest(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + Path: "/foo/bar", + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.Path).To(ConsistOf("foo", "bar")) +} + +func Test_ShouldCreateTemplatingDataPathsFromRequestWithNoPaths(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.Path).To(BeEmpty()) +} + +func Test_ShouldCreateTemplatingDataQueryParamsFromRequest(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + Query: map[string][]string{ + "cheese": {"1", "3"}, + "ham": {"2"}, + }, + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.QueryParam).To(HaveKeyWithValue("cheese", []string{"1", "3"})) + Expect(actual.Request.QueryParam).To(HaveKeyWithValue("ham", []string{"2"})) + Expect(actual.Request.QueryParam).To(HaveLen(2)) +} + +func Test_ShouldCreateTemplatingDataQueryParamsFromRequestWithNoQueryParams(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.QueryParam).To(BeEmpty()) +} + +func Test_ShouldCreateTemplatingDataHttpScheme(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.Scheme).To(Equal("http")) +} + +func Test_ShouldCreateTemplatingDataHeaderFromRequest(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + Headers: map[string][]string{ + "cheese": {"1", "3"}, + "ham": {"2"}, + }, + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.Header).To(HaveKeyWithValue("cheese", []string{"1", "3"})) + Expect(actual.Request.Header).To(HaveKeyWithValue("ham", []string{"2"})) + Expect(actual.Request.Header).To(HaveLen(2)) +} + +func Test_ShouldCreateTemplatingDataHeaderFromRequestWithNoHeader(t *testing.T) { + RegisterTestingT(t) + + actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(actual.Request.Header).To(BeEmpty()) +} + +func TestApplyTemplateWithRequestDetails(t *testing.T) { + RegisterTestingT(t) + + requestDetails := &models.RequestDetails{ + Method: "GET", + Scheme: "http", + Destination: "foo.com", + Path: "/foo/bar", + Query: map[string][]string{ + "singular": {"1"}, + "multiple": {"2", "3"}, + }, + Headers: map[string][]string{ + "X-Singular": {"a"}, + "X-Multiple": {"b", "c"}, + }, + } + + template, err := ApplyTemplate(requestDetails, + make(map[string]string), + ` +Scheme: {{ Request.Scheme }} + +Query param value: {{ Request.QueryParam.singular }} + +Query param value by index: {{ Request.QueryParam.multiple.[0] }} +Query param value by index: {{ Request.QueryParam.multiple.[1] }} + +List of query param values: {{ Request.QueryParam.multiple}} +Looping through query params: {{#each Request.QueryParam.multiple}}{{ this }}-{{/each}} + +Header value: {{ Request.Header.X-Singular }} +Header value by index: {{ Request.Header.X-Multiple.[0] }} +Header value by index: {{ Request.Header.X-Multiple.[1] }} +List of header values: {{ Request.Header.X-Multiple}} + +Path param value: {{ Request.Path.[0] }} +All path param values: {{ Request.Path }} +Looping through path params: {{#each Request.Path}}{{ this }}-{{/each}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal(` +Scheme: http + +Query param value: 1 + +Query param value by index: 2 +Query param value by index: 3 + +List of query param values: 23 +Looping through query params: 2-3- + +Header value: a +Header value by index: b +Header value by index: c +List of header values: bc + +Path param value: foo +All path param values: foobar +Looping through path params: foo-bar-`)) +} + +func TestTemplatingWithParametersWhichDoNotExistDoNotErrorAndAreEmpty(t *testing.T) { + RegisterTestingT(t) + + requestDetails := &models.RequestDetails{ + Method: "GET", + Scheme: "http", + Destination: "foo.com", + } + + template, err := ApplyTemplate(requestDetails, + map[string]string{ + "one": "A", + "two": "B", + }, + ` +Scheme:{{ Request.Scheme }} + +Query param value:{{ Request.QueryParam.singular }} + +Query param value by index:{{ Request.QueryParam.multiple.[0] }} +Query param value by index:{{ Request.QueryParam.multiple.[1] }} + +List of query param values:{{ Request.QueryParam.multiple}} +Looping through query params:{{#each Request.QueryParam.multiple}}{{ this }}{{/each}} + +Header value: {{ Request.Header.X-Singular }} +Header value by index: {{ Request.Header.X-Multiple.[0] }} +Header value by index: {{ Request.Header.X-Multiple.[1] }} +List of header values: {{ Request.Header.X-Multiple}} + +Path param value:{{ Request.Path.[0] }} +All path param values:{{ Request.Path }} +Looping through path params:{{#each Request.Path}}{{ this }}{{/each}} + +State One: {{ State.one }} +State Two: {{ State.two }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal(` +Scheme:http + +Query param value: + +Query param value by index: +Query param value by index: + +List of query param values: +Looping through query params: + +Header value: +Header value by index: +Header value by index: +List of header values: + +Path param value: +All path param values: +Looping through path params: + +State One: A +State Two: B`)) +} + +func Test_ApplyTemplate_now(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{now "" "unix"}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{now "" "unix"}}`)))) +} + +func Test_ApplyTemplate_randomString(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomString}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomString}}`)))) +} + +func Test_ApplyTemplate_randomStringLength(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomStringLength 2}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomStringLength 2}}`)))) + Expect(template).To(HaveLen(2)) +} + +func Test_ApplyTemplate_randomBoolean(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomBoolean}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomBoolean}}`)))) +} + +func Test_ApplyTemplate_randomInteger(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomInteger}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomInteger}}`)))) +} + +func Test_ApplyTemplate_randomIntegerRange(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIntegerRange 7 8}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomIntegerRange 7 8}}`)))) +} + +func Test_ApplyTemplate_randomFloat(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomFloat}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomFloat}}`)))) +} + +func Test_ApplyTemplate_randomFloatRange(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomFloatRange 7.0 8.0}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomFloatRange 7.0 8.0}}`)))) +} + +func Test_ApplyTemplate_randomEmail(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomEmai}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Not(Equal(ContainSubstring(`{{randomEmail}}`)))) +} + +func Test_ApplyTemplate_randomIPv4(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIPv4}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomIPv4}}`)))) +} + +func Test_ApplyTemplate_randomIPv6(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIPv6}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomIPv6}}`)))) +} + +func Test_ApplyTemplate_randomUuid(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomUuid}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Not(Equal(ContainSubstring(`{{randomUuid}}`)))) +} + +func Test_ApplyTemplate_Request_Body_Jsonpath(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{ "name": "Ben" }`, + }, make(map[string]string), `{{ Request.Body 'jsonpath' '$.name' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("Ben")) +} + +func Test_ApplyTemplate_Request_Body_JsonPath_Unescaped(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{ "name": "O'Reilly" }`, + }, make(map[string]string), `{{{ Request.Body 'jsonpath' '$.name' }}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("O'Reilly")) +} + +func Test_ApplyTemplate_Request_Body_Jsonpath_LargeInt(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{ "id": 5553686208582 }`, + }, make(map[string]string), `{{ Request.Body 'jsonpath' '$.id' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("5553686208582")) +} + +func Test_ApplyTemplate_Request_Body_Jsonpath_From_Xml(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `Johnny`, + }, make(map[string]string), `{{ Request.Body 'jsonpathfromxml' '$.name' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("Johnny")) +} + +func Test_ApplyTemplate_ReplaceStringInQueryParams(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Query: map[string][]string{ + "sound": {"oink,oink,oink"}, + }, + }, make(map[string]string), `{{ replace Request.QueryParam.sound 'oink' 'moo' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal(`moo,moo,moo`)) +} + +func Test_VarSetToNilInCaseOfInvalidArgsPassed(t *testing.T) { + RegisterTestingT(t) + templator := templating.NewTemplator() + + vars := &models.Variables{ + models.Variable{ + Name: "varOne", + Function: "faker", + }, + } + + actual := templator.NewTemplatingData(&models.RequestDetails{ + Scheme: "http", + Destination: "test.com", + }, nil, &models.Literals{}, vars, make(map[string]string), &journal.Journal{}) + + Expect(actual.Vars["varOne"]).To(BeNil()) + +} + +func Test_VarSetToProperValueInCaseOfRequestDetailsPassedAsArgument(t *testing.T) { + RegisterTestingT(t) + templator := templating.NewTemplator() + argumentsArray := toInterfaceSlice([]string{"Request.Path.[1]", ","}) + vars := &models.Variables{ + models.Variable{ + Name: "splitRequestPath", + Function: "split", + Arguments: argumentsArray, + }, + } + + actual := templator.NewTemplatingData(&models.RequestDetails{ + Path: "/part1/foo,bar", + }, nil, &models.Literals{}, vars, make(map[string]string), &journal.Journal{}) + + Expect(actual.Vars["splitRequestPath"]).ToNot(BeNil()) + Expect(len(actual.Vars["splitRequestPath"].([]string))).To(Equal(2)) + Expect(actual.Vars["splitRequestPath"].([]string)[0]).To(Equal("foo")) + Expect(actual.Vars["splitRequestPath"].([]string)[1]).To(Equal("bar")) + +} + +func Test_ApplyTemplate_add_integers(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 1 2 '0'}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("3")) +} + +func Test_ApplyTemplate_add_floats(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 '0.00'}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("1.44")) +} + +func Test_ApplyTemplate_add_floats_withRoundUp(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 '0.0'}} and {{ add 0.1 1.56 '0.0'}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("1.4 and 1.7")) +} + +func Test_ApplyTemplate_add_number_without_format(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 ''}} and {{ add 1 2 ''}} and {{ add 0 0 ''}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("1.44 and 3 and 0")) +} + +func Test_ApplyTemplate_add_NotNumber(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 'a' 'b' '0.00'}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("NaN")) +} + +func Test_ApplyTemplate_subtract_numbers(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ subtract 10 0.99 ''}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("9.01")) +} + +func Test_ApplyTemplate_mutiply_numbers(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ multiply 10 0.99 ''}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("9.9")) +} + +func Test_ApplyTemplate_divide_numbers(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ divide 10 2.5 ''}}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("4")) +} + +func Test_ApplyTemplate_Arithmetic_Ops_With_Each_Block(t *testing.T) { + RegisterTestingT(t) + + templator := templating.NewTemplator() + + requestDetails := &models.RequestDetails{ + Body: `{"lineitems":{"lineitem":[{"upc":"1001","quantity":"1","price":"3.50"},{"upc":"1002","quantity":"2","price":"4.50"}]}}`, + } + responseBody := `{{#each (Request.Body 'jsonpath' '$.lineitems.lineitem') }} {{ addToArray 'subtotal' (multiply (this.price) (this.quantity) '') true }} {{/each}} total: {{ sum (getArray 'subtotal') '0.00' }}` + + template, _ := templator.ParseTemplate(responseBody) + state := make(map[string]string) + result, err := templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) + + Expect(err).To(BeNil()) + Expect(result).To(Equal(` 3.5 9 total: 12.50`)) + + // Running the second time should produce the same result because each execution has its own context data. + result, err = templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) + Expect(err).To(BeNil()) + Expect(result).To(Equal(` 3.5 9 total: 12.50`)) +} + +func Test_ApplyTemplate_PutAndGetValue(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{ "id": 5553686208582 }`, + }, make(map[string]string), `{{ putValue 'id' (Request.Body 'jsonpath' '$.id') true }} The ID was {{ getValue 'id' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("5553686208582 The ID was 5553686208582")) +} + +func Test_ApplyTemplate_PutAndGetValue_SuppressOutput(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `{ "id": 5553686208582 }`, + }, make(map[string]string), `{{ putValue 'id' (Request.Body 'jsonpath' '$.id') false }}The ID was {{ getValue 'id' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("The ID was 5553686208582")) +} + +func Test_ApplyTemplate_setStatusCode(t *testing.T) { + RegisterTestingT(t) + + templator := templating.NewTemplator() + + template, err := templator.ParseTemplate(`{{ setStatusCode 400 }}`) + Expect(err).To(BeNil()) + + response := &models.ResponseDetails{} + result, err := templator.RenderTemplate(template, &models.RequestDetails{}, response, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(err).To(BeNil()) + Expect(result).To(Equal("")) + Expect(response.Status).To(Equal(400)) +} + +func Test_ApplyTemplate_setStatusCode_should_ignore_invalid_code(t *testing.T) { + RegisterTestingT(t) + + templator := templating.NewTemplator() + + template, err := templator.ParseTemplate(`{{ setStatusCode 600 }}`) + Expect(err).To(BeNil()) + + response := &models.ResponseDetails{} + result, err := templator.RenderTemplate(template, &models.RequestDetails{}, response, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) + + Expect(err).To(BeNil()) + Expect(result).To(Equal("")) + Expect(response.Status).To(Equal(0)) +} + +func Test_ApplyTemplate_setStatusCode_should_handle_nil_response(t *testing.T) { + RegisterTestingT(t) + + // ApplyTemplate pass a nil response value to RenderTemplate + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ setStatusCode 400 }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("")) +} + +func toInterfaceSlice(arguments []string) []interface{} { + argumentsArray := make([]interface{}, len(arguments)) + + for i, s := range arguments { + argumentsArray[i] = s + } + return argumentsArray +} + +func ApplyTemplate(requestDetails *models.RequestDetails, state map[string]string, responseBody string) (string, error) { + + templator := templating.NewTemplator() + dataSource1, _ := templating.NewCsvDataSource("test-csv", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") + dataSource2, _ := templating.NewCsvDataSource("test-csv2", "id,name,marks\n1,Test1,55\n2,Test2,56\n5553686208582,Test3,66\n") + templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv", dataSource1) + templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv2", dataSource2) + + template, err := templator.ParseTemplate(responseBody) + Expect(err).To(BeNil()) + return templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) +} diff --git a/core/templating/templating_test.go b/core/templating/templating_test.go index 27bffd305..0c112cd3c 100644 --- a/core/templating/templating_test.go +++ b/core/templating/templating_test.go @@ -15,7 +15,7 @@ import ( func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv1' 'Id' '2' 'Marks'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '2' 'Marks'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`56`)) @@ -24,7 +24,7 @@ func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { func Test_ApplyTemplate_ParseCsvAndReturnFallbackStringIfNoMatchFound(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv1' 'Id' '51' 'Marks'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '51' 'Marks'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`ABSENT`)) @@ -76,7 +76,7 @@ func Test_ApplyTemplate_ParseCsv_WithEachBlockAndMissingDataSource(t *testing.T) func Test_ApplyTemplate_MatchingRowsCsvAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv1' 'id' '2')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '2')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test2`)) @@ -94,7 +94,7 @@ func Test_ApplyTemplate_MatchingRowsCsvMissingDataSource(t *testing.T) { func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv1' 'id' '99')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '99')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(``)) @@ -104,7 +104,7 @@ func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { func Test_ApplyTemplate_CsvAsArrayAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv1')}}{{#each this}}{{this}}{{/each}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv')}}{{#each this}}{{this}}{{/each}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`idnamemarks1Test1552Test256*DummyABSENT`)) @@ -123,7 +123,7 @@ func Test_ApplyTemplate_CsvAsArrayMissingDataSource(t *testing.T) { func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv1')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test1Test2Dummy`)) @@ -143,19 +143,23 @@ func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) { func Test_ApplyTemplate_CsvAddRow(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv1' (getArray 'newMark')}}{{csv 'test-csv1' 'id' '99' 'name'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}{{csv 'test-csv' 'id' '99' 'name'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Violet`)) + //Revert the data to original state + _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '99' false}}`) } func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'id' '2' true}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '*' true}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`1`)) + //Revert the data to original state + _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '*' false}}{{addToArray 'newMark' 'Dummy' false}}{{addToArray 'newMark' 'ABSENT' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}`) } func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { @@ -170,7 +174,7 @@ func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'identity' '2' true}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'identity' '2' true}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(``)) @@ -179,7 +183,7 @@ func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { func Test_ApplyTemplate_CsvCountRows(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv1'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`3`)) @@ -194,6 +198,80 @@ func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { Expect(template).To(Equal(``)) } +func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test1`)) +} + +// func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { +// RegisterTestingT(t) + +// template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL 'SELECT * FROM test-csv')}}{{this.name}}{{/each}}`) + +// Expect(err).To(BeNil()) +// Expect(template).To(Equal(`Test1Test2Dummy`)) +// } + +func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString2(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`Test1Test2Dummy`)) +} + +func Test_ApplyTemplate_CsvSQL_SelectAll(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv WHERE 1==1")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1,Test1,55;2,Test2,56;*,Dummy,ABSENT;`)) +} + +func Test_ApplyTemplate_CsvSQL_Update(t *testing.T) { + RegisterTestingT(t) + + // First, update the data + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) + Expect(err).To(BeNil()) + + // Then, check the result + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`60`)) +} + +func Test_ApplyTemplate_CsvSQL_Delete(t *testing.T) { + RegisterTestingT(t) + + // First, delete the row + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "DELETE FROM test-csv2 WHERE id == '5553686208582'"}}`) + Expect(err).To(BeNil()) + + // Then, check the result + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv2")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(`1,Test1,55;2,Test2,56;`)) // Test3 entry should be deleted +} + +func Test_ApplyTemplate_CsvSQL_InvalidQuery(t *testing.T) { + RegisterTestingT(t) + + // An invalid query that should fail + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT foo FROM bar WHERE id == '1'")}} {{/each}}`) + + Expect(err).To(BeNil()) + Expect(template).To(Equal(``)) // No valid data should be returned +} + func Test_ApplyTemplate_EachBlockWithSplitTemplatingFunction(t *testing.T) { RegisterTestingT(t) @@ -847,10 +925,11 @@ func toInterfaceSlice(arguments []string) []interface{} { } func ApplyTemplate(requestDetails *models.RequestDetails, state map[string]string, responseBody string) (string, error) { + templator := templating.NewTemplator() - dataSource1, _ := templating.NewCsvDataSource("test-csv1", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") + dataSource1, _ := templating.NewCsvDataSource("test-csv", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") dataSource2, _ := templating.NewCsvDataSource("test-csv2", "id,name,marks\n1,Test1,55\n2,Test2,56\n5553686208582,Test3,66\n") - templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv1", dataSource1) + templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv", dataSource1) templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv2", dataSource2) template, err := templator.ParseTemplate(responseBody) diff --git a/docs/pages/keyconcepts/templating/templating.rst b/docs/pages/keyconcepts/templating/templating.rst index 929659603..fff283e79 100644 --- a/docs/pages/keyconcepts/templating/templating.rst +++ b/docs/pages/keyconcepts/templating/templating.rst @@ -173,7 +173,9 @@ Fakers that require arguments are currently not supported. CSV Data Source ~~~~~~~~~~~~~~~ -You can both query data from a CSV data source as well as manipulate data within a data source byadding to it and deleting from it. +You can both query data from a CSV data source as well as manipulate data within a data source by adding to it and deleting from it. +Hoverfly supports a number of templating methods for simple read, update and delete functions. +In addition Hoverfly supports a templating function that allows simple SQL like commands for SELECT, UPDATE and DELETE. Reading from a CSV Data Source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -183,11 +185,9 @@ You can read data from a CSV data source in a number of ways. The most basic is to return the value of one field (selected-column) given a field name to search (column-name) and a value to search for in that field (query-value). Of course the query-value would normally be pulled from the request. -.. code:: json +.. code:: handlebars - { - "body": "{\"name\": \"{{csv '(data-source-name)' '(column-name)' '(query-value)' '(selected-column)' }}\"}" - } + {{csv '(data-source-name)' '(column-name)' '(query-value)' '(selected-column)' }} .. note:: @@ -239,30 +239,30 @@ Additional functions are avaiable to query the CSV data source to return all or This makes it simpler to render back into the template, as you can use the {{this}} expression with the column names to identify which fields you want to render. -To return all the data from the csv as an array of maps: +To return all the data from the csv as an array of maps. (Note that you would need to wrap this in an #each block. See examples below.): -.. code:: json +.. code:: handlebars - { - "body": "{\"name\": \"{{csvAsMap '(data-source-name)' }}\"}" - } + {{csvAsMap '(data-source-name)' }} + -To return filtered data from the csv as an array of maps: +To return filtered data from the csv as an array of maps (Note that you would need to wrap this in an #each block. See examples below.): -.. code:: json +.. code:: handlebars - { - "body": "{\"name\": \"{{csvMatchingRows '(data-source-name)' '(column-name)' '(query-value)' '(selected-column)'}}\"}" - } + {{csvMatchingRows '(data-source-name)' '(column-name)' '(query-value)' '(selected-column)'}} -To return all the data from the csv as an array arrays: +To return all the data from the csv as an array arrays (Note that you would need to wrap this in an #each block. See examples below.): -.. code:: json +.. code:: handlebars - { - "body": "{\"name\": \"{{csvAsArray '(data-source-name)' }}\"}" - } + {{csvAsArray '(data-source-name)' }} + +To return filtered data from the csv as an array of maps using the SQL like dialect. Again this needs to be wrapped in an #each block: + +.. code:: handlebars + {{csvSQL '(sql-select-query)'}} Example: Start Hoverfly with a CSV data source (pets.csv) provided below. @@ -280,70 +280,85 @@ Example: Start Hoverfly with a CSV data source (pets.csv) provided below. +-----------+-----------+---------+-----------+ ++--------------------------+------------------------------------------+-----------------------------------------+ +| Description | Example | Result | ++--------------------------+------------------------------------------+-----------------------------------------+ +| Return all the results | { | { | +| in an array of maps | "All-The-Pets": [ | "All-The-Pets": [ | +| and render them in JSON | {{#each (csvAsMap 'pets')}} | { | +| *Note the use of | { | "id": 1000, | +| this.column-name | "id":{{this.id}}, | "category": "cats", | +| as we have map | "category":"{{this.category}}", | "name": "Sylvester", | +| | "name":"{{this.name}}", | "status": "available" | +| | "status":"{{this.status}}" | }, | +| | }{{#unless @last}},{{/unless}} | { | +| | {{/each}} | "id": 1001, | +| | ] | "category": "dogs", | +| | } | "name": "Zipper", | +| | | "status": "available" | +| | | }, | +| | | { | +| | | "id": 1002, | +| | | "category": "dogs", | +| | | "name": "Teddy", | +| | | "status": "sold" | +| | | } | +| | | ] | +| | | } | ++--------------------------+------------------------------------------+-----------------------------------------+ +| Return filtered data as | { | { | +| an array of maps | "Dogs-Only": [ | "Dogs-Only": [ | +| and render them in JSON | {{#each (csvMatchingRows 'pets' | { | +| *Note the use of | 'category' 'dogs')}} | "id": 1001, | +| this.column-name | { | "category": "dogs", | +| as we have a map | "id":{{this.id}}, | "name": "Zipper", | +| | "category":"{{this.category}}", | "status": "available" | +| | "name":"{{this.name}}", | }, | +| | "status":"{{this.status}}" | { | +| | }{{#unless @last}},{{/unless}} | "id": 1002, | +| | {{/each}} | "category": "dogs", | +| | ] | "name": "Teddy", | +| | } | "status": "sold" | +| | | } | +| | | ] | +| | | } | ++--------------------------+------------------------------------------+-----------------------------------------+ +| SELECT data using a SQL | { | { | +| like syntax. | "Dogs-With-Big-Ids-Only": [ | "Dogs-With-Big-Ids-Only": [ | +| | {{#each (csvSQL "SELECT * FROM pets | { | +| | WHERE category == 'dogs' AND id >= | "id":2000, | +| | '2000'")}} | "category":"dogs", | +| | { | "name":"Violet", | +| | "id":{{this.id}}, | "status":"sold" | +| | "category":"{{this.category}}", | } | +| | "name":"{{this.name}}", | ] | +| | "status":"{{this.status}}" | } | +| | }{{#unless @last}},{{/unless}} | | +| | {{/each}} | | +| | ] | | +| | } | | ++--------------------------+------------------------------------------+-----------------------------------------+ +| Return all the data as | {{#each (csvAsArray 'pets')}} | id category name status | +| an array of arrays | {{#each this}}{{this}} {{/each}} | | +| | | 1000 cats Sylvester available | +| | | | +| | | 1001 dogs Zipper available | +| | | | +| | | 1002 dogs Teddy sold | ++--------------------------+------------------------------------------+-----------------------------------------+ -+--------------------------+------------------------------------------------------------+-----------------------------------------+ -| Description | Example | Result | -+--------------------------+------------------------------------------------------------+-----------------------------------------+ -| Return all the results | { | { | -| in an array of maps | "All-The-Pets": [ | "All-The-Pets": [ | -| and render them in JSON | {{#each (csvAsMap 'pets')}} | { | -| *Note the use of | { | "id": 1000, | -| this.column-name | "id":{{this.id}}, | "category": "cats", | -| as we have map | "category":"{{this.category}}", | "name": "Sylvester", | -| | "name":"{{this.name}}", | "status": "available" | -| | "status":"{{this.status}}" | }, | -| | }{{#unless @last}},{{/unless}} | { | -| | {{/each}} | "id": 1001, | -| | ] | "category": "dogs", | -| | } | "name": "Zipper", | -| | | "status": "available" | -| | | }, | -| | | { | -| | | "id": 1002, | -| | | "category": "dogs", | -| | | "name": "Teddy", | -| | | "status": "sold" | -| | | } | -| | | ] | -| | | } | -+--------------------------+------------------------------------------------------------+-----------------------------------------+ -| Return filtered data as | { | { | -| an array of maps | "Dogs-Only": [ | "Dogs-Only": [ | -| and render them in JSON | {{#each (csvMatchingRows 'pets' 'category' 'dogs')}} | { | -| *Note the use of | { | "id": 1001, | -| this.column-name | "id":{{this.id}}, | "category": "dogs", | -| as we have map | "category":"{{this.category}}", | "name": "Zipper", | -| | "name":"{{this.name}}", | "status": "available" | -| | "status":"{{this.status}}" | }, | -| | }{{#unless @last}},{{/unless}} | { | -| | {{/each}} | "id": 1002, | -| | ] | "category": "dogs", | -| | } | "name": "Teddy", | -| | | "status": "sold" | -| | | } | -| | | ] | -| | | } | -+--------------------------+------------------------------------------------------------+-----------------------------------------+ -| Return all the data as | {{#each (csvAsArray 'pets')}} | id category name status | -| an array of arrays | {{#each this}}{{this}} {{/each}} | | -| | | 1000 cats Sylvester available | -| | | | -| | | 1001 dogs Zipper available | -| | | | -| | | 1002 dogs Teddy sold | -+--------------------------+------------------------------------------------------------+-----------------------------------------+ Adding data to a CSV Data Source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ While the service is running you can add new rows of data into the data source. This is not peristent, it is only manipulated in memory -and so it only lasts for as long as the service is running. The rows are not actually written to the file. +and so it only lasts for as long as the service is running and between calls. The rows are not actually written to the file. -.. code:: json +This is currently the only way to add rows to the datasource. There is no support for SQL INSERT statements. - { - "body": "{\"name\": \"{{csvAddRow '(data-source-name)' (array-of-values)}}\"}" - } +.. code:: handlebars + + {{csvAddRow '(data-source-name)' (array-of-values)}} To use this function you first need to construct an array containing the row of string values to store in the csv data source. @@ -365,13 +380,14 @@ Deleting data from a CSV Data Source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ While the service is running you can delete rows of data from the data source. This is not peristent, it is only manipulated in memory -and so it only lasts for as long as the service is running. The rows are not actually deleted from the file. +and so it only lasts for as long as the service is running and between calls. The rows are not actually deleted from the file. -.. code:: json +There are two ways to delete rows. This first simple method will delete rows that match one exact condition. + +.. code:: handlebars + + {{csvDeleteRows '(data-source-name)' '(column-name)' '(query-value)' (output-result)}} - { - "body": "{\"name\": \"{{csvDeleteRows '(data-source-name)' '(column-name)' '(query-value)' (output-result)}}\"}" - } To delete rows from the csv data source your specify the value that a specific column must have to be deleted. @@ -380,9 +396,9 @@ To delete all the pets where the category is cats from the pets csv data store: ``{{ csvDeleteRows 'pets' 'category' 'cats' false }}`` Note that the last parameter of "output-result" is a boolean. It is not enclosed in quotes. The function will return the number of rows -affected which can either be suppressed by passing false, or passed into another function if you need to make logical decisions based on the number of -rows affected by passing in true. If csvDeleteRows is not enclosed within another function it will output the number of rows -deleted to the template. +affected which can either be suppressed by passing false to this parameter, or passed into another function if you need to make logical +decisions based on the number of rows affected by passing in true as the last parameter. If csvDeleteRows is not enclosed within another +function it will output the number of rows deleted to the template. ``{{#equal (csvDeleteRows 'pets' 'category' 'cats' true) '0'}}`` `` {{ setStatusCode '404' }}`` @@ -392,17 +408,59 @@ deleted to the template. `` {"Message":"All cats deleted"}`` ``{{/equal}}`` +Deleting data from a CSV Data Source using SQL like syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: handlebars + + {{csvSQL '(sql-delete-statement)'}} + +Example: +``{{ csvSQL "DELETE FROM pets WHERE id > '20'" }}`` + +Update the data in a CSV Data Source using SQL like syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: handlebars + + {{csvSQL '(sql-update-statement)'}} + +Example: +``{{ csvSQL "UPDATE pets SET status = 'sold' WHERE id > '20' AND category == 'cats'" }}`` + +Using SQL like syntax to query and manipulate data sources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can use a simplified SQL like syntax to select, update and delete rows. + +INSERT is not supported. To add data you must use the csvAddRow template function. +SELECT [column-names] FROM [data-source-name] WHERE [conditions] (* can be used to indicate all colmuns) +UPDATE [data-source-name] SET [[column-name] = '[value]',] WHERE [conditions] +DELETE FROM [data-source-name] WHERE [conditions] + +Only simple conditions are supported. + +You can chain conditions using AND. OR is not supported. + +The following comparison operators are supported in conditions: +== equals +>= greater than or equal to +<= less than or equal to +!= not equal to + +Capitalisation of SQL keywords is required. +Spaces between components of the SQL statement are required. +Every value provided to a condition whether a number or a string must be enclosed in quotes. +Joins across different data sources are not supported. + Counting the rows in a CSV Data Source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can return the number of rows in a csv dataset. This will be 1 less than the number of rows as the first row contains the column names. -.. code:: json +.. code:: handlebars - { - "body": "{\"name\": \"{{csvCountRows '(data-source-name)'}}\"}" - } + {{csvCountRows '(data-source-name)'}} Journal Entry Data diff --git a/vendor/github.com/SpectoLabs/raymond/helper.go b/vendor/github.com/SpectoLabs/raymond/helper.go index 23b7c3490..e1ce7acf6 100644 --- a/vendor/github.com/SpectoLabs/raymond/helper.go +++ b/vendor/github.com/SpectoLabs/raymond/helper.go @@ -29,6 +29,7 @@ func init() { RegisterHelper("unless", unlessHelper) RegisterHelper("with", withHelper) RegisterHelper("each", eachHelper) + RegisterHelper("first", firstHelper) RegisterHelper("log", logHelper) RegisterHelper("lookup", lookupHelper) RegisterHelper("equal", equalHelper) @@ -389,6 +390,57 @@ func eachHelper(context interface{}, options *Options) interface{} { return result } +func firstHelper(context interface{}, options *Options) interface{} { + if !IsTrue(context) { + return options.Inverse() + } + + val := reflect.ValueOf(context) + var result string + + switch val.Kind() { + case reflect.Array, reflect.Slice: + if val.Len() > 0 { + // computes private data + data := options.newIterDataFrame(val.Len(), 0, nil) + + // evaluates block + result = options.evalBlock(val.Index(0).Interface(), data, 0) + } + case reflect.Map: + keys := val.MapKeys() + if len(keys) > 0 { + key := keys[0].Interface() + ctx := val.MapIndex(keys[0]).Interface() + + // computes private data + data := options.newIterDataFrame(len(keys), 0, key) + + // evaluates block + result = options.evalBlock(ctx, data, key) + } + case reflect.Struct: + if val.NumField() > 0 { + // get the first exported field + for i := 0; i < val.NumField(); i++ { + if tField := val.Type().Field(i); tField.PkgPath == "" { + key := tField.Name + ctx := val.Field(i).Interface() + + // computes private data + data := options.newIterDataFrame(1, 0, key) + + // evaluates block + result = options.evalBlock(ctx, data, key) + break // only process the first field + } + } + } + } + + return result +} + // #log helper func logHelper(message string) interface{} { log.Print(message) From f646d99c9dc30a6d3e5edc0c84b0451b4b33b664 Mon Sep 17 00:00:00 2001 From: stuioco Date: Mon, 26 Aug 2024 09:32:01 +0100 Subject: [PATCH 5/9] Fixes based on initial review --- core/templating/datasource.go | 9 - core/templating/datasource_query.go | 35 +- core/templating/datasource_query_test.go | 21 +- core/templating/template_datasource.go | 13 + core/templating/template_helpers.go | 125 +-- core/templating/templating.go | 2 +- core/templating/templating_test backup.txt | 934 ------------------ core/templating/templating_test.go | 12 +- .../keyconcepts/templating/templating.rst | 14 +- 9 files changed, 79 insertions(+), 1086 deletions(-) delete mode 100644 core/templating/templating_test backup.txt diff --git a/core/templating/datasource.go b/core/templating/datasource.go index cf5281e1d..e1e3abc83 100644 --- a/core/templating/datasource.go +++ b/core/templating/datasource.go @@ -53,12 +53,3 @@ func getData(source *DataSource) (string, error) { } return csvData.String(), nil } - -func dataSourceExists(dataSources map[string]*DataSource, name string) bool { - for _, dataSource := range dataSources { - if dataSource.Name == name { - return true - } - } - return false -} diff --git a/core/templating/datasource_query.go b/core/templating/datasource_query.go index fd18bb1b1..baf4d6ac6 100644 --- a/core/templating/datasource_query.go +++ b/core/templating/datasource_query.go @@ -2,11 +2,8 @@ package templating import ( "errors" - "fmt" "regexp" "strings" - - log "github.com/sirupsen/logrus" ) // RowMap represents a single row in the result set @@ -28,15 +25,15 @@ type SQLStatement struct { DataSourceName string } -func parseQuery(query string, datasource map[string]*DataSource) (SQLStatement, error) { +func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement, error) { query = strings.TrimSpace(query) - var queryType string + var commandType string if strings.HasPrefix(strings.ToUpper(query), "SELECT") { - queryType = "SELECT" + commandType = "SELECT" } else if strings.HasPrefix(strings.ToUpper(query), "UPDATE") { - queryType = "UPDATE" + commandType = "UPDATE" } else if strings.HasPrefix(strings.ToUpper(query), "DELETE") { - queryType = "DELETE" + commandType = "DELETE" } else { return SQLStatement{}, errors.New("invalid query type") } @@ -45,14 +42,10 @@ func parseQuery(query string, datasource map[string]*DataSource) (SQLStatement, var matches []string var columnsPart, dataSourceName, wherePart string - switch queryType { + switch commandType { case "SELECT": selectRegex = regexp.MustCompile(`(?i)^SELECT\s+(.+)\s+FROM\s+([\w-]+)\s*(?:WHERE\s+(.+))?$`) matches = selectRegex.FindStringSubmatch(query) - for i, _ := range matches { - log.Debug(fmt.Sprint(":", matches[i])) - } - if len(matches) < 3 { return SQLStatement{}, errors.New("invalid query format") @@ -63,11 +56,11 @@ func parseQuery(query string, datasource map[string]*DataSource) (SQLStatement, if !dataSourceExists(datasource, dataSourceName) { return SQLStatement{}, errors.New("data source does not exist") } - //if len(matches) == 4 && len(matches[3]) > 0 { + if len(matches) == 4 { wherePart = matches[3] } - headers := datasource[dataSourceName].Data[0] + headers := datasource.DataSources[dataSourceName].Data[0] columns := parseColumns(columnsPart, headers) conditions, err := parseConditions(wherePart) @@ -76,7 +69,7 @@ func parseQuery(query string, datasource map[string]*DataSource) (SQLStatement, } return SQLStatement{ - Type: queryType, + Type: commandType, Columns: columns, Conditions: conditions, DataSourceName: dataSourceName, @@ -206,10 +199,10 @@ func parseConditions(wherePart string) ([]Condition, error) { } // ExecuteSelectQuery executes a SELECT query and returns the results as a slice of RowMaps -func executeSelectQuery(data [][]string, query SQLStatement) []RowMap { - headers := data[0] // First row as header +func executeSqlSelectQuery(data *[][]string, query SQLStatement) []RowMap { + headers := (*data)[0] // First row as header results := []RowMap{} - for _, row := range data[1:] { + for _, row := range (*data)[1:] { rowMap := mapRow(headers, row) if matchesConditions(rowMap, query.Conditions) { results = append(results, projectRow(rowMap, query.Columns)) @@ -219,7 +212,7 @@ func executeSelectQuery(data [][]string, query SQLStatement) []RowMap { } // ExecuteUpdateQuery executes an UPDATE query and modifies the data in-place -func executeUpdateQuery(data *[][]string, query SQLStatement) error { +func executeSqlUpdateCommand(data *[][]string, query SQLStatement) error { if len(*data) < 2 { return errors.New("no data available to update") } @@ -242,7 +235,7 @@ func executeUpdateQuery(data *[][]string, query SQLStatement) error { } // ExecuteDeleteQuery executes a DELETE query and modifies the data in-place -func executeDeleteQuery(data *[][]string, query SQLStatement) error { +func executeSqlDeleteCommand(data *[][]string, query SQLStatement) error { if len(*data) < 2 { return errors.New("no data available to delete") } diff --git a/core/templating/datasource_query_test.go b/core/templating/datasource_query_test.go index 3f03f32ed..94cb45ad4 100644 --- a/core/templating/datasource_query_test.go +++ b/core/templating/datasource_query_test.go @@ -6,8 +6,9 @@ import ( "testing" ) -func TestParseQuery(t *testing.T) { - dataSource := map[string]*DataSource{ +func TestParseCommand(t *testing.T) { + + dataSources := map[string]*DataSource{ "employees": { SourceType: "csv", Name: "employees", @@ -19,6 +20,8 @@ func TestParseQuery(t *testing.T) { mu: sync.Mutex{}, }, } + templateDataSource := NewTemplateDataSource() + templateDataSource.DataSources = dataSources tests := []struct { query string @@ -57,7 +60,7 @@ func TestParseQuery(t *testing.T) { } for _, test := range tests { - result, err := parseQuery(test.query, dataSource) + result, err := parseSqlCommand(test.query, templateDataSource) if test.expectError { if err == nil { t.Errorf("expected error but got none for query: %s", test.query) @@ -175,7 +178,7 @@ func TestParseConditions_MultipleConditions(t *testing.T) { } } -func TestExecuteSelectQuery(t *testing.T) { +func TestExecuteSqlSelectQuery(t *testing.T) { data := [][]string{ {"id", "name", "age", "department"}, {"1", "John Doe", "30", "Engineering"}, @@ -193,13 +196,13 @@ func TestExecuteSelectQuery(t *testing.T) { {"name": "John Doe", "age": "30"}, } - result := executeSelectQuery(data, query) + result := executeSqlSelectQuery(&data, query) if !reflect.DeepEqual(result, expected) { t.Errorf("expected %v, got %v", expected, result) } } -func TestExecuteUpdateQuery(t *testing.T) { +func TestExecuteSqlUpdateCommand(t *testing.T) { data := [][]string{ {"id", "name", "age", "department"}, {"1", "John Doe", "30", "Engineering"}, @@ -219,7 +222,7 @@ func TestExecuteUpdateQuery(t *testing.T) { {"2", "Jane Smith", "40", "Marketing"}, } - err := executeUpdateQuery(&data, query) + err := executeSqlUpdateCommand(&data, query) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -228,7 +231,7 @@ func TestExecuteUpdateQuery(t *testing.T) { } } -func TestExecuteDeleteQuery(t *testing.T) { +func TestExecuteSqlDeleteCommand(t *testing.T) { data := [][]string{ {"id", "name", "age", "department"}, {"1", "John Doe", "30", "Engineering"}, @@ -246,7 +249,7 @@ func TestExecuteDeleteQuery(t *testing.T) { {"2", "Jane Smith", "40", "Marketing"}, } - err := executeDeleteQuery(&data, query) + err := executeSqlDeleteCommand(&data, query) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/core/templating/template_datasource.go b/core/templating/template_datasource.go index 8ba629155..aee97b841 100644 --- a/core/templating/template_datasource.go +++ b/core/templating/template_datasource.go @@ -26,6 +26,7 @@ func (templateDataSource *TemplateDataSource) SetDataSource(dataSourceName strin func (templateDataSource *TemplateDataSource) DeleteDataSource(dataSourceName string) { templateDataSource.RWMutex.Lock() + if _, ok := templateDataSource.DataSources[dataSourceName]; ok { delete(templateDataSource.DataSources, dataSourceName) } @@ -36,3 +37,15 @@ func (templateDataSource *TemplateDataSource) GetAllDataSources() map[string]*Da return templateDataSource.DataSources } + +func dataSourceExists(templateDataSource *TemplateDataSource, name string) bool { + templateDataSource.RWMutex.Lock() + defer templateDataSource.RWMutex.Unlock() + + for _, dataSource := range templateDataSource.DataSources { + if dataSource.Name == name { + return true + } + } + return false +} diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index c295d3d9b..42b146816 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -135,76 +135,6 @@ func (t templateHelpers) isBool(s string) bool { return err == nil } -// func isGreaterThan(valueToCheck, minimumValue string) bool { -// num1, err := strconv.ParseFloat(valueToCheck, 64) -// if err != nil { -// return false -// } -// num2, err := strconv.ParseFloat(minimumValue, 64) -// if err != nil { -// return false -// } -// return num1 > num2 -// } - -// func (t templateHelpers) isGreaterThan(valueToCheck, minimumValue string) bool { -// return isGreaterThan(valueToCheck, minimumValue) -// } - -// func isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { -// num1, err := strconv.ParseFloat(valueToCheck, 64) -// if err != nil { -// return false -// } -// num2, err := strconv.ParseFloat(minimumValue, 64) -// if err != nil { -// return false -// } -// return num1 >= num2 -// } - -// func (t templateHelpers) isGreaterThanOrEqual(valueToCheck, minimumValue string) bool { -// return isGreaterThan(valueToCheck, minimumValue) -// } - -// func isLessThan(valueToCheck, maximumValue string) bool { -// num1, err := strconv.ParseFloat(valueToCheck, 64) -// if err != nil { -// return false -// } -// num2, err := strconv.ParseFloat(maximumValue, 64) -// if err != nil { -// return false -// } -// return num1 < num2 -// } - -// func (t templateHelpers) isLessThan(valueToCheck, maximumValue string) bool { -// return isLessThan(valueToCheck, maximumValue) -// } - -// func isLessThanOrEqual(valueToCheck, maximumValue string) bool { -// num1, err := strconv.ParseFloat(valueToCheck, 64) -// if err != nil { -// return false -// } -// num2, err := strconv.ParseFloat(maximumValue, 64) -// if err != nil { -// return false -// } -// return num1 <= num2 -// } - -// func (t templateHelpers) isLessThanOrEqual(valueToCheck, maximumValue string) bool { -// return isLessThan(valueToCheck, maximumValue) -// } - -// func (t templateHelpers) isBetween(valueToCheck, minimumValue, maximumValue string) bool { -// return t.isGreaterThan(valueToCheck, minimumValue) && t.isLessThan(valueToCheck, maximumValue) -// } - -//######################## - // Comparison function type that takes two float64 values and returns a boolean. type comparator func(float64, float64) bool @@ -258,7 +188,6 @@ func (t templateHelpers) isBetween(value, min, max string) bool { return t.isGreaterThan(value, min) && t.isLessThan(value, max) } -// ######################## func (t templateHelpers) matchesRegex(valueToCheck, pattern string) bool { re, err := regexp.Compile(pattern) if err != nil { @@ -312,7 +241,7 @@ func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, se templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return getEvaluationString("csv", options) } source.mu.Lock() @@ -347,11 +276,11 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return []RowMap{} } if len(source.Data) < 1 { - log.Debug("no data available in datasource " + dataSourceName) + log.Error("no data available in datasource " + dataSourceName) return []RowMap{} } source.mu.Lock() @@ -366,7 +295,7 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField } } if fieldIndex == -1 { - log.Debug("could not find search field name " + searchFieldName) + log.Error("could not find search field name " + searchFieldName) return []RowMap{} } @@ -393,7 +322,7 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string { defer source.mu.Unlock() return source.Data } else { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return [][]string{} } } @@ -403,13 +332,13 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []RowMap { templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return []RowMap{} } source.mu.Lock() defer source.mu.Unlock() if len(source.Data) < 1 { - log.Debug("no data available in datasource " + dataSourceName) + log.Error("no data available in datasource " + dataSourceName) return []RowMap{} } headers := source.Data[0] @@ -434,7 +363,7 @@ func (t templateHelpers) csvAddRow(dataSourceName string, newRow []string) strin defer source.mu.Unlock() source.Data = append(source.Data, newRow) } else { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) } return "" } @@ -443,13 +372,13 @@ func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFi templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return "" } source.mu.Lock() defer source.mu.Unlock() if len(source.Data) == 0 { - log.Debug("datasource " + dataSourceName + " is empty") + log.Error("datasource " + dataSourceName + " is empty") return "" } header := source.Data[0] @@ -461,7 +390,7 @@ func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFi } } if fieldIndex == -1 { - log.Debug("could not find field name " + searchFieldName + " in datasource " + dataSourceName) + log.Error("could not find field name " + searchFieldName + " in datasource " + dataSourceName) return "" } filteredData := [][]string{header} @@ -484,7 +413,7 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { templateDataSources := t.TemplateDataSource.DataSources source, exists := templateDataSources[dataSourceName] if !exists { - log.Debug("could not find datasource " + dataSourceName) + log.Error("could not find datasource " + dataSourceName) return "" } source.mu.Lock() @@ -496,21 +425,19 @@ func (t templateHelpers) csvCountRows(dataSourceName string) string { return fmt.Sprintf("%d", numRows) } -func (t templateHelpers) csvSQL(queryString string) []RowMap { - templateDataSources := t.TemplateDataSource.DataSources - // Parse the query string to get the SelectQuery - query, err := parseQuery(strings.TrimSpace(queryString), templateDataSources) +func (t templateHelpers) csvSqlCommand(commandString string) []RowMap { + + // Parse the command string to get the Sql command + command, err := parseSqlCommand(strings.TrimSpace(commandString), t.TemplateDataSource) if err != nil { - // Debugging: Print the parsed query - fmt.Printf("Error parsing query:: %+v\n", err) - log.Debug("Error parsing query:", err) + log.Error("Error parsing sql command:", err) return []RowMap{} } // Find the data source by name - source, exists := templateDataSources[query.DataSourceName] + source, exists := t.TemplateDataSource.DataSources[command.DataSourceName] if !exists { - log.Debug("Could not find datasource " + query.DataSourceName) + log.Error("Could not find datasource " + command.DataSourceName) return []RowMap{} } @@ -520,25 +447,25 @@ func (t templateHelpers) csvSQL(queryString string) []RowMap { var results []RowMap var execErr error - switch query.Type { + switch command.Type { case "SELECT": - results = executeSelectQuery(source.Data, query) //This is not by reference as we are not changing the data + results = executeSqlSelectQuery(&source.Data, command) case "UPDATE": - execErr = executeUpdateQuery(&source.Data, query) //This must be by reference as we are not changing the data + execErr = executeSqlUpdateCommand(&source.Data, command) case "DELETE": - execErr = executeDeleteQuery(&source.Data, query) //This must be by reference as we are not changing the data + execErr = executeSqlDeleteCommand(&source.Data, command) default: - log.Debug(fmt.Errorf("unsupported query type %s", query.Type)) + log.Error(fmt.Errorf("unsupported query type %s", command.Type)) return nil } if execErr != nil { - log.Debug(fmt.Errorf("error executing query: %w", execErr)) + log.Error(fmt.Errorf("error executing query: %w", execErr)) return nil } // For SELECT queries, return results - if query.Type == "SELECT" { + if command.Type == "SELECT" { return results } diff --git a/core/templating/templating.go b/core/templating/templating.go index 65c00cf24..94daf4528 100644 --- a/core/templating/templating.go +++ b/core/templating/templating.go @@ -109,7 +109,7 @@ func NewTemplator() *Templator { helperMethodMap["csvAddRow"] = t.csvAddRow helperMethodMap["csvDeleteRows"] = t.csvDeleteRows helperMethodMap["csvCountRows"] = t.csvCountRows - helperMethodMap["csvSQL"] = t.csvSQL + helperMethodMap["csvSqlCommand"] = t.csvSqlCommand helperMethodMap["journal"] = t.parseJournalBasedOnIndex helperMethodMap["hasJournalKey"] = t.hasJournalKey helperMethodMap["setStatusCode"] = t.setStatusCode diff --git a/core/templating/templating_test backup.txt b/core/templating/templating_test backup.txt deleted file mode 100644 index 414277391..000000000 --- a/core/templating/templating_test backup.txt +++ /dev/null @@ -1,934 +0,0 @@ -package templating_test - -import ( - "testing" - - "github.com/SpectoLabs/hoverfly/core/journal" - - "github.com/SpectoLabs/hoverfly/core/models" - "github.com/SpectoLabs/hoverfly/core/templating" - . "github.com/onsi/gomega" -) - -// We need to run the csv templating tests first because the datasource is part of the templatingHelper which can be registered only once -// for each runtime, raymond unfortunately doesn't provide a way to unregister helpers for us to isolate the test data. -func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '2' 'Marks'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`56`)) -} - -func Test_ApplyTemplate_ParseCsvAndReturnFallbackStringIfNoMatchFound(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '51' 'Marks'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`ABSENT`)) -} - -func Test_ApplyTemplate_ParseCsvAndReturnQueryStringIfNoMatchFound(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv2' 'Id' '51' 'Marks'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`{{ csv test-csv2 Id 51 Marks }}`)) -} - -func Test_ApplyTemplate_ParseCsvByPassingRequestParamAndReturnMatchValue(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Query: map[string][]string{"Id": {"1"}}, - }, make(map[string]string), `{{csv 'test-csv2' 'Id' 'Request.QueryParam.Id.[0]' 'Marks'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`55`)) -} - -func Test_ApplyTemplate_ParseCsv_WithMissingDataSource(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Query: map[string][]string{"Id": {"1"}}, - }, make(map[string]string), `{{csv 'test-csv3' 'Id' 55 'Marks'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`{{ csv test-csv3 Id 55 Marks }}`)) -} - -func Test_ApplyTemplate_ParseCsv_WithEachBlockAndMissingDataSource(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{"products": [1, 2]}`, - }, make(map[string]string), `{{#each (Request.Body 'jsonpath' '$.products')}} {{@index}} : Product Name with productId {{this}} is {{csv 'products' 'productId' this 'productName'}} {{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(` 0 : Product Name with productId 1 is {{ csv products productId 1 productName }} 1 : Product Name with productId 2 is {{ csv products productId 2 productName }} `)) -} - -// -------------------------------------- -func Test_ApplyTemplate_MatchingRowsCsvAndReturnMatchedString(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '2')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Test2`)) -} - -func Test_ApplyTemplate_MatchingRowsCsvMissingDataSource(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv99' 'id' '2')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '99')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -// ------------------------------- -func Test_ApplyTemplate_CsvAsArrayAndReturnMatchedString(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv')}}{{#each this}}{{this}}{{/each}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`idnamemarks1Test1552Test256*DummyABSENT`)) -} - -func Test_ApplyTemplate_CsvAsArrayMissingDataSource(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv99')}}{{#each this}}{{this}}{{/each}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -// ------------------------------- -func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Test1Test2Dummy`)) -} - -func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv99')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -// ------------------------------- - -func Test_ApplyTemplate_CsvAddRow(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}{{csv 'test-csv' 'id' '99' 'name'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Violet`)) -} - -func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '2' true}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`1`)) -} - -func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv99' 'id' '2' true}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'identity' '2' true}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -func Test_ApplyTemplate_CsvCountRows(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`3`)) -} - -func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv99'}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) -} - -func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Test1`)) -} - -// func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { -// RegisterTestingT(t) - -// template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL 'SELECT * FROM test-csv')}}{{this.name}}{{/each}}`) - -// Expect(err).To(BeNil()) -// Expect(template).To(Equal(`Test1Test2Dummy`)) -// } - -func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString2(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Test1Test2Dummy`)) -} - -func Test_ApplyTemplate_CsvSQL_SelectAll(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`1,Test1,55;2,Test2,56;*,Dummy,ABSENT;`)) -} - -func Test_ApplyTemplate_CsvSQL_Update(t *testing.T) { - RegisterTestingT(t) - - // First, update the data - _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) - Expect(err).To(BeNil()) - - // Then, check the result - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`60`)) -} - -func Test_ApplyTemplate_CsvSQL_Delete(t *testing.T) { - RegisterTestingT(t) - - // First, delete the row - _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "DELETE FROM test-csv2 WHERE id == '5553686208582'"}}`) - Expect(err).To(BeNil()) - - // Then, check the result - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv2")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`1,Test1,55;2,Test2,56;`)) // Test3 entry should be deleted -} - -func Test_ApplyTemplate_CsvSQL_InvalidQuery(t *testing.T) { - RegisterTestingT(t) - - // An invalid query that should fail - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "SELECT foo FROM bar WHERE id == '1'"}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(``)) // No valid data should be returned -} - -func Test_ApplyTemplate_EachBlockWithSplitTemplatingFunction(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), - `{{#each (split 'one,two,three' ',') }}{{this}} | {{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`one | two | three | `)) -} - -func Test_ApplyTemplate_EachBlockWithCsvTemplatingFunction(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Query: map[string][]string{"Ids": {"1", "2"}}, - }, make(map[string]string), `{{#each (Request.QueryParam.Ids) }}{{csv 'test-csv2' 'Id' this 'Marks'}} | {{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`55 | 56 | `)) -} - -func Test_ApplyTemplate_EachBlockWithCsvTemplatingFunctionAndLargeInteger(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{"ids": [1, 5553686208582]}`, - }, make(map[string]string), `{{#each (Request.Body 'jsonpath' '$.ids') }}{{csv 'test-csv2' 'Id' this 'Marks'}} | {{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`55 | 66 | `)) -} - -func Test_ShouldCreateTemplatingDataPathsFromRequest(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - Path: "/foo/bar", - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.Path).To(ConsistOf("foo", "bar")) -} - -func Test_ShouldCreateTemplatingDataPathsFromRequestWithNoPaths(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.Path).To(BeEmpty()) -} - -func Test_ShouldCreateTemplatingDataQueryParamsFromRequest(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - Query: map[string][]string{ - "cheese": {"1", "3"}, - "ham": {"2"}, - }, - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.QueryParam).To(HaveKeyWithValue("cheese", []string{"1", "3"})) - Expect(actual.Request.QueryParam).To(HaveKeyWithValue("ham", []string{"2"})) - Expect(actual.Request.QueryParam).To(HaveLen(2)) -} - -func Test_ShouldCreateTemplatingDataQueryParamsFromRequestWithNoQueryParams(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.QueryParam).To(BeEmpty()) -} - -func Test_ShouldCreateTemplatingDataHttpScheme(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.Scheme).To(Equal("http")) -} - -func Test_ShouldCreateTemplatingDataHeaderFromRequest(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - Headers: map[string][]string{ - "cheese": {"1", "3"}, - "ham": {"2"}, - }, - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.Header).To(HaveKeyWithValue("cheese", []string{"1", "3"})) - Expect(actual.Request.Header).To(HaveKeyWithValue("ham", []string{"2"})) - Expect(actual.Request.Header).To(HaveLen(2)) -} - -func Test_ShouldCreateTemplatingDataHeaderFromRequestWithNoHeader(t *testing.T) { - RegisterTestingT(t) - - actual := templating.NewTemplator().NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - }, nil, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(actual.Request.Header).To(BeEmpty()) -} - -func TestApplyTemplateWithRequestDetails(t *testing.T) { - RegisterTestingT(t) - - requestDetails := &models.RequestDetails{ - Method: "GET", - Scheme: "http", - Destination: "foo.com", - Path: "/foo/bar", - Query: map[string][]string{ - "singular": {"1"}, - "multiple": {"2", "3"}, - }, - Headers: map[string][]string{ - "X-Singular": {"a"}, - "X-Multiple": {"b", "c"}, - }, - } - - template, err := ApplyTemplate(requestDetails, - make(map[string]string), - ` -Scheme: {{ Request.Scheme }} - -Query param value: {{ Request.QueryParam.singular }} - -Query param value by index: {{ Request.QueryParam.multiple.[0] }} -Query param value by index: {{ Request.QueryParam.multiple.[1] }} - -List of query param values: {{ Request.QueryParam.multiple}} -Looping through query params: {{#each Request.QueryParam.multiple}}{{ this }}-{{/each}} - -Header value: {{ Request.Header.X-Singular }} -Header value by index: {{ Request.Header.X-Multiple.[0] }} -Header value by index: {{ Request.Header.X-Multiple.[1] }} -List of header values: {{ Request.Header.X-Multiple}} - -Path param value: {{ Request.Path.[0] }} -All path param values: {{ Request.Path }} -Looping through path params: {{#each Request.Path}}{{ this }}-{{/each}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal(` -Scheme: http - -Query param value: 1 - -Query param value by index: 2 -Query param value by index: 3 - -List of query param values: 23 -Looping through query params: 2-3- - -Header value: a -Header value by index: b -Header value by index: c -List of header values: bc - -Path param value: foo -All path param values: foobar -Looping through path params: foo-bar-`)) -} - -func TestTemplatingWithParametersWhichDoNotExistDoNotErrorAndAreEmpty(t *testing.T) { - RegisterTestingT(t) - - requestDetails := &models.RequestDetails{ - Method: "GET", - Scheme: "http", - Destination: "foo.com", - } - - template, err := ApplyTemplate(requestDetails, - map[string]string{ - "one": "A", - "two": "B", - }, - ` -Scheme:{{ Request.Scheme }} - -Query param value:{{ Request.QueryParam.singular }} - -Query param value by index:{{ Request.QueryParam.multiple.[0] }} -Query param value by index:{{ Request.QueryParam.multiple.[1] }} - -List of query param values:{{ Request.QueryParam.multiple}} -Looping through query params:{{#each Request.QueryParam.multiple}}{{ this }}{{/each}} - -Header value: {{ Request.Header.X-Singular }} -Header value by index: {{ Request.Header.X-Multiple.[0] }} -Header value by index: {{ Request.Header.X-Multiple.[1] }} -List of header values: {{ Request.Header.X-Multiple}} - -Path param value:{{ Request.Path.[0] }} -All path param values:{{ Request.Path }} -Looping through path params:{{#each Request.Path}}{{ this }}{{/each}} - -State One: {{ State.one }} -State Two: {{ State.two }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal(` -Scheme:http - -Query param value: - -Query param value by index: -Query param value by index: - -List of query param values: -Looping through query params: - -Header value: -Header value by index: -Header value by index: -List of header values: - -Path param value: -All path param values: -Looping through path params: - -State One: A -State Two: B`)) -} - -func Test_ApplyTemplate_now(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{now "" "unix"}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{now "" "unix"}}`)))) -} - -func Test_ApplyTemplate_randomString(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomString}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomString}}`)))) -} - -func Test_ApplyTemplate_randomStringLength(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomStringLength 2}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomStringLength 2}}`)))) - Expect(template).To(HaveLen(2)) -} - -func Test_ApplyTemplate_randomBoolean(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomBoolean}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomBoolean}}`)))) -} - -func Test_ApplyTemplate_randomInteger(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomInteger}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomInteger}}`)))) -} - -func Test_ApplyTemplate_randomIntegerRange(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIntegerRange 7 8}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomIntegerRange 7 8}}`)))) -} - -func Test_ApplyTemplate_randomFloat(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomFloat}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomFloat}}`)))) -} - -func Test_ApplyTemplate_randomFloatRange(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomFloatRange 7.0 8.0}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomFloatRange 7.0 8.0}}`)))) -} - -func Test_ApplyTemplate_randomEmail(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomEmai}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Not(Equal(ContainSubstring(`{{randomEmail}}`)))) -} - -func Test_ApplyTemplate_randomIPv4(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIPv4}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomIPv4}}`)))) -} - -func Test_ApplyTemplate_randomIPv6(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomIPv6}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomIPv6}}`)))) -} - -func Test_ApplyTemplate_randomUuid(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{randomUuid}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Not(Equal(ContainSubstring(`{{randomUuid}}`)))) -} - -func Test_ApplyTemplate_Request_Body_Jsonpath(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{ "name": "Ben" }`, - }, make(map[string]string), `{{ Request.Body 'jsonpath' '$.name' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("Ben")) -} - -func Test_ApplyTemplate_Request_Body_JsonPath_Unescaped(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{ "name": "O'Reilly" }`, - }, make(map[string]string), `{{{ Request.Body 'jsonpath' '$.name' }}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("O'Reilly")) -} - -func Test_ApplyTemplate_Request_Body_Jsonpath_LargeInt(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{ "id": 5553686208582 }`, - }, make(map[string]string), `{{ Request.Body 'jsonpath' '$.id' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("5553686208582")) -} - -func Test_ApplyTemplate_Request_Body_Jsonpath_From_Xml(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `Johnny`, - }, make(map[string]string), `{{ Request.Body 'jsonpathfromxml' '$.name' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("Johnny")) -} - -func Test_ApplyTemplate_ReplaceStringInQueryParams(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Query: map[string][]string{ - "sound": {"oink,oink,oink"}, - }, - }, make(map[string]string), `{{ replace Request.QueryParam.sound 'oink' 'moo' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal(`moo,moo,moo`)) -} - -func Test_VarSetToNilInCaseOfInvalidArgsPassed(t *testing.T) { - RegisterTestingT(t) - templator := templating.NewTemplator() - - vars := &models.Variables{ - models.Variable{ - Name: "varOne", - Function: "faker", - }, - } - - actual := templator.NewTemplatingData(&models.RequestDetails{ - Scheme: "http", - Destination: "test.com", - }, nil, &models.Literals{}, vars, make(map[string]string), &journal.Journal{}) - - Expect(actual.Vars["varOne"]).To(BeNil()) - -} - -func Test_VarSetToProperValueInCaseOfRequestDetailsPassedAsArgument(t *testing.T) { - RegisterTestingT(t) - templator := templating.NewTemplator() - argumentsArray := toInterfaceSlice([]string{"Request.Path.[1]", ","}) - vars := &models.Variables{ - models.Variable{ - Name: "splitRequestPath", - Function: "split", - Arguments: argumentsArray, - }, - } - - actual := templator.NewTemplatingData(&models.RequestDetails{ - Path: "/part1/foo,bar", - }, nil, &models.Literals{}, vars, make(map[string]string), &journal.Journal{}) - - Expect(actual.Vars["splitRequestPath"]).ToNot(BeNil()) - Expect(len(actual.Vars["splitRequestPath"].([]string))).To(Equal(2)) - Expect(actual.Vars["splitRequestPath"].([]string)[0]).To(Equal("foo")) - Expect(actual.Vars["splitRequestPath"].([]string)[1]).To(Equal("bar")) - -} - -func Test_ApplyTemplate_add_integers(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 1 2 '0'}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("3")) -} - -func Test_ApplyTemplate_add_floats(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 '0.00'}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("1.44")) -} - -func Test_ApplyTemplate_add_floats_withRoundUp(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 '0.0'}} and {{ add 0.1 1.56 '0.0'}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("1.4 and 1.7")) -} - -func Test_ApplyTemplate_add_number_without_format(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 0.1 1.34 ''}} and {{ add 1 2 ''}} and {{ add 0 0 ''}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("1.44 and 3 and 0")) -} - -func Test_ApplyTemplate_add_NotNumber(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ add 'a' 'b' '0.00'}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("NaN")) -} - -func Test_ApplyTemplate_subtract_numbers(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ subtract 10 0.99 ''}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("9.01")) -} - -func Test_ApplyTemplate_mutiply_numbers(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ multiply 10 0.99 ''}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("9.9")) -} - -func Test_ApplyTemplate_divide_numbers(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ divide 10 2.5 ''}}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("4")) -} - -func Test_ApplyTemplate_Arithmetic_Ops_With_Each_Block(t *testing.T) { - RegisterTestingT(t) - - templator := templating.NewTemplator() - - requestDetails := &models.RequestDetails{ - Body: `{"lineitems":{"lineitem":[{"upc":"1001","quantity":"1","price":"3.50"},{"upc":"1002","quantity":"2","price":"4.50"}]}}`, - } - responseBody := `{{#each (Request.Body 'jsonpath' '$.lineitems.lineitem') }} {{ addToArray 'subtotal' (multiply (this.price) (this.quantity) '') true }} {{/each}} total: {{ sum (getArray 'subtotal') '0.00' }}` - - template, _ := templator.ParseTemplate(responseBody) - state := make(map[string]string) - result, err := templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) - - Expect(err).To(BeNil()) - Expect(result).To(Equal(` 3.5 9 total: 12.50`)) - - // Running the second time should produce the same result because each execution has its own context data. - result, err = templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) - Expect(err).To(BeNil()) - Expect(result).To(Equal(` 3.5 9 total: 12.50`)) -} - -func Test_ApplyTemplate_PutAndGetValue(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{ "id": 5553686208582 }`, - }, make(map[string]string), `{{ putValue 'id' (Request.Body 'jsonpath' '$.id') true }} The ID was {{ getValue 'id' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("5553686208582 The ID was 5553686208582")) -} - -func Test_ApplyTemplate_PutAndGetValue_SuppressOutput(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{ - Body: `{ "id": 5553686208582 }`, - }, make(map[string]string), `{{ putValue 'id' (Request.Body 'jsonpath' '$.id') false }}The ID was {{ getValue 'id' }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("The ID was 5553686208582")) -} - -func Test_ApplyTemplate_setStatusCode(t *testing.T) { - RegisterTestingT(t) - - templator := templating.NewTemplator() - - template, err := templator.ParseTemplate(`{{ setStatusCode 400 }}`) - Expect(err).To(BeNil()) - - response := &models.ResponseDetails{} - result, err := templator.RenderTemplate(template, &models.RequestDetails{}, response, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(err).To(BeNil()) - Expect(result).To(Equal("")) - Expect(response.Status).To(Equal(400)) -} - -func Test_ApplyTemplate_setStatusCode_should_ignore_invalid_code(t *testing.T) { - RegisterTestingT(t) - - templator := templating.NewTemplator() - - template, err := templator.ParseTemplate(`{{ setStatusCode 600 }}`) - Expect(err).To(BeNil()) - - response := &models.ResponseDetails{} - result, err := templator.RenderTemplate(template, &models.RequestDetails{}, response, &models.Literals{}, &models.Variables{}, make(map[string]string), &journal.Journal{}) - - Expect(err).To(BeNil()) - Expect(result).To(Equal("")) - Expect(response.Status).To(Equal(0)) -} - -func Test_ApplyTemplate_setStatusCode_should_handle_nil_response(t *testing.T) { - RegisterTestingT(t) - - // ApplyTemplate pass a nil response value to RenderTemplate - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{ setStatusCode 400 }}`) - - Expect(err).To(BeNil()) - - Expect(template).To(Equal("")) -} - -func toInterfaceSlice(arguments []string) []interface{} { - argumentsArray := make([]interface{}, len(arguments)) - - for i, s := range arguments { - argumentsArray[i] = s - } - return argumentsArray -} - -func ApplyTemplate(requestDetails *models.RequestDetails, state map[string]string, responseBody string) (string, error) { - - templator := templating.NewTemplator() - dataSource1, _ := templating.NewCsvDataSource("test-csv", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") - dataSource2, _ := templating.NewCsvDataSource("test-csv2", "id,name,marks\n1,Test1,55\n2,Test2,56\n5553686208582,Test3,66\n") - templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv", dataSource1) - templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv2", dataSource2) - - template, err := templator.ParseTemplate(responseBody) - Expect(err).To(BeNil()) - return templator.RenderTemplate(template, requestDetails, nil, &models.Literals{}, &models.Variables{}, state, &journal.Journal{}) -} diff --git a/core/templating/templating_test.go b/core/templating/templating_test.go index 0c112cd3c..fc36013ed 100644 --- a/core/templating/templating_test.go +++ b/core/templating/templating_test.go @@ -201,7 +201,7 @@ func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test1`)) @@ -228,7 +228,7 @@ func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString2(t *testing.T) { func Test_ApplyTemplate_CsvSQL_SelectAll(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv WHERE 1==1")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT * FROM test-csv WHERE 1==1")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`1,Test1,55;2,Test2,56;*,Dummy,ABSENT;`)) @@ -238,11 +238,11 @@ func Test_ApplyTemplate_CsvSQL_Update(t *testing.T) { RegisterTestingT(t) // First, update the data - _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSqlCommand "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) Expect(err).To(BeNil()) // Then, check the result - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`60`)) @@ -252,11 +252,11 @@ func Test_ApplyTemplate_CsvSQL_Delete(t *testing.T) { RegisterTestingT(t) // First, delete the row - _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSQL "DELETE FROM test-csv2 WHERE id == '5553686208582'"}}`) + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSqlCommand "DELETE FROM test-csv2 WHERE id == '5553686208582'"}}`) Expect(err).To(BeNil()) // Then, check the result - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL "SELECT * FROM test-csv2")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT * FROM test-csv2")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`1,Test1,55;2,Test2,56;`)) // Test3 entry should be deleted diff --git a/docs/pages/keyconcepts/templating/templating.rst b/docs/pages/keyconcepts/templating/templating.rst index fff283e79..c7ddd8762 100644 --- a/docs/pages/keyconcepts/templating/templating.rst +++ b/docs/pages/keyconcepts/templating/templating.rst @@ -262,7 +262,7 @@ To return filtered data from the csv as an array of maps using the SQL like dial .. code:: handlebars - {{csvSQL '(sql-select-query)'}} + {{csvSqlCommand '(sql-select-query)'}} Example: Start Hoverfly with a CSV data source (pets.csv) provided below. @@ -325,8 +325,8 @@ Example: Start Hoverfly with a CSV data source (pets.csv) provided below. +--------------------------+------------------------------------------+-----------------------------------------+ | SELECT data using a SQL | { | { | | like syntax. | "Dogs-With-Big-Ids-Only": [ | "Dogs-With-Big-Ids-Only": [ | -| | {{#each (csvSQL "SELECT * FROM pets | { | -| | WHERE category == 'dogs' AND id >= | "id":2000, | +| | {{#each (csvSqlCommand "SELECT * FROM | { | +| | pets WHERE category == 'dogs' AND id >= | "id":2000, | | | '2000'")}} | "category":"dogs", | | | { | "name":"Violet", | | | "id":{{this.id}}, | "status":"sold" | @@ -413,20 +413,20 @@ Deleting data from a CSV Data Source using SQL like syntax .. code:: handlebars - {{csvSQL '(sql-delete-statement)'}} + {{csvSqlCommand '(sql-delete-statement)'}} Example: -``{{ csvSQL "DELETE FROM pets WHERE id > '20'" }}`` +``{{ csvSqlCommand "DELETE FROM pets WHERE id > '20'" }}`` Update the data in a CSV Data Source using SQL like syntax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: handlebars - {{csvSQL '(sql-update-statement)'}} + {{csvSqlCommand '(sql-update-statement)'}} Example: -``{{ csvSQL "UPDATE pets SET status = 'sold' WHERE id > '20' AND category == 'cats'" }}`` +``{{ csvSqlCommand "UPDATE pets SET status = 'sold' WHERE id > '20' AND category == 'cats'" }}`` Using SQL like syntax to query and manipulate data sources ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 71d0bd9eadc3148f17a8b9c0d229192aae8aa778 Mon Sep 17 00:00:00 2001 From: stuioco Date: Mon, 26 Aug 2024 18:34:55 +0100 Subject: [PATCH 6/9] Changes and fixes based on review --- ...ce_query.go => datasource_sql_over_csv.go} | 85 +++++++++++++++---- ...est.go => datasource_sql_over_csv_test.go} | 65 +++++++++++--- core/templating/template_helpers.go | 18 +--- core/templating/templating_test.go | 54 ++++-------- .../keyconcepts/templating/templating.rst | 17 ++++ 5 files changed, 161 insertions(+), 78 deletions(-) rename core/templating/{datasource_query.go => datasource_sql_over_csv.go} (82%) rename core/templating/{datasource_query_test.go => datasource_sql_over_csv_test.go} (80%) diff --git a/core/templating/datasource_query.go b/core/templating/datasource_sql_over_csv.go similarity index 82% rename from core/templating/datasource_query.go rename to core/templating/datasource_sql_over_csv.go index baf4d6ac6..4d17aefc6 100644 --- a/core/templating/datasource_query.go +++ b/core/templating/datasource_sql_over_csv.go @@ -3,7 +3,10 @@ package templating import ( "errors" "regexp" + "strconv" "strings" + + log "github.com/sirupsen/logrus" ) // RowMap represents a single row in the result set @@ -61,7 +64,10 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement wherePart = matches[3] } headers := datasource.DataSources[dataSourceName].Data[0] - columns := parseColumns(columnsPart, headers) + columns, err := parseColumns(columnsPart, headers) + if err != nil { + return SQLStatement{}, err + } conditions, err := parseConditions(wherePart) if err != nil { @@ -128,16 +134,28 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement } // parseColumns determines the columns to select based on the query part and headers -func parseColumns(columnsPart string, headers []string) []string { +func parseColumns(columnsPart string, headers []string) ([]string, error) { columnsPart = strings.TrimSpace(columnsPart) if columnsPart == "*" { - return headers + return headers, nil } columns := strings.Split(columnsPart, ",") for i, column := range columns { + if !stringExists(headers, strings.TrimSpace(column)) { + return nil, errors.New("invalid column provided: " + strings.TrimSpace(column)) + } columns[i] = strings.TrimSpace(column) } - return columns + return columns, nil +} + +func stringExists(slice []string, target string) bool { + for _, s := range slice { + if s == target { + return true + } + } + return false } // TrimQuotes trims matching single or double quotes from the outer edges of a string. @@ -212,17 +230,24 @@ func executeSqlSelectQuery(data *[][]string, query SQLStatement) []RowMap { } // ExecuteUpdateQuery executes an UPDATE query and modifies the data in-place -func executeSqlUpdateCommand(data *[][]string, query SQLStatement) error { +func executeSqlUpdateCommand(data *[][]string, query SQLStatement) []RowMap { if len(*data) < 2 { - return errors.New("no data available to update") + log.Error("no data available to update") + return []RowMap{ + { + "rowsAffected": "0", + }, + } } headers := (*data)[0] conditions := query.Conditions setClauses := query.SetClauses + rowsAffected := 0 for i, row := range (*data)[1:] { rowMap := mapRow(headers, row) if matchesConditions(rowMap, conditions) { + rowsAffected += 1 for column, newValue := range setClauses { colIndex := indexOf(headers, column) if colIndex != -1 { @@ -231,26 +256,52 @@ func executeSqlUpdateCommand(data *[][]string, query SQLStatement) error { } } } - return nil + return []RowMap{ + { + "rowsAffected": strconv.Itoa(rowsAffected), + }, + } } -// ExecuteDeleteQuery executes a DELETE query and modifies the data in-place -func executeSqlDeleteCommand(data *[][]string, query SQLStatement) error { +// executeSqlDeleteCommand executes a DELETE query and modifies the data in-place +func executeSqlDeleteCommand(data *[][]string, query SQLStatement) []RowMap { if len(*data) < 2 { - return errors.New("no data available to delete") + log.Println("no data available to delete") + return []RowMap{ + { + "rowsAffected": "0", + }, + } } - headers := (*data)[0] conditions := query.Conditions - filteredData := [][]string{headers} - for _, row := range (*data)[1:] { + rowsAffected := 0 + // Iterate in reverse to avoid index shifting issues + for i := len(*data) - 1; i > 0; i-- { + row := (*data)[i] rowMap := mapRow(headers, row) - if !matchesConditions(rowMap, conditions) { - filteredData = append(filteredData, row) + if matchesConditions(rowMap, conditions) { + removeRow(data, i) + rowsAffected++ } } - *data = filteredData - return nil + return []RowMap{ + { + "rowsAffected": strconv.Itoa(rowsAffected), + }, + } +} + +// removeRow removes the row at index rowIndex from the data. +func removeRow(data *[][]string, rowIndex int) { + if rowIndex < 0 || rowIndex >= len(*data) { + // Return early if rowIndex is out of bounds + return + } + // Overwrite the row at rowIndex with the next rows + copy((*data)[rowIndex:], (*data)[rowIndex+1:]) + // Truncate the slice to remove the last row which is now duplicate + (*data) = (*data)[:len(*data)-1] } // Helper function to find the index of a column header diff --git a/core/templating/datasource_query_test.go b/core/templating/datasource_sql_over_csv_test.go similarity index 80% rename from core/templating/datasource_query_test.go rename to core/templating/datasource_sql_over_csv_test.go index 94cb45ad4..1aac981e0 100644 --- a/core/templating/datasource_query_test.go +++ b/core/templating/datasource_sql_over_csv_test.go @@ -53,6 +53,10 @@ func TestParseCommand(t *testing.T) { expected: SQLStatement{Type: "DELETE", Conditions: []Condition{{Column: "age", Operator: "<", Value: "30"}}, DataSourceName: "employees"}, expectError: false, }, + { + query: "SELECT * FROM chillieplants WHERE department == 'Engineering'", + expectError: true, + }, { query: "INVALID QUERY", expectError: true, @@ -202,7 +206,7 @@ func TestExecuteSqlSelectQuery(t *testing.T) { } } -func TestExecuteSqlUpdateCommand(t *testing.T) { +func TestExecuteSqlUpdateCommand_DataResult(t *testing.T) { data := [][]string{ {"id", "name", "age", "department"}, {"1", "John Doe", "30", "Engineering"}, @@ -222,16 +226,37 @@ func TestExecuteSqlUpdateCommand(t *testing.T) { {"2", "Jane Smith", "40", "Marketing"}, } - err := executeSqlUpdateCommand(&data, query) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + _ = executeSqlUpdateCommand(&data, query) + if !reflect.DeepEqual(data, expected) { t.Errorf("expected %v, got %v", expected, data) } } -func TestExecuteSqlDeleteCommand(t *testing.T) { +func TestExecuteSqlUpdateCommand_RowCountResult(t *testing.T) { + data := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + query := SQLStatement{ + Type: "UPDATE", + Conditions: []Condition{{Column: "name", Operator: "==", Value: "John Doe"}}, + SetClauses: map[string]string{"age": "35"}, + DataSourceName: "employees", + } + + expected := "1" + + result := executeSqlUpdateCommand(&data, query) + + if result[0]["rowsAffected"] != expected { + t.Errorf("expected %v, got %v", expected, result[0]["rowsAffected"]) + } +} + +func TestExecuteSqlDeleteCommand_DataResult(t *testing.T) { data := [][]string{ {"id", "name", "age", "department"}, {"1", "John Doe", "30", "Engineering"}, @@ -249,11 +274,31 @@ func TestExecuteSqlDeleteCommand(t *testing.T) { {"2", "Jane Smith", "40", "Marketing"}, } - err := executeSqlDeleteCommand(&data, query) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + _ = executeSqlDeleteCommand(&data, query) + if !reflect.DeepEqual(data, expected) { t.Errorf("expected %v, got %v", expected, data) } } + +func TestExecuteSqlDeleteCommand_RowCountResult(t *testing.T) { + data := [][]string{ + {"id", "name", "age", "department"}, + {"1", "John Doe", "30", "Engineering"}, + {"2", "Jane Smith", "40", "Marketing"}, + } + + query := SQLStatement{ + Type: "DELETE", + Conditions: []Condition{{Column: "age", Operator: "==", Value: "30"}}, + DataSourceName: "employees", + } + + expected := "1" + + result := executeSqlDeleteCommand(&data, query) + + if result[0]["rowsAffected"] != expected { + t.Errorf("expected %v, got %v", expected, result[0]["rowsAffected"]) + } +} diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index 42b146816..8e51ffdd7 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -445,32 +445,20 @@ func (t templateHelpers) csvSqlCommand(commandString string) []RowMap { defer source.mu.Unlock() var results []RowMap - var execErr error switch command.Type { case "SELECT": results = executeSqlSelectQuery(&source.Data, command) case "UPDATE": - execErr = executeSqlUpdateCommand(&source.Data, command) + results = executeSqlUpdateCommand(&source.Data, command) case "DELETE": - execErr = executeSqlDeleteCommand(&source.Data, command) + results = executeSqlDeleteCommand(&source.Data, command) default: log.Error(fmt.Errorf("unsupported query type %s", command.Type)) return nil } - if execErr != nil { - log.Error(fmt.Errorf("error executing query: %w", execErr)) - return nil - } - - // For SELECT queries, return results - if command.Type == "SELECT" { - return results - } - - // No results to return for UPDATE and DELETE queries - return nil + return results } func (t templateHelpers) parseJournalBasedOnIndex(indexName, keyValue, dataSource, queryType, lookupQuery string, options *raymond.Options) interface{} { diff --git a/core/templating/templating_test.go b/core/templating/templating_test.go index fc36013ed..965281ec3 100644 --- a/core/templating/templating_test.go +++ b/core/templating/templating_test.go @@ -15,7 +15,7 @@ import ( func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '2' 'Marks'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv1' 'Id' '2' 'Marks'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`56`)) @@ -24,7 +24,7 @@ func Test_ApplyTemplate_ParseCsvAndReturnMatchedString(t *testing.T) { func Test_ApplyTemplate_ParseCsvAndReturnFallbackStringIfNoMatchFound(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv' 'Id' '51' 'Marks'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csv 'test-csv1' 'Id' '51' 'Marks'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`ABSENT`)) @@ -76,7 +76,7 @@ func Test_ApplyTemplate_ParseCsv_WithEachBlockAndMissingDataSource(t *testing.T) func Test_ApplyTemplate_MatchingRowsCsvAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '2')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv1' 'id' '2')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test2`)) @@ -94,7 +94,7 @@ func Test_ApplyTemplate_MatchingRowsCsvMissingDataSource(t *testing.T) { func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv' 'id' '99')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvMatchingRows 'test-csv1' 'id' '99')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(``)) @@ -104,7 +104,7 @@ func Test_ApplyTemplate_MatchingRowsCsvInvalidKey(t *testing.T) { func Test_ApplyTemplate_CsvAsArrayAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv')}}{{#each this}}{{this}}{{/each}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsArray 'test-csv1')}}{{#each this}}{{this}}{{/each}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`idnamemarks1Test1552Test256*DummyABSENT`)) @@ -123,7 +123,7 @@ func Test_ApplyTemplate_CsvAsArrayMissingDataSource(t *testing.T) { func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv1')}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test1Test2Dummy`)) @@ -143,23 +143,23 @@ func Test_ApplyTemplate_CsvAsMapMissingDataSource(t *testing.T) { func Test_ApplyTemplate_CsvAddRow(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}{{csv 'test-csv' 'id' '99' 'name'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '99' false}}{{addToArray 'newMark' 'Violet' false}}{{addToArray 'newMark' '55' false}}{{csvAddRow 'test-csv1' (getArray 'newMark')}}{{csv 'test-csv1' 'id' '99' 'name'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Violet`)) //Revert the data to original state - _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '99' false}}`) + _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'id' '99' false}}`) } func Test_ApplyTemplate_CsvDeleteRows(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'id' '*' true}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'id' '*' true}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`1`)) //Revert the data to original state - _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '*' false}}{{addToArray 'newMark' 'Dummy' false}}{{addToArray 'newMark' 'ABSENT' false}}{{csvAddRow 'test-csv' (getArray 'newMark')}}`) + _, _ = ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{addToArray 'newMark' '*' false}}{{addToArray 'newMark' 'Dummy' false}}{{addToArray 'newMark' 'ABSENT' false}}{{csvAddRow 'test-csv1' (getArray 'newMark')}}`) } func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { @@ -174,7 +174,7 @@ func Test_ApplyTemplate_CsvDeleteMissingDataset(t *testing.T) { func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv' 'identity' '2' true}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvDeleteRows 'test-csv1' 'identity' '2' true}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(``)) @@ -183,7 +183,7 @@ func Test_ApplyTemplate_CsvDeleteMissingField(t *testing.T) { func Test_ApplyTemplate_CsvCountRows(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv'}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvCountRows 'test-csv1'}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`3`)) @@ -201,34 +201,16 @@ func Test_ApplyTemplate_CsvCountRowsMissingDataset(t *testing.T) { func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT name FROM test-csv WHERE id == '1'")}}{{this.name}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT name FROM test-csv1 WHERE id == '1'")}}{{this.name}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`Test1`)) } -// func Test_ApplyTemplate_CsvSQL_Select(t *testing.T) { -// RegisterTestingT(t) - -// template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSQL 'SELECT * FROM test-csv')}}{{this.name}}{{/each}}`) - -// Expect(err).To(BeNil()) -// Expect(template).To(Equal(`Test1Test2Dummy`)) -// } - -func Test_ApplyTemplate_CsvAsMapAndReturnMatchedString2(t *testing.T) { - RegisterTestingT(t) - - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvAsMap 'test-csv')}}{{this.name}}{{/each}}`) - - Expect(err).To(BeNil()) - Expect(template).To(Equal(`Test1Test2Dummy`)) -} - func Test_ApplyTemplate_CsvSQL_SelectAll(t *testing.T) { RegisterTestingT(t) - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT * FROM test-csv WHERE 1==1")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT * FROM test-csv1 WHERE 1==1")}}{{this.id}},{{this.name}},{{this.marks}};{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`1,Test1,55;2,Test2,56;*,Dummy,ABSENT;`)) @@ -238,11 +220,11 @@ func Test_ApplyTemplate_CsvSQL_Update(t *testing.T) { RegisterTestingT(t) // First, update the data - _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSqlCommand "UPDATE test-csv SET marks = '60' WHERE id == '1'"}}`) + _, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{csvSqlCommand "UPDATE test-csv1 SET marks = '60' WHERE id == '1'"}}`) Expect(err).To(BeNil()) // Then, check the result - template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT marks FROM test-csv WHERE id == '1'")}}{{this.marks}}{{/each}}`) + template, err := ApplyTemplate(&models.RequestDetails{}, make(map[string]string), `{{#each (csvSqlCommand "SELECT marks FROM test-csv1 WHERE id == '1'")}}{{this.marks}}{{/each}}`) Expect(err).To(BeNil()) Expect(template).To(Equal(`60`)) @@ -927,9 +909,9 @@ func toInterfaceSlice(arguments []string) []interface{} { func ApplyTemplate(requestDetails *models.RequestDetails, state map[string]string, responseBody string) (string, error) { templator := templating.NewTemplator() - dataSource1, _ := templating.NewCsvDataSource("test-csv", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") + dataSource1, _ := templating.NewCsvDataSource("test-csv1", "id,name,marks\n1,Test1,55\n2,Test2,56\n*,Dummy,ABSENT") dataSource2, _ := templating.NewCsvDataSource("test-csv2", "id,name,marks\n1,Test1,55\n2,Test2,56\n5553686208582,Test3,66\n") - templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv", dataSource1) + templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv1", dataSource1) templator.TemplateHelper.TemplateDataSource.SetDataSource("test-csv2", dataSource2) template, err := templator.ParseTemplate(responseBody) diff --git a/docs/pages/keyconcepts/templating/templating.rst b/docs/pages/keyconcepts/templating/templating.rst index c7ddd8762..4f80f075a 100644 --- a/docs/pages/keyconcepts/templating/templating.rst +++ b/docs/pages/keyconcepts/templating/templating.rst @@ -418,6 +418,16 @@ Deleting data from a CSV Data Source using SQL like syntax Example: ``{{ csvSqlCommand "DELETE FROM pets WHERE id > '20'" }}`` +Calling the csvSqlCommand for Delete and Update commands will return a map with a single value in it called rowsAffected. +If you call it without wrapping it in an #each block or a #first block then you will have the map written into your template. + +To call it with {{#first}}, and to do something with the resultant map you can do so like this: +(You can also simply ignore the rows affected) + +``{{#first (csvSqlCommand ("DELETE FROM pets WHERE category == 'birds'")}} +Rows Affected: {{this.rowsAffected}} +{{/first}}`` + Update the data in a CSV Data Source using SQL like syntax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -428,6 +438,13 @@ Update the data in a CSV Data Source using SQL like syntax Example: ``{{ csvSqlCommand "UPDATE pets SET status = 'sold' WHERE id > '20' AND category == 'cats'" }}`` +Again, you probably don't want the raw map rendered into your template so you can wrap it in a #first block which will allow +you to do something with the resultant map or ignore it completely as below: + +``{{#first (csvSqlCommand ("UPDATE pets SET price = '10000', name = 'LIONEL' WHERE category == 'cats'")}} +{{/first}}`` + + Using SQL like syntax to query and manipulate data sources ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can use a simplified SQL like syntax to select, update and delete rows. From 33f692ba501bd735bd3f5998ef0715f482ce0932 Mon Sep 17 00:00:00 2001 From: stuioco Date: Tue, 27 Aug 2024 09:59:09 +0100 Subject: [PATCH 7/9] Fixed return types for sql commands --- core/templating/datasource_sql_over_csv.go | 55 ++++++++----------- .../datasource_sql_over_csv_test.go | 27 ++++++--- core/templating/template_datasource.go | 2 +- core/templating/template_helpers.go | 8 ++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/core/templating/datasource_sql_over_csv.go b/core/templating/datasource_sql_over_csv.go index 4d17aefc6..d1468f88f 100644 --- a/core/templating/datasource_sql_over_csv.go +++ b/core/templating/datasource_sql_over_csv.go @@ -3,7 +3,6 @@ package templating import ( "errors" "regexp" - "strconv" "strings" log "github.com/sirupsen/logrus" @@ -56,7 +55,7 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement } columnsPart = matches[1] dataSourceName = matches[2] - if !dataSourceExists(datasource, dataSourceName) { + if !datasource.DataSourceExists(dataSourceName) { return SQLStatement{}, errors.New("data source does not exist") } @@ -87,15 +86,18 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement return SQLStatement{}, errors.New("invalid UPDATE query format") } dataSourceName = matches[1] - if !dataSourceExists(datasource, dataSourceName) { + if !datasource.DataSourceExists(dataSourceName) { return SQLStatement{}, errors.New("data source does not exist") } setPart := matches[2] if len(matches) == 4 { wherePart = matches[3] } - - setClauses := parseSetClauses(setPart) + headers := datasource.DataSources[dataSourceName].Data[0] + setClauses, err := parseSetClauses(setPart, headers) + if err != nil { + return SQLStatement{}, err + } conditions, err := parseConditions(wherePart) if err != nil { return SQLStatement{}, err @@ -114,7 +116,7 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement return SQLStatement{}, errors.New("invalid DELETE query format") } dataSourceName = matches[1] - if !dataSourceExists(datasource, dataSourceName) { + if !datasource.DataSourceExists(dataSourceName) { return SQLStatement{}, errors.New("data source does not exist") } if len(matches) == 3 { @@ -179,16 +181,18 @@ func trimQuotes(s string) string { } // parseSetClauses parses the SET part of an UPDATE query -func parseSetClauses(setPart string) map[string]string { +func parseSetClauses(setPart string, headers []string) (map[string]string, error) { setClauses := make(map[string]string) parts := strings.Split(setPart, ",") for _, part := range parts { keyValue := strings.Split(strings.TrimSpace(part), "=") - if len(keyValue) == 2 { + if !stringExists(headers, strings.TrimSpace(keyValue[0])) { + return nil, errors.New("invalid column provided: " + strings.TrimSpace(keyValue[0])) + } else if len(keyValue) == 2 { setClauses[strings.TrimSpace(keyValue[0])] = trimQuotes(strings.TrimSpace(keyValue[1])) } } - return setClauses + return setClauses, nil } // parseConditions parses the WHERE part of the query into a slice of Conditions and returns an error if any issues are found. @@ -230,14 +234,10 @@ func executeSqlSelectQuery(data *[][]string, query SQLStatement) []RowMap { } // ExecuteUpdateQuery executes an UPDATE query and modifies the data in-place -func executeSqlUpdateCommand(data *[][]string, query SQLStatement) []RowMap { +func executeSqlUpdateCommand(data *[][]string, query SQLStatement) int { if len(*data) < 2 { - log.Error("no data available to update") - return []RowMap{ - { - "rowsAffected": "0", - }, - } + log.Debug("no data available to update") + return 0 } headers := (*data)[0] @@ -256,23 +256,16 @@ func executeSqlUpdateCommand(data *[][]string, query SQLStatement) []RowMap { } } } - return []RowMap{ - { - "rowsAffected": strconv.Itoa(rowsAffected), - }, - } + return rowsAffected } // executeSqlDeleteCommand executes a DELETE query and modifies the data in-place -func executeSqlDeleteCommand(data *[][]string, query SQLStatement) []RowMap { +func executeSqlDeleteCommand(data *[][]string, query SQLStatement) int { if len(*data) < 2 { - log.Println("no data available to delete") - return []RowMap{ - { - "rowsAffected": "0", - }, - } + log.Debug("no data available to delete") + return 0 } + headers := (*data)[0] conditions := query.Conditions rowsAffected := 0 @@ -285,11 +278,7 @@ func executeSqlDeleteCommand(data *[][]string, query SQLStatement) []RowMap { rowsAffected++ } } - return []RowMap{ - { - "rowsAffected": strconv.Itoa(rowsAffected), - }, - } + return rowsAffected } // removeRow removes the row at index rowIndex from the data. diff --git a/core/templating/datasource_sql_over_csv_test.go b/core/templating/datasource_sql_over_csv_test.go index 1aac981e0..6e2b7d4d2 100644 --- a/core/templating/datasource_sql_over_csv_test.go +++ b/core/templating/datasource_sql_over_csv_test.go @@ -100,19 +100,30 @@ func TestTrimQuotes(t *testing.T) { } } -func TestParseSetClauses(t *testing.T) { +func TestParseSetClauses_ValidInput(t *testing.T) { input := "age = '35', department = 'Engineering'" + inputHeaders := []string{"age", "department"} expected := map[string]string{ "age": "35", "department": "Engineering", } - result := parseSetClauses(input) + result, _ := parseSetClauses(input, inputHeaders) if !reflect.DeepEqual(result, expected) { t.Errorf("expected %v, got %v", expected, result) } } +func TestParseSetClauses_InvalidInput(t *testing.T) { + input := "age = '35', department = 'Engineering'" + inputHeaders := []string{"fruit", "category"} + + _, err := parseSetClauses(input, inputHeaders) + if err == nil { + t.Errorf("expected error but got none.") + } +} + func TestParseConditions_ValidInput(t *testing.T) { wherePart := "id == '1' AND name != 'John' AND age >= '30'" expected := []Condition{ @@ -247,12 +258,12 @@ func TestExecuteSqlUpdateCommand_RowCountResult(t *testing.T) { DataSourceName: "employees", } - expected := "1" + expected := 1 result := executeSqlUpdateCommand(&data, query) - if result[0]["rowsAffected"] != expected { - t.Errorf("expected %v, got %v", expected, result[0]["rowsAffected"]) + if result != expected { + t.Errorf("expected %v, got %v", expected, result) } } @@ -294,11 +305,11 @@ func TestExecuteSqlDeleteCommand_RowCountResult(t *testing.T) { DataSourceName: "employees", } - expected := "1" + expected := 1 result := executeSqlDeleteCommand(&data, query) - if result[0]["rowsAffected"] != expected { - t.Errorf("expected %v, got %v", expected, result[0]["rowsAffected"]) + if result != expected { + t.Errorf("expected %v, got %v", expected, result) } } diff --git a/core/templating/template_datasource.go b/core/templating/template_datasource.go index aee97b841..7165fb10e 100644 --- a/core/templating/template_datasource.go +++ b/core/templating/template_datasource.go @@ -38,7 +38,7 @@ func (templateDataSource *TemplateDataSource) GetAllDataSources() map[string]*Da return templateDataSource.DataSources } -func dataSourceExists(templateDataSource *TemplateDataSource, name string) bool { +func (templateDataSource *TemplateDataSource) DataSourceExists(name string) bool { templateDataSource.RWMutex.Lock() defer templateDataSource.RWMutex.Unlock() diff --git a/core/templating/template_helpers.go b/core/templating/template_helpers.go index 8e51ffdd7..f3f8460f8 100644 --- a/core/templating/template_helpers.go +++ b/core/templating/template_helpers.go @@ -450,9 +450,13 @@ func (t templateHelpers) csvSqlCommand(commandString string) []RowMap { case "SELECT": results = executeSqlSelectQuery(&source.Data, command) case "UPDATE": - results = executeSqlUpdateCommand(&source.Data, command) + rowsAffected := executeSqlUpdateCommand(&source.Data, command) + log.Debug(strconv.Itoa(rowsAffected) + " rows affected by " + commandString) + return nil case "DELETE": - results = executeSqlDeleteCommand(&source.Data, command) + rowsAffected := executeSqlDeleteCommand(&source.Data, command) + log.Debug(strconv.Itoa(rowsAffected) + " rows affected by " + commandString) + return nil default: log.Error(fmt.Errorf("unsupported query type %s", command.Type)) return nil From 8456ff443646d7ada3535e11746e7b1f76a36da1 Mon Sep 17 00:00:00 2001 From: stuioco Date: Tue, 27 Aug 2024 16:10:19 +0100 Subject: [PATCH 8/9] Updated docs for sql update and delete --- docs/pages/keyconcepts/templating/templating.rst | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/pages/keyconcepts/templating/templating.rst b/docs/pages/keyconcepts/templating/templating.rst index 4f80f075a..b18d345f0 100644 --- a/docs/pages/keyconcepts/templating/templating.rst +++ b/docs/pages/keyconcepts/templating/templating.rst @@ -418,15 +418,10 @@ Deleting data from a CSV Data Source using SQL like syntax Example: ``{{ csvSqlCommand "DELETE FROM pets WHERE id > '20'" }}`` -Calling the csvSqlCommand for Delete and Update commands will return a map with a single value in it called rowsAffected. -If you call it without wrapping it in an #each block or a #first block then you will have the map written into your template. +Calling the csvSqlCommand for Delete and Update commands will execute the statements without outputting anything to the template. +The rows affected will be logged for debugging purposes. -To call it with {{#first}}, and to do something with the resultant map you can do so like this: -(You can also simply ignore the rows affected) -``{{#first (csvSqlCommand ("DELETE FROM pets WHERE category == 'birds'")}} -Rows Affected: {{this.rowsAffected}} -{{/first}}`` Update the data in a CSV Data Source using SQL like syntax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -438,11 +433,6 @@ Update the data in a CSV Data Source using SQL like syntax Example: ``{{ csvSqlCommand "UPDATE pets SET status = 'sold' WHERE id > '20' AND category == 'cats'" }}`` -Again, you probably don't want the raw map rendered into your template so you can wrap it in a #first block which will allow -you to do something with the resultant map or ignore it completely as below: - -``{{#first (csvSqlCommand ("UPDATE pets SET price = '10000', name = 'LIONEL' WHERE category == 'cats'")}} -{{/first}}`` Using SQL like syntax to query and manipulate data sources From 084d5cfbd120d300ff069a17af4c4bfcb197948f Mon Sep 17 00:00:00 2001 From: stuioco Date: Tue, 27 Aug 2024 16:30:21 +0100 Subject: [PATCH 9/9] Updated go.mod and ran go mod vendor for latest Spectolabs/raymond --- go.mod | 2 +- go.sum | 2 ++ vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 72513e875..83f6d7947 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/SpectoLabs/goproxy v0.0.0-20240717215706-55e01f38b2c9 github.com/SpectoLabs/goproxy/ext v0.0.0-20220724222243-c982a2c966ae github.com/SpectoLabs/goxml2json v0.0.0-20240121223617-8e03292c14ea - github.com/SpectoLabs/raymond v2.0.3-0.20240313210732-e0e216cf0920+incompatible + github.com/SpectoLabs/raymond v2.0.3-0.20240827093205-07f3a7bebd7d+incompatible github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 github.com/beevik/etree v1.1.0 github.com/boltdb/bolt v1.2.1-0.20160424201119-d97499360d1e diff --git a/go.sum b/go.sum index a3ba946b9..b529d9db2 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/SpectoLabs/goxml2json v0.0.0-20240121223617-8e03292c14ea h1:VAu3xT0OS github.com/SpectoLabs/goxml2json v0.0.0-20240121223617-8e03292c14ea/go.mod h1:P8Yk6l9gKxTfgnJeXVEQKCH7nGgvhNdeLs0zVs2h2KM= github.com/SpectoLabs/raymond v2.0.3-0.20240313210732-e0e216cf0920+incompatible h1:qI0OVNVzpRLUMHD3lxGOXSHsH87xHp09IuQm9hjTfTA= github.com/SpectoLabs/raymond v2.0.3-0.20240313210732-e0e216cf0920+incompatible/go.mod h1:Cv+3TFLm3T7C7ML7gEK4HnHcTpfc4HjJFWvS91cc6tw= +github.com/SpectoLabs/raymond v2.0.3-0.20240827093205-07f3a7bebd7d+incompatible h1:WQc9MpeneDjPv0utx7P5RmXPLfyWpbqXb+LiHK+hZxg= +github.com/SpectoLabs/raymond v2.0.3-0.20240827093205-07f3a7bebd7d+incompatible/go.mod h1:Cv+3TFLm3T7C7ML7gEK4HnHcTpfc4HjJFWvS91cc6tw= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 h1:dESITdufxuiwgQh1YPiPupEXORHTYvY8tr40nvrWelo= github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496/go.mod h1:+GxMEKI0Va2U8h3os6oiUAetHAlGMvxjdpAH/9uvUMA= diff --git a/vendor/modules.txt b/vendor/modules.txt index b57435b89..55b11816a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -24,7 +24,7 @@ github.com/SpectoLabs/goproxy/ext/auth # github.com/SpectoLabs/goxml2json v0.0.0-20240121223617-8e03292c14ea ## explicit; go 1.16 github.com/SpectoLabs/goxml2json -# github.com/SpectoLabs/raymond v2.0.3-0.20240313210732-e0e216cf0920+incompatible +# github.com/SpectoLabs/raymond v2.0.3-0.20240827093205-07f3a7bebd7d+incompatible ## explicit github.com/SpectoLabs/raymond # github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496