Skip to content

Commit 9834af7

Browse files
idrissneumannidrissneumann
committed
Issue #4: find the timestamp field dynamically
1 parent dd30b40 commit 9834af7

File tree

3 files changed

+102
-50
lines changed

3 files changed

+102
-50
lines changed

‎pkg/quickwit/quickwit.go

Lines changed: 72 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"bytes"
55
"context"
66
"encoding/json"
7-
"errors"
87
"fmt"
98
"io"
109
"net/http"
@@ -26,6 +25,69 @@ type QuickwitDatasource struct {
2625
dsInfo es.DatasourceInfo
2726
}
2827

28+
type QuickwitMapping struct {
29+
IndexConfig struct {
30+
DocMapping struct {
31+
TimestampField string `json:"timestamp_field"`
32+
FieldMappings []struct {
33+
Name string `json:"name"`
34+
InputFormats []string `json:"input_formats"`
35+
} `json:"field_mappings"`
36+
} `json:"doc_mapping"`
37+
} `json:"index_config"`
38+
}
39+
40+
func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) {
41+
mappingEndpointUrl := qwUrl + "/indexes/" + index
42+
qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl)
43+
r, err := cli.Get(mappingEndpointUrl)
44+
if err != nil {
45+
errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
46+
qwlog.Error(errMsg)
47+
return "", "", err
48+
}
49+
50+
statusCode := r.StatusCode
51+
if statusCode < 200 || statusCode >= 400 {
52+
errMsg := fmt.Sprintf("Error when calling url = %s: statusCode = %d", mappingEndpointUrl, statusCode)
53+
qwlog.Error(errMsg)
54+
return "", "", fmt.Errorf(errMsg)
55+
}
56+
57+
defer r.Body.Close()
58+
body, err := io.ReadAll(r.Body)
59+
if err != nil {
60+
errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
61+
qwlog.Error(errMsg)
62+
return "", "", err
63+
}
64+
65+
var payload QuickwitMapping
66+
err = json.Unmarshal(body, &payload)
67+
if err != nil {
68+
errMsg := fmt.Sprintf("Unmarshalling body error: %s", string(body))
69+
qwlog.Error(errMsg)
70+
return "", "", fmt.Errorf(errMsg)
71+
}
72+
73+
timestampFieldName := payload.IndexConfig.DocMapping.TimestampField
74+
timestampFieldFormat := "undef"
75+
for _, field := range payload.IndexConfig.DocMapping.FieldMappings {
76+
if field.Name == timestampFieldName && len(field.InputFormats) > 0 {
77+
timestampFieldFormat = field.InputFormats[0]
78+
break
79+
}
80+
}
81+
82+
if timestampFieldFormat == "undef" {
83+
errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName))
84+
return timestampFieldName, "", fmt.Errorf(errMsg)
85+
}
86+
87+
qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, timestampFieldFormat))
88+
return timestampFieldName, timestampFieldFormat, nil
89+
}
90+
2991
// Creates a Quickwit datasource.
3092
func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
3193
qwlog.Debug("Initializing new data source instance")
@@ -50,19 +112,8 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
50112
return nil, err
51113
}
52114

53-
timeField, ok := jsonData["timeField"].(string)
54-
if !ok {
55-
return nil, errors.New("timeField cannot be cast to string")
56-
}
57-
58-
if timeField == "" {
59-
return nil, errors.New("a time field name is required")
60-
}
61-
62-
timeOutputFormat, ok := jsonData["timeOutputFormat"].(string)
63-
if !ok {
64-
return nil, errors.New("timeOutputFormat cannot be cast to string")
65-
}
115+
timeField, toerr := jsonData["timeField"].(string)
116+
timeOutputFormat, toferr := jsonData["timeOutputFormat"].(string)
66117

67118
logLevelField, ok := jsonData["logLevelField"].(string)
68119
if !ok {
@@ -96,6 +147,13 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
96147
maxConcurrentShardRequests = 256
97148
}
98149

150+
if !toerr || !toferr {
151+
timeField, timeOutputFormat, err = getTimestampFieldInfos(index, settings.URL, httpCli)
152+
if nil != err {
153+
return nil, err
154+
}
155+
}
156+
99157
configuredFields := es.ConfiguredFields{
100158
TimeField: timeField,
101159
TimeOutputFormat: timeOutputFormat,

‎src/configuration/ConfigEditor.tsx

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -49,24 +49,6 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => {
4949
width={40}
5050
/>
5151
</InlineField>
52-
<InlineField label="Timestamp field" labelWidth={26} tooltip="Timestamp field of your index. Required.">
53-
<Input
54-
id="quickwit_index_timestamp_field"
55-
value={value.jsonData.timeField}
56-
onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeField: event.currentTarget.value}})}
57-
placeholder="timestamp"
58-
width={40}
59-
/>
60-
</InlineField>
61-
<InlineField label="Timestamp output format" labelWidth={26} tooltip="If you don't remember the output format, check the datasource and the error message will give you a hint.">
62-
<Input
63-
id="quickwit_index_timestamp_field_output_format"
64-
value={value.jsonData.timeOutputFormat}
65-
onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeOutputFormat: event.currentTarget.value}})}
66-
placeholder="unix_timestamp_millisecs"
67-
width={40}
68-
/>
69-
</InlineField>
7052
<InlineField label="Message field name" labelWidth={26} tooltip="Field used to display a log line in the Explore view">
7153
<Input
7254
id="quickwit_log_message_field"

‎src/datasource.ts

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,31 @@ export class QuickwitDataSource
7272
super(instanceSettings);
7373
const settingsData = instanceSettings.jsonData || ({} as QuickwitOptions);
7474
this.index = settingsData.index || '';
75-
this.timeField = settingsData.timeField || '';
76-
this.timeOutputFormat = settingsData.timeOutputFormat || '';
77-
this.logMessageField = settingsData.logMessageField || '';
78-
this.logLevelField = settingsData.logLevelField || '';
75+
this.timeField = ''
76+
this.timeOutputFormat = ''
7977
this.queryBuilder = new ElasticQueryBuilder({
8078
timeField: this.timeField,
8179
});
80+
from(this.getResource('indexes/' + this.index)).pipe(
81+
map((indexMetadata) => {
82+
let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
83+
let timestampFieldName = indexMetadata.index_config.doc_mapping.timestamp_field
84+
let timestampField = fields.find((field) => field.json_path === timestampFieldName);
85+
let timestampFormat = timestampField ? timestampField.field_mapping.output_format || '' : ''
86+
let timestampFieldInfos = { 'field': timestampFieldName, 'format': timestampFormat }
87+
console.log("timestampFieldInfos = " + JSON.stringify(timestampFieldInfos))
88+
return timestampFieldInfos
89+
})
90+
).subscribe(result => {
91+
this.timeField = result.field;
92+
this.timeOutputFormat = result.format;
93+
this.queryBuilder = new ElasticQueryBuilder({
94+
timeField: this.timeField,
95+
});
96+
});
97+
98+
this.logMessageField = settingsData.logMessageField || '';
99+
this.logLevelField = settingsData.logLevelField || '';
82100
this.dataLinks = settingsData.dataLinks || [];
83101
this.languageProvider = new ElasticsearchLanguageProvider(this);
84102
}
@@ -104,12 +122,7 @@ export class QuickwitDataSource
104122
message: 'Cannot save datasource, `index` is required',
105123
};
106124
}
107-
if (this.timeField === '' ) {
108-
return {
109-
status: 'error',
110-
message: 'Cannot save datasource, `timeField` is required',
111-
};
112-
}
125+
113126
return lastValueFrom(
114127
from(this.getResource('indexes/' + this.index)).pipe(
115128
mergeMap((indexMetadata) => {
@@ -140,21 +153,19 @@ export class QuickwitDataSource
140153
if (this.timeField === '') {
141154
return `Time field must not be empty`;
142155
}
143-
if (indexMetadata.index_config.doc_mapping.timestamp_field !== this.timeField) {
144-
return `No timestamp field named '${this.timeField}' found`;
145-
}
156+
146157
let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
147158
let timestampField = fields.find((field) => field.json_path === this.timeField);
159+
148160
// Should never happen.
149161
if (timestampField === undefined) {
150162
return `No field named '${this.timeField}' found in the doc mapping. This should never happen.`;
151163
}
152-
if (timestampField.field_mapping.output_format !== this.timeOutputFormat) {
153-
return `Timestamp output format is declared as '${timestampField.field_mapping.output_format}' in the doc mapping, not '${this.timeOutputFormat}'.`;
154-
}
164+
165+
let timeOutputFormat = timestampField.field_mapping.output_format || 'unknown';
155166
const supportedTimestampOutputFormats = ['unix_timestamp_secs', 'unix_timestamp_millis', 'unix_timestamp_micros', 'unix_timestamp_nanos', 'iso8601', 'rfc3339'];
156-
if (!supportedTimestampOutputFormats.includes(this.timeOutputFormat)) {
157-
return `Timestamp output format '${this.timeOutputFormat} is not yet supported.`;
167+
if (!supportedTimestampOutputFormats.includes(timeOutputFormat)) {
168+
return `Timestamp output format '${timeOutputFormat} is not yet supported.`;
158169
}
159170
return;
160171
}
@@ -303,6 +314,7 @@ export class QuickwitDataSource
303314
ignore_unavailable: true,
304315
index: this.index,
305316
});
317+
306318
let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
307319
esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf().toString());
308320
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf().toString());

0 commit comments

Comments
 (0)