Skip to content

Commit 4e7a873

Browse files
idrissneumannidrissneumann
committed
Issue #4: find the timestamp field dynamically
1 parent 67f56ba commit 4e7a873

File tree

3 files changed

+102
-50
lines changed

3 files changed

+102
-50
lines changed

‎pkg/quickwit/quickwit.go

Lines changed: 72 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"bytes"
55
"context"
66
"encoding/json"
7-
"errors"
87
"fmt"
98
"io"
109
"net/http"
@@ -26,6 +25,69 @@ type QuickwitDatasource struct {
2625
dsInfo es.DatasourceInfo
2726
}
2827

28+
type QuickwitMapping struct {
29+
IndexConfig struct {
30+
DocMapping struct {
31+
TimestampField string `json:"timestamp_field"`
32+
FieldMappings []struct {
33+
Name string `json:"name"`
34+
InputFormats []string `json:"input_formats"`
35+
} `json:"field_mappings"`
36+
} `json:"doc_mapping"`
37+
} `json:"index_config"`
38+
}
39+
40+
func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) {
41+
mappingEndpointUrl := qwUrl + "/indexes/" + index
42+
qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl)
43+
r, err := cli.Get(mappingEndpointUrl)
44+
if err != nil {
45+
errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
46+
qwlog.Error(errMsg)
47+
return "", "", err
48+
}
49+
50+
statusCode := r.StatusCode
51+
if statusCode < 200 || statusCode >= 400 {
52+
errMsg := fmt.Sprintf("Error when calling url = %s: statusCode = %d", mappingEndpointUrl, statusCode)
53+
qwlog.Error(errMsg)
54+
return "", "", fmt.Errorf(errMsg)
55+
}
56+
57+
defer r.Body.Close()
58+
body, err := io.ReadAll(r.Body)
59+
if err != nil {
60+
errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
61+
qwlog.Error(errMsg)
62+
return "", "", err
63+
}
64+
65+
var payload QuickwitMapping
66+
err = json.Unmarshal(body, &payload)
67+
if err != nil {
68+
errMsg := fmt.Sprintf("Unmarshalling body error: %s", string(body))
69+
qwlog.Error(errMsg)
70+
return "", "", fmt.Errorf(errMsg)
71+
}
72+
73+
timestampFieldName := payload.IndexConfig.DocMapping.TimestampField
74+
timestampFieldFormat := "undef"
75+
for _, field := range payload.IndexConfig.DocMapping.FieldMappings {
76+
if field.Name == timestampFieldName && len(field.InputFormats) > 0 {
77+
timestampFieldFormat = field.InputFormats[0]
78+
break
79+
}
80+
}
81+
82+
if timestampFieldFormat == "undef" {
83+
errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName))
84+
return timestampFieldName, "", fmt.Errorf(errMsg)
85+
}
86+
87+
qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, timestampFieldFormat))
88+
return timestampFieldName, timestampFieldFormat, nil
89+
}
90+
2991
// Creates a Quickwit datasource.
3092
func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
3193
qwlog.Debug("Initializing new data source instance")
@@ -50,19 +112,8 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
50112
return nil, err
51113
}
52114

53-
timeField, ok := jsonData["timeField"].(string)
54-
if !ok {
55-
return nil, errors.New("timeField cannot be cast to string")
56-
}
57-
58-
if timeField == "" {
59-
return nil, errors.New("a time field name is required")
60-
}
61-
62-
timeOutputFormat, ok := jsonData["timeOutputFormat"].(string)
63-
if !ok {
64-
return nil, errors.New("timeOutputFormat cannot be cast to string")
65-
}
115+
timeField, toOk := jsonData["timeField"].(string)
116+
timeOutputFormat, tofOk := jsonData["timeOutputFormat"].(string)
66117

67118
logLevelField, ok := jsonData["logLevelField"].(string)
68119
if !ok {
@@ -96,6 +147,13 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
96147
maxConcurrentShardRequests = 256
97148
}
98149

150+
if !toOk || !tofOk {
151+
timeField, timeOutputFormat, err = getTimestampFieldInfos(index, settings.URL, httpCli)
152+
if nil != err {
153+
return nil, err
154+
}
155+
}
156+
99157
configuredFields := es.ConfiguredFields{
100158
TimeField: timeField,
101159
TimeOutputFormat: timeOutputFormat,

‎src/configuration/ConfigEditor.tsx

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,24 +73,6 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => {
7373
width={40}
7474
/>
7575
</InlineField>
76-
<InlineField label="Timestamp field" labelWidth={26} tooltip="Timestamp field of your index. Required.">
77-
<Input
78-
id="quickwit_index_timestamp_field"
79-
value={value.jsonData.timeField}
80-
onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeField: event.currentTarget.value}})}
81-
placeholder="timestamp"
82-
width={40}
83-
/>
84-
</InlineField>
85-
<InlineField label="Timestamp output format" labelWidth={26} tooltip="If you don't remember the output format, check the datasource and the error message will give you a hint.">
86-
<Input
87-
id="quickwit_index_timestamp_field_output_format"
88-
value={value.jsonData.timeOutputFormat}
89-
onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeOutputFormat: event.currentTarget.value}})}
90-
placeholder="unix_timestamp_millisecs"
91-
width={40}
92-
/>
93-
</InlineField>
9476
<InlineField label="Message field name" labelWidth={26} tooltip="Field used to display a log line in the Explore view">
9577
<Input
9678
id="quickwit_log_message_field"

‎src/datasource.ts

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,31 @@ export class QuickwitDataSource
7979
super(instanceSettings);
8080
const settingsData = instanceSettings.jsonData || ({} as QuickwitOptions);
8181
this.index = settingsData.index || '';
82-
this.timeField = settingsData.timeField || '';
83-
this.timeOutputFormat = settingsData.timeOutputFormat || '';
84-
this.logMessageField = settingsData.logMessageField || '';
85-
this.logLevelField = settingsData.logLevelField || '';
82+
this.timeField = ''
83+
this.timeOutputFormat = ''
8684
this.queryBuilder = new ElasticQueryBuilder({
8785
timeField: this.timeField,
8886
});
87+
from(this.getResource('indexes/' + this.index)).pipe(
88+
map((indexMetadata) => {
89+
let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
90+
let timestampFieldName = indexMetadata.index_config.doc_mapping.timestamp_field
91+
let timestampField = fields.find((field) => field.json_path === timestampFieldName);
92+
let timestampFormat = timestampField ? timestampField.field_mapping.output_format || '' : ''
93+
let timestampFieldInfos = { 'field': timestampFieldName, 'format': timestampFormat }
94+
console.log("timestampFieldInfos = " + JSON.stringify(timestampFieldInfos))
95+
return timestampFieldInfos
96+
})
97+
).subscribe(result => {
98+
this.timeField = result.field;
99+
this.timeOutputFormat = result.format;
100+
this.queryBuilder = new ElasticQueryBuilder({
101+
timeField: this.timeField,
102+
});
103+
});
104+
105+
this.logMessageField = settingsData.logMessageField || '';
106+
this.logLevelField = settingsData.logLevelField || '';
89107
this.dataLinks = settingsData.dataLinks || [];
90108
this.languageProvider = new ElasticsearchLanguageProvider(this);
91109
}
@@ -111,12 +129,7 @@ export class QuickwitDataSource
111129
message: 'Cannot save datasource, `index` is required',
112130
};
113131
}
114-
if (this.timeField === '' ) {
115-
return {
116-
status: 'error',
117-
message: 'Cannot save datasource, `timeField` is required',
118-
};
119-
}
132+
120133
return lastValueFrom(
121134
from(this.getResource('indexes/' + this.index)).pipe(
122135
mergeMap((indexMetadata) => {
@@ -147,21 +160,19 @@ export class QuickwitDataSource
147160
if (this.timeField === '') {
148161
return `Time field must not be empty`;
149162
}
150-
if (indexMetadata.index_config.doc_mapping.timestamp_field !== this.timeField) {
151-
return `No timestamp field named '${this.timeField}' found`;
152-
}
163+
153164
let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
154165
let timestampField = fields.find((field) => field.json_path === this.timeField);
166+
155167
// Should never happen.
156168
if (timestampField === undefined) {
157169
return `No field named '${this.timeField}' found in the doc mapping. This should never happen.`;
158170
}
159-
if (timestampField.field_mapping.output_format !== this.timeOutputFormat) {
160-
return `Timestamp output format is declared as '${timestampField.field_mapping.output_format}' in the doc mapping, not '${this.timeOutputFormat}'.`;
161-
}
171+
172+
let timeOutputFormat = timestampField.field_mapping.output_format || 'unknown';
162173
const supportedTimestampOutputFormats = ['unix_timestamp_secs', 'unix_timestamp_millis', 'unix_timestamp_micros', 'unix_timestamp_nanos', 'iso8601', 'rfc3339'];
163-
if (!supportedTimestampOutputFormats.includes(this.timeOutputFormat)) {
164-
return `Timestamp output format '${this.timeOutputFormat} is not yet supported.`;
174+
if (!supportedTimestampOutputFormats.includes(timeOutputFormat)) {
175+
return `Timestamp output format '${timeOutputFormat} is not yet supported.`;
165176
}
166177
return;
167178
}
@@ -310,6 +321,7 @@ export class QuickwitDataSource
310321
ignore_unavailable: true,
311322
index: this.index,
312323
});
324+
313325
let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
314326
esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf().toString());
315327
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf().toString());

0 commit comments

Comments
 (0)