Skip to content

Commit f35ff31

Browse files
committed
feat: Implement better during/range temporal comparisons
* Refactor temporal comparisons to provide granular "during" thresholds * Use duration for ignoring discovery with aggressive mode in Deezer #296
1 parent d1fc6dc commit f35ff31

File tree

8 files changed

+127
-57
lines changed

8 files changed

+127
-57
lines changed

src/backend/scrobblers/AbstractScrobbleClient.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import {
1010
PlayObject,
1111
QueuedScrobble,
1212
TA_CLOSE,
13+
TA_DEFAULT_ACCURACY,
14+
TA_DURING,
1315
TA_FUZZY,
1416
TrackStringOptions,
1517
} from "../../core/Atomic.js";
@@ -42,7 +44,7 @@ import { messageWithCauses, messageWithCausesTruncatedDefault } from "../utils/E
4244
import { compareScrobbleArtists, compareScrobbleTracks, normalizeStr } from "../utils/StringUtils.js";
4345
import {
4446
comparePlayTemporally,
45-
temporalAccuracyIsAtLeast,
47+
hasAcceptableTemporalAccuracy,
4648
temporalAccuracyToString,
4749
temporalPlayComparisonSummary,
4850
} from "../utils/TimeUtils.js";
@@ -315,7 +317,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
315317

316318
const matchPlayDate = dtInvariantMatches.find((x: ScrobbledPlayObject) => {
317319
const temporalComparison = comparePlayTemporally(x.play, playObj);
318-
return temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match)
320+
return hasAcceptableTemporalAccuracy(temporalComparison.match)
319321
});
320322

321323
return [matchPlayDate, dtInvariantMatches];
@@ -403,9 +405,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i
403405

404406
const temporalComparison = comparePlayTemporally(x, playObj);
405407
let timeMatch = 0;
406-
if(temporalAccuracyIsAtLeast(TA_CLOSE, temporalComparison.match)) {
408+
if(hasAcceptableTemporalAccuracy(temporalComparison.match)) {
407409
timeMatch = 1;
408-
} else if(temporalComparison.match === TA_FUZZY) {
410+
} else if(hasAcceptableTemporalAccuracy(temporalComparison.match, [TA_FUZZY, TA_DURING])) {
409411
timeMatch = 0.6;
410412
}
411413

src/backend/sources/AbstractSource.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { childLogger, LogDataPretty } from '@foxxmd/logging';
22
import dayjs, { Dayjs } from "dayjs";
33
import { EventEmitter } from "events";
44
import { FixedSizeList } from "fixed-size-list";
5-
import { PlayObject, TA_CLOSE } from "../../core/Atomic.js";
5+
import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY } from "../../core/Atomic.js";
66
import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js";
77
import AbstractComponent from "../common/AbstractComponent.js";
88
import {
@@ -33,7 +33,7 @@ import {
3333
sortByNewestPlayDate,
3434
sortByOldestPlayDate,
3535
} from "../utils.js";
36-
import { comparePlayTemporally, temporalAccuracyIsAtLeast, timeToHumanTimestamp, todayAwareFormat } from "../utils/TimeUtils.js";
36+
import { timeToHumanTimestamp, todayAwareFormat } from "../utils/TimeUtils.js";
3737
import { getRoot } from '../ioc.js';
3838
import { componentFileLogger } from '../common/logging.js';
3939
import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js';
@@ -178,7 +178,7 @@ export default abstract class AbstractSource extends AbstractComponent implement
178178
for(const list of lists) {
179179
const existing = list.find(x => {
180180
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
181-
return genericSourcePlayMatch(e, candidate, TA_CLOSE);
181+
return genericSourcePlayMatch(e, candidate);
182182
});
183183
if(existing) {
184184
return existing;

src/backend/sources/DeezerInternalSource.ts

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import dayjs from "dayjs";
22
import EventEmitter from "events";
33
import request, { Request, Response, SuperAgent } from 'superagent';
4-
import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_FUZZY } from "../../core/Atomic.js";
4+
import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_DURING, TA_EXACT, TA_FUZZY, TemporalAccuracy } from "../../core/Atomic.js";
55
import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig, TRANSFORM_HOOK } from "../common/infrastructure/Atomic.js";
66
import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js";
77
import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js";
@@ -10,6 +10,7 @@ import { CookieJar, Cookie } from 'tough-cookie';
1010
import { MixedCookieAgent } from 'http-cookie-agent/http';
1111
import MemorySource from "./MemorySource.js";
1212
import { genericSourcePlayMatch } from "../utils/PlayComparisonUtils.js";
13+
import { TemporalPlayComparisonOptions } from "../utils/TimeUtils.js";
1314

1415
interface DeezerHistoryResponse {
1516
errors: []
@@ -204,15 +205,24 @@ export default class DeezerInternalSource extends MemorySource {
204205
for(const list of lists) {
205206
const existing = list.find(x => {
206207
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
207-
return genericSourcePlayMatch(e, candidate, TA_CLOSE);
208+
return genericSourcePlayMatch(e, candidate);
208209
});
209210
if(existing) {
210211
return existing;
211212
}
212213
if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
213214
const fuzzyIndex = list.findIndex(x => {
214215
const e = this.transformPlay(x, TRANSFORM_HOOK.existing);
215-
return genericSourcePlayMatch(e, candidate, TA_FUZZY, {fuzzyDiffThreshold: this.config.options?.fuzzyDiscoveryIgnore === 'aggressive' ? 40 : undefined});
216+
let temporalOptions: TemporalPlayComparisonOptions = {};
217+
const temporalAccuracy: TemporalAccuracy[] = [TA_EXACT, TA_CLOSE, TA_FUZZY];
218+
if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {
219+
temporalOptions = {
220+
fuzzyDiffThreshold: Math.max(100, x.data.duration * 0.5),
221+
duringReferences: ['duration', 'listenedFor', 'range']
222+
}
223+
temporalAccuracy.push(TA_DURING);
224+
}
225+
return genericSourcePlayMatch(e, candidate, temporalAccuracy, temporalOptions);
216226
});
217227
if(fuzzyIndex !== -1) {
218228
if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') {

src/backend/sources/JellyfinSource.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Logger } from "@foxxmd/logging";
22
import dayjs from "dayjs";
33
import EventEmitter from "events";
4-
import { PlayObject, TA_CLOSE } from "../../core/Atomic.js";
4+
import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT } from "../../core/Atomic.js";
55
import {
66
buildTrackString,
77
combinePartsToString,
@@ -19,7 +19,7 @@ import {
1919
import { parseDurationFromTimestamp } from '../utils/TimeUtils.js';
2020
import {
2121
comparePlayTemporally,
22-
temporalAccuracyIsAtLeast,
22+
hasAcceptableTemporalAccuracy,
2323
temporalPlayComparisonSummary,
2424
} from "../utils/TimeUtils.js";
2525
import MemorySource from "./MemorySource.js";
@@ -313,7 +313,7 @@ export default class JellyfinSource extends MemorySource {
313313
314314
Temporal Comparison => ${temporalPlayComparisonSummary(temporalResult, currPlay, playObj)}`);
315315
}
316-
if(temporalAccuracyIsAtLeast(TA_CLOSE,temporalResult.match)) {
316+
if(hasAcceptableTemporalAccuracy(temporalResult.match)) {
317317
existingTracked = currPlay;
318318
}
319319
break;

src/backend/tests/source/source.test.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,20 @@ describe('Deezer Internal Source', function() {
545545
expect(discovered.length).to.eq(0);
546546
});
547547

548+
it('does not discover play found during duration of previous', function() {
549+
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
550+
const targetPlay = normalizedPlays[normalizedPlays.length - 2]
551+
const duringPlay = clone(targetPlay);
552+
duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's');
553+
554+
const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
555+
source.discover([...normalizedPlays, interimPlay]);
556+
557+
const discovered = source.discover([duringPlay]);
558+
559+
expect(discovered.length).to.eq(0);
560+
});
561+
548562
it('does not discover fuzzy play with delay of up to 40 seconds', function() {
549563
const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80});
550564
const targetPlay = normalizedPlays[normalizedPlays.length - 2]

src/backend/utils/PlayComparisonUtils.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { getListDiff, ListDiff } from "@donedeal0/superdiff";
2-
import { PlayObject, TA_CLOSE, TemporalAccuracy } from "../../core/Atomic.js";
2+
import { PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../core/Atomic.js";
33
import { buildTrackString } from "../../core/StringUtils.js";
44
import { playObjDataMatch } from "../utils.js";
5-
import { comparePlayTemporally, temporalAccuracyIsAtLeast, TemporalPlayComparisonOptions } from "./TimeUtils.js";
5+
import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js";
66

77

88
export const metaInvariantTransform = (play: PlayObject): PlayObject => {
@@ -224,4 +224,6 @@ export const humanReadableDiff = (aPlay: PlayObject[], bPlay: PlayObject[], resu
224224
}).join('\n');
225225
}
226226

227-
export const genericSourcePlayMatch = (a: PlayObject, b: PlayObject, t: TemporalAccuracy = TA_CLOSE, temporalOptions?: TemporalPlayComparisonOptions): boolean => playObjDataMatch(a, b) && temporalAccuracyIsAtLeast(t, comparePlayTemporally(a, b, temporalOptions).match);
227+
export const genericSourcePlayMatch = (a: PlayObject, b: PlayObject, t?: TemporalAccuracy[], temporalOptions?: TemporalPlayComparisonOptions): boolean =>
228+
playObjDataMatch(a, b)
229+
&& hasAcceptableTemporalAccuracy(comparePlayTemporally(a, b, temporalOptions).match, t);

src/backend/utils/TimeUtils.ts

Lines changed: 64 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import dayjs, { Dayjs } from "dayjs";
22
import isToday from 'dayjs/plugin/isToday.js';
33
import {
4+
AcceptableTemporalDuringReference,
45
PlayObject,
56
SCROBBLE_TS_SOC_END,
67
SCROBBLE_TS_SOC_START,
78
ScrobbleTsSOC,
89
TA_CLOSE,
10+
TA_DEFAULT_ACCURACY,
11+
TA_DURING,
912
TA_EXACT,
1013
TA_FUZZY,
1114
TA_NONE,
@@ -38,19 +41,18 @@ export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, exis
3841
parts.push(`Play Diff: ${formatNumber(data.date.diff, {toFixed: 0})}s (Needed <${data.date.threshold}s)`)
3942
}
4043
if (data.date.fuzzyDurationDiff !== undefined) {
41-
parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= 10s)`);
44+
parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`);
4245
}
4346
if (data.date.fuzzyListenedDiff !== undefined) {
44-
parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= 10s)`);
47+
parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`);
4548
}
46-
if (data.range !== undefined) {
47-
if (data.range === false) {
48-
parts.push('Candidate not played during Existing tracked listening');
49-
} else {
50-
parts.push(`Candidate played during tracked listening range from Existing ${data.range[0].timestamp.format('HH:mm:ssZ')} => ${data.range[1].timestamp.format('HH:mm:ssZ')}`);
51-
}
52-
} else {
49+
50+
if(data.range === undefined) {
5351
parts.push('Range Comparison N/A');
52+
} else if(data.range.type === 'none') {
53+
parts.push(`Candidate not played during Existing ${data.duringReferences.join(' or ')}`);
54+
} else {
55+
parts.push(`Candidate played during tracked listening range from Existing "${data.range.type}" ${data.range.timestamps[0].format('HH:mm:ssZ')} => ${data.range.timestamps[1].format('HH:mm:ssZ')}`);
5456
}
5557
return parts.join(' | ');
5658
}
@@ -59,14 +61,11 @@ export interface TemporalPlayComparisonOptions {
5961
diffThreshold?: number,
6062
fuzzyDuration?: boolean,
6163
fuzzyDiffThreshold?: number
62-
useListRanges?: boolean
64+
duringReferences?: AcceptableTemporalDuringReference
6365
}
6466

6567
export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: PlayObject, options: TemporalPlayComparisonOptions = {}): TemporalPlayComparison => {
6668

67-
const result: TemporalPlayComparison = {
68-
match: TA_NONE
69-
};
7069

7170
const {
7271
meta: {
@@ -103,9 +102,14 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
103102
diffThreshold = lowGranularitySources.some(x => x.toLocaleLowerCase() === source) ? 60 : 10,
104103
fuzzyDuration = false,
105104
fuzzyDiffThreshold = 10,
106-
useListRanges = true,
105+
duringReferences = ['range']
107106
} = options;
108107

108+
const result: TemporalPlayComparison = {
109+
match: TA_NONE,
110+
duringReferences
111+
};
112+
109113
// cant compare!
110114
if (existingTsSOCDate === undefined || candidateTsSOCDate === undefined) {
111115
return result;
@@ -120,7 +124,8 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
120124
const scrobblePlayDiff = Math.abs(existingTsSOCDate.unix() - candidateTsSOCDate.unix());
121125
result.date = {
122126
threshold: diffThreshold,
123-
diff: scrobblePlayDiff
127+
diff: scrobblePlayDiff,
128+
fuzzyDiffThreshold
124129
};
125130

126131
if(scrobblePlayDiff <= 1) {
@@ -129,22 +134,50 @@ export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: P
129134
result.match = TA_CLOSE;
130135
}
131136

132-
if (useListRanges && existingRanges !== undefined) {
133-
// since we know when the existing track was listened to
134-
// we can check if the new track play date took place while the existing one was being listened to
135-
// which would indicate (assuming same source) the new track is a duplicate
136-
for (const range of existingRanges) {
137-
if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) {
138-
result.range = range;
139-
if(!temporalAccuracyIsAtLeast(TA_CLOSE, result.match)) {
140-
result.match = TA_CLOSE;
137+
if(result.match !== TA_NONE) {
138+
return result;
139+
}
140+
141+
if(duringReferences.length > 0) {
142+
143+
if (duringReferences.includes('range') && existingRanges !== undefined) {
144+
// since we know when the existing track was listened to
145+
// we can check if the new track play date took place while the existing one was being listened to
146+
// which would indicate (assuming same source) the new track is a duplicate
147+
for (const range of existingRanges) {
148+
if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) {
149+
result.range = {
150+
type: 'range',
151+
timestamps: [range.start.timestamp, range.end.timestamp]
152+
}
153+
result.match = TA_DURING;
154+
return result;
155+
}
156+
}
157+
}
158+
159+
if(duringReferences.includes('listenedFor') && existingPlay.data.listenedFor !== undefined) {
160+
if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's'))) {
161+
result.match = TA_DURING;
162+
result.range = {
163+
type: 'listenedFor',
164+
timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's')]
141165
}
142-
break;
166+
return result;
143167
}
144168
}
145-
if (result.range === undefined) {
146-
result.range = false;
169+
170+
if(duringReferences.includes('duration') && existingPlay.data.duration !== undefined) {
171+
if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's'))) {
172+
result.match = TA_DURING;
173+
result.range = {
174+
type: 'duration',
175+
timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's')]
176+
}
177+
return result;
178+
}
147179
}
180+
148181
}
149182

150183
// if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end
@@ -199,15 +232,7 @@ export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, seco
199232
}
200233
}
201234

202-
export const temporalAccuracyIsAtLeast = (expected: TemporalAccuracy, found: TemporalAccuracy): boolean => {
203-
if(typeof expected === 'number') {
204-
if(typeof found === 'number') {
205-
return found <= expected;
206-
}
207-
return false;
208-
}
209-
return found === false;
210-
}
235+
export const hasAcceptableTemporalAccuracy = (found: TemporalAccuracy, expected: TemporalAccuracy[] = TA_DEFAULT_ACCURACY): boolean => expected.includes(found);
211236

212237
export const temporalAccuracyToString = (acc: TemporalAccuracy): string => {
213238
switch(acc) {
@@ -217,7 +242,9 @@ export const temporalAccuracyToString = (acc: TemporalAccuracy): string => {
217242
return 'close';
218243
case 3:
219244
return 'fuzzy';
220-
case false:
245+
case 4:
246+
return 'during';
247+
case 99:
221248
return 'no correlation';
222249
}
223250
}

0 commit comments

Comments
 (0)