From 54eb4f11d725424fe30bd9ac649950cd0dac063f Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sat, 27 Dec 2025 10:36:16 +0100 Subject: [PATCH 01/53] fix(SyncProcess): Reduce size of continuation on disk Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 123 ++++++++++++++------------- src/lib/strategies/Unidirectional.ts | 122 ++++++++++++++------------ 2 files changed, 132 insertions(+), 113 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index bbde5702bc..1ec69e8ddb 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -106,31 +106,40 @@ export default class SyncProcess { } getMembersToPersist() { - return [ - // Stage 0 - 'localScanResult', - 'serverScanResult', - - // Stage 1 - 'localPlanStage1', - 'serverPlanStage1', - - // Stage 2 - 'localPlanStage2', - 'serverPlanStage2', - - // Stage 3 - 'planStage3Local', - 'planStage3Server', - 'localDonePlan', - 'serverDonePlan', - 'prelimLocalReorders', - 'prelimServerReorders', - - // Stage 4 - 'localReorders', - 'serverReorders', - ] + const members = [] + // Stage 0 + if (!this.serverPlanStage1 || !this.localPlanStage1) { + members.push('localScanResult') + members.push('serverScanResult') + } + + // Stage 1 + if (!this.serverPlanStage2 || !this.localPlanStage2) { + members.push('localPlanStage1') + members.push('serverPlanStage1') + } + + // Stage 2 + if (!this.planStage3Local || !this.planStage3Server) { + members.push('localPlanStage2') + members.push('serverPlanStage2') + } + + // Stage 3 + if (this.actionsDone < this.actionsPlanned) { + members.push('planStage3Local') + members.push('planStage3Server') + members.push('localDonePlan') + members.push('serverDonePlan') + members.push('prelimLocalReorders') + members.push('prelimServerReorders') + } + + // Stage 4 + members.push('localReorders') + members.push('serverReorders') + + return members } getMappingsInstance(): Mappings { @@ -224,7 +233,7 @@ export default class SyncProcess { Logger.log({localTreeRoot: this.localTreeRoot, serverTreeRoot: this.serverTreeRoot, cacheTreeRoot: this.cacheTreeRoot}) - if (!this.localScanResult && !this.serverScanResult) { + if (!this.localScanResult && !this.serverScanResult && !this.localPlanStage1 && !this.serverPlanStage1 && !this.localPlanStage2 && !this.serverPlanStage2 && !this.planStage3Local && !this.planStage3Server) { const { localScanResult, serverScanResult } = await this.getDiffs() Logger.log({ localScanResult, serverScanResult }) this.localScanResult = localScanResult @@ -241,21 +250,14 @@ export default class SyncProcess { throw new CancelledSyncError() } - if (!this.serverPlanStage1) { + if (!this.serverPlanStage1 && !this.localPlanStage1 && !this.serverPlanStage2 && !this.localPlanStage2 && !this.planStage3Local && !this.planStage3Server) { this.serverPlanStage1 = await this.reconcileDiffs(this.localScanResult, this.serverScanResult, ItemLocation.SERVER) - } - - if (this.canceled) { - throw new CancelledSyncError() - } - - if (!this.localPlanStage1) { this.localPlanStage1 = await this.reconcileDiffs(this.serverScanResult, this.localScanResult, ItemLocation.LOCAL) } let mappingsSnapshot: MappingSnapshot - if (!this.serverPlanStage2) { + if (!this.serverPlanStage2 && !this.localPlanStage2 && !this.planStage3Local && !this.planStage3Server) { // have to get snapshot after reconciliation, because of concurrent creation reconciliation mappingsSnapshot = this.mappings.getSnapshot() Logger.log('Mapping server plan') @@ -267,15 +269,7 @@ export default class SyncProcess { REMOVE: this.serverPlanStage1.REMOVE.map(mappingsSnapshot, ItemLocation.SERVER), REORDER: this.serverPlanStage1.REORDER, } - } - if (this.canceled) { - throw new CancelledSyncError() - } - - if (!this.localPlanStage2) { - // have to get snapshot after reconciliation, because of concurrent creation reconciliation - if (!mappingsSnapshot) mappingsSnapshot = this.mappings.getSnapshot() Logger.log('Mapping local plan') this.localPlanStage2 = { @@ -293,10 +287,15 @@ export default class SyncProcess { Logger.log({localPlan: this.localPlanStage2, serverPlan: this.serverPlanStage2}) - this.applyDeletionFailsafe(ItemLocation.LOCAL, this.localTreeRoot, this.localPlanStage2.REMOVE) - this.applyAdditionFailsafe(ItemLocation.LOCAL, this.localTreeRoot, this.localPlanStage2.CREATE) - this.applyDeletionFailsafe(ItemLocation.SERVER, this.serverTreeRoot, this.serverPlanStage2.REMOVE) - this.applyAdditionFailsafe(ItemLocation.SERVER, this.serverTreeRoot, this.serverPlanStage2.CREATE) + if (this.serverPlanStage2) { + this.applyDeletionFailsafe(ItemLocation.SERVER, this.serverTreeRoot, this.serverPlanStage2.REMOVE) + this.applyAdditionFailsafe(ItemLocation.SERVER, this.serverTreeRoot, this.serverPlanStage2.CREATE) + } + + if (this.localPlanStage2) { + this.applyDeletionFailsafe(ItemLocation.LOCAL, this.localTreeRoot, this.localPlanStage2.REMOVE) + this.applyAdditionFailsafe(ItemLocation.LOCAL, this.localTreeRoot, this.localPlanStage2.CREATE) + } if (!this.localDonePlan) { this.localDonePlan = { @@ -316,18 +315,20 @@ export default class SyncProcess { } } - if (!this.prelimLocalReorders) { + if (!this.prelimLocalReorders && this.localPlanStage2) { this.prelimLocalReorders = this.localPlanStage2.REORDER this.prelimServerReorders = this.serverPlanStage2.REORDER } if (!this.actionsPlanned) { - this.actionsPlanned = Object.values(this.serverPlanStage2).reduce((acc, diff) => diff.getActions().length + acc, 0) + - Object.values(this.localPlanStage2).reduce((acc, diff) => diff.getActions().length + acc, 0) + this.actionsPlanned = Object.values(this.serverPlanStage2 || this.planStage3Server).reduce((acc, diff) => diff.getActions().length + acc, 0) + + Object.values(this.localPlanStage2 || this.planStage3Local).reduce((acc, diff) => diff.getActions().length + acc, 0) } - Logger.log('Executing server stage2 plan') - await this.executeStage2(this.server, this.serverPlanStage2, ItemLocation.SERVER, this.serverDonePlan, this.prelimServerReorders) + if (this.serverPlanStage2) { + Logger.log('Executing server stage2 plan') + await this.executeStage2(this.server, this.serverPlanStage2, ItemLocation.SERVER, this.serverDonePlan, this.prelimServerReorders) + } if (!this.planStage3Server) { if (this.canceled) { @@ -349,15 +350,19 @@ export default class SyncProcess { } } - Logger.log('Executing local stage 3 plan') - await this.executeStage3(this.server, this.planStage3Server, ItemLocation.SERVER, this.serverDonePlan) + if (this.planStage3Server) { + Logger.log('Executing server stage 3 plan') + await this.executeStage3(this.server, this.planStage3Server, ItemLocation.SERVER, this.serverDonePlan) + } if (this.canceled) { throw new CancelledSyncError() } - Logger.log('Executing local stage 2 plan') - await this.executeStage2(this.localTree, this.localPlanStage2, ItemLocation.LOCAL, this.localDonePlan, this.prelimLocalReorders) + if (this.localPlanStage2) { + Logger.log('Executing local stage 2 plan') + await this.executeStage2(this.localTree, this.localPlanStage2, ItemLocation.LOCAL, this.localDonePlan, this.prelimLocalReorders) + } if (!this.planStage3Local) { if (this.canceled) { @@ -379,8 +384,10 @@ export default class SyncProcess { } } - Logger.log('Executing local stage 3 plan') - await this.executeStage3(this.localTree, this.planStage3Local, ItemLocation.LOCAL, this.localDonePlan) + if (this.planStage3Local) { + Logger.log('Executing local stage 3 plan') + await this.executeStage3(this.localTree, this.planStage3Local, ItemLocation.LOCAL, this.localDonePlan) + } // Remove mappings only after both plans have been executed await Parallel.map(this.localDonePlan.REMOVE.getActions(), async(action) => diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 136121cce3..0734754050 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -32,20 +32,24 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { } getMembersToPersist() { - return [ - // Stage 0 - 'localScanResult', - 'serverScanResult', + const members = [] + // Stage 0 + if (!this.revertPlan) { + members.push('localScanResult') + members.push('serverScanResult') + } - // Stage 1 - 'revertPlan', - 'revertDonePlan', + // Stage 1 + if (this.actionsDone < this.actionsPlanned) { + members.push('revertPlan') + members.push('revertDonePlan') + } - // Stage 2 - 'revertReorders', + // Stage 2 + members.push('revertReorders') - 'direction', - ] + members.push('direction') + return members } async getDiffs(): Promise<{ @@ -160,7 +164,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { cacheTreeRoot: this.cacheTreeRoot, }) - if (!this.localScanResult && !this.serverScanResult) { + if (!this.localScanResult && !this.serverScanResult && !this.revertPlan) { const { localScanResult, serverScanResult } = await this.getDiffs() Logger.log({ localScanResult, serverScanResult }) this.localScanResult = localScanResult @@ -200,56 +204,64 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { throw new CancelledSyncError() } - this.actionsPlanned = Object.values(this.revertPlan).reduce( - (acc, diff) => diff.getActions().length + acc, - 0 - ) - - if (this.direction === ItemLocation.LOCAL) { - this.applyDeletionFailsafe( - ItemLocation.LOCAL, - this.localTreeRoot, - this.revertPlan.REMOVE - ) - this.applyAdditionFailsafe( - ItemLocation.LOCAL, - this.localTreeRoot, - this.revertPlan.CREATE - ) - } else { - this.applyDeletionFailsafe( - ItemLocation.SERVER, - this.serverTreeRoot, - this.revertPlan.REMOVE - ) - this.applyAdditionFailsafe( - ItemLocation.SERVER, - this.serverTreeRoot, - this.revertPlan.CREATE + if (!this.actionsPlanned && this.revertPlan) { + this.actionsPlanned = Object.values(this.revertPlan).reduce( + (acc, diff) => diff.getActions().length + acc, + 0 ) } + if (this.revertPlan) { + if (this.direction === ItemLocation.LOCAL) { + this.applyDeletionFailsafe( + ItemLocation.LOCAL, + this.localTreeRoot, + this.revertPlan.REMOVE + ) + this.applyAdditionFailsafe( + ItemLocation.LOCAL, + this.localTreeRoot, + this.revertPlan.CREATE + ) + } else { + this.applyDeletionFailsafe( + ItemLocation.SERVER, + this.serverTreeRoot, + this.revertPlan.REMOVE + ) + this.applyAdditionFailsafe( + ItemLocation.SERVER, + this.serverTreeRoot, + this.revertPlan.CREATE + ) + } + } + if (this.canceled) { throw new CancelledSyncError() } - Logger.log('Executing ' + this.direction + ' revert plan') + if (this.revertPlan) { + Logger.log('Executing ' + this.direction + ' revert plan') - this.revertDonePlan = { - CREATE: new Diff(), - UPDATE: new Diff(), - MOVE: new Diff(), - REMOVE: new Diff(), - REORDER: new Diff(), - } + if (!this.revertDonePlan) { + this.revertDonePlan = { + CREATE: new Diff(), + UPDATE: new Diff(), + MOVE: new Diff(), + REMOVE: new Diff(), + REORDER: new Diff(), + } + } - await this.executeRevert( - target, - this.revertPlan, - this.direction, - this.revertDonePlan, - sourceScanResult.REORDER - ) + await this.executeRevert( + target, + this.revertPlan, + this.direction, + this.revertDonePlan, + sourceScanResult.REORDER + ) + } if (this.direction === ItemLocation.LOCAL) { this.revertDonePlan.REMOVE.getActions().forEach((action) => @@ -261,7 +273,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) } - if ('orderFolder' in this.server && !this.revertReorders) { + if ('orderFolder' in target && !this.revertReorders) { const mappingsSnapshot = this.mappings.getSnapshot() Logger.log('Mapping reorderings') this.revertReorders = sourceScanResult.REORDER.map( @@ -270,7 +282,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) } - if ('orderFolder' in this.server && 'orderFolder' in target) { + if (this.revertReorders && 'orderFolder' in target) { await this.executeReorderings(target, this.revertReorders) } From 658ef86900e5f732d7807fbc08c0da080244aeb6 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sat, 27 Dec 2025 11:19:28 +0100 Subject: [PATCH 02/53] fix(SyncProcess): Fix getmembersToPersist Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 14 +++++++++++--- src/lib/strategies/Unidirectional.ts | 12 ++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index 1ec69e8ddb..a35f1808d9 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -108,19 +108,27 @@ export default class SyncProcess { getMembersToPersist() { const members = [] // Stage 0 - if (!this.serverPlanStage1 || !this.localPlanStage1) { + if ( + (!this.serverPlanStage1 || !this.localPlanStage1) && + (!this.serverPlanStage2 || !this.localPlanStage2) && + (!this.planStage3Local || !this.planStage3Server) && + this.actionsPlanned === 0 + ) { members.push('localScanResult') members.push('serverScanResult') } // Stage 1 - if (!this.serverPlanStage2 || !this.localPlanStage2) { + if ( + (!this.serverPlanStage2 || !this.localPlanStage2) && + (!this.planStage3Local || !this.planStage3Server) && this.actionsPlanned === 0 + ) { members.push('localPlanStage1') members.push('serverPlanStage1') } // Stage 2 - if (!this.planStage3Local || !this.planStage3Server) { + if ((!this.planStage3Local || !this.planStage3Server) && this.actionsPlanned === 0) { members.push('localPlanStage2') members.push('serverPlanStage2') } diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 0734754050..9ea59495b0 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -34,7 +34,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { getMembersToPersist() { const members = [] // Stage 0 - if (!this.revertPlan) { + if (!this.revertPlan && this.actionsPlanned === 0) { members.push('localScanResult') members.push('serverScanResult') } @@ -308,7 +308,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { await Parallel.each( sourceScanResult.CREATE.getActions(), - async (action) => { + async(action) => { // recreate it on slave resource otherwise const payload = await this.translateCompleteItem( action.payload, @@ -331,7 +331,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { await Parallel.each( targetScanResult.CREATE.getActions(), - async (action) => { + async(action) => { slavePlan.REMOVE.commit({ ...action, type: ActionType.REMOVE }) }, ACTION_CONCURRENCY @@ -339,7 +339,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { await Parallel.each( targetScanResult.UPDATE.getActions(), - async (action) => { + async(action) => { const payload = action.oldItem.cloneWithLocation( false, action.payload.location @@ -360,7 +360,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { await Parallel.each( targetScanResult.MOVE.getActions(), - async (action) => { + async(action) => { const payload = action.payload.cloneWithLocation( false, action.oldItem.location @@ -389,7 +389,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) if (newItem instanceof Folder) { const nonexistingItems = [] - await newItem.traverse(async (child, parentFolder) => { + await newItem.traverse(async(child, parentFolder) => { child.id = Mappings.mapId(mappingsSnapshot, child, fakeLocation) if (typeof child.id === 'undefined') { nonexistingItems.push(child) From 3e1b4b912374216308b8ae8e6d4a8fa68a0459e5 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 10:24:25 +0100 Subject: [PATCH 03/53] fix(index): Update index incrementally instead of recreating it every time Signed-off-by: Marcel Klehr --- src/lib/CachingTreeWrapper.ts | 6 ++- src/lib/Tree.ts | 65 +++++++++++++++++--------- src/lib/adapters/Caching.ts | 17 ++++--- src/lib/adapters/NextcloudBookmarks.ts | 37 +++++++++++---- 4 files changed, 84 insertions(+), 41 deletions(-) diff --git a/src/lib/CachingTreeWrapper.ts b/src/lib/CachingTreeWrapper.ts index f787dc255c..4022b60fe4 100644 --- a/src/lib/CachingTreeWrapper.ts +++ b/src/lib/CachingTreeWrapper.ts @@ -26,9 +26,10 @@ export default class CachingTreeWrapper implements OrderFolderResource { public location: L public isRoot = false private hashValue: Record + public index: IItemIndex constructor({ id, @@ -194,8 +195,9 @@ export class Bookmark { return result } - createIndex(): any { - return { [this.id]: this } + createIndex(): IItemIndex { + this.index = { bookmark: {[this.id]: this}, folder: {} } + return this.index } // TODO: Make this return the correct type based on the type param @@ -258,7 +260,7 @@ export class Folder { public isRoot = false public loaded = true public location: L - private index: IItemIndex + public index: IItemIndex constructor({ id, @@ -363,7 +365,7 @@ export class Folder { ): Promise { await Parallel.each( this.children, - async (item) => { + async(item) => { await fn(item, this) if (item.type === 'folder') { // give the browser time to breathe @@ -560,24 +562,43 @@ export class Folder { createIndex(): IItemIndex { this.index = { folder: { [this.id]: this }, - bookmark: this.children - .filter((child) => child instanceof Bookmark) - .reduce((obj, child) => { - obj[child.id] = child - return obj - }, {}), + bookmark: {} } - this.children - .filter((child) => child instanceof Folder) - .map((child) => child.createIndex()) - .forEach((subIndex) => { + for (const child of this.children) { + if (child instanceof Bookmark) { + this.index.bookmark[child.id] = child + } else if (child instanceof Folder) { + const subIndex = child.createIndex() Object.assign(this.index.folder, subIndex.folder) Object.assign(this.index.bookmark, subIndex.bookmark) - }) + } + } + return this.index } + updateIndex(item: TItem) { + const itemIndex = item.index || item.createIndex() + let currentItem = item + while (currentItem) { + Object.assign(currentItem.index.folder, itemIndex.folder) + Object.assign(currentItem.index.bookmark, itemIndex.bookmark) + currentItem = this.index.folder[currentItem.parentId] + } + } + + removeFromIndex(item: TItem) { + if (!item) return + if (item.parentId) { + let parentFolder = this.index.folder[item.parentId] + while (parentFolder) { + delete parentFolder.index[item.type][item.id] + parentFolder = this.index.folder[parentFolder.parentId] + } + } + } + inspect(depth = 0): string { return ( Array(depth < 0 ? 0 : depth) @@ -618,13 +639,13 @@ export class Folder { ...obj, children: obj.children ? obj.children.map((child) => { - // Firefox seems to set 'url' even for folders - if ('url' in child && typeof child.url === 'string') { - return Bookmark.hydrate(child) - } else { - return Folder.hydrate(child) - } - }) + // Firefox seems to set 'url' even for folders + if ('url' in child && typeof child.url === 'string') { + return Bookmark.hydrate(child) + } else { + return Folder.hydrate(child) + } + }) : null, }) } diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index 43bd1ffaf9..5a4ffde0af 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -80,7 +80,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource): Promise { @@ -131,7 +132,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource): Promise { @@ -142,7 +143,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource):Promise { @@ -218,7 +220,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource):Promise> { @@ -237,7 +239,8 @@ export default class CachingAdapter implements Adapter, BulkImportResource):Promise { @@ -573,7 +580,7 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes parent.children = parent.children.filter( (child) => String(child.id) !== String(id) ) - this.tree.createIndex() + this.tree.removeFromIndex(oldFolder) } } @@ -691,7 +698,7 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes this.list && this.list.push(upstreamMark) if (this.tree) { newParentFolder.children.push(upstreamMark) - this.tree.createIndex() + this.tree.updateIndex(upstreamMark) } return bm.id @@ -742,11 +749,21 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes throw e } + if (oldParentId) { + const oldParentFolder = this.tree.findFolder(oldParentId) + const oldBm = oldParentFolder.findBookmark(newBm.id) + oldParentFolder.children = oldParentFolder.children.filter( + (item) => !(item.type === 'bookmark' && item.id === newBm.id) + ) + if (oldBm && this.tree) { + this.tree.removeFromIndex(oldBm) + } + } if (!newFolder.children.find(item => String(item.id) === String(newBm.id) && item.type === 'bookmark')) { newFolder.children.push(newBm) } newBm.id = upstreamId + ';' + newBm.parentId - this.tree.createIndex() + this.tree.updateIndex(newBm) return newBm.id }) From 426a078c9e7a674d3d697832b0b60658294d0d99 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 10:26:51 +0100 Subject: [PATCH 04/53] fix(continuation): Fix continuation loading Signed-off-by: Marcel Klehr --- src/lib/Diff.ts | 301 +++++++++++++++++++++------ src/lib/strategies/Default.ts | 92 ++++---- src/lib/strategies/Unidirectional.ts | 65 +++++- 3 files changed, 333 insertions(+), 125 deletions(-) diff --git a/src/lib/Diff.ts b/src/lib/Diff.ts index 03c0cb9842..db2ebf0914 100644 --- a/src/lib/Diff.ts +++ b/src/lib/Diff.ts @@ -78,15 +78,19 @@ export type MapLocation, NewLocat ReorderAction : never -export default class Diff> { +export default class Diff< + L1 extends TItemLocation, + L2 extends TItemLocation, + A extends Action +> { private readonly actions: A[] constructor() { this.actions = [] } - clone(filter: (action:A)=>boolean = () => true): Diff { - const newDiff : Diff = new Diff + clone(filter: (action: A) => boolean = () => true): Diff { + const newDiff: Diff = new Diff() this.getActions().forEach((action: A) => { if (filter(action)) { newDiff.commit(action) @@ -96,33 +100,55 @@ export default class Diff, item2: TItem, itemTree: Folder, cache: Record): boolean { - const cacheKey = 'contains:' + Mappings.mapId(mappingsSnapshot, item2, ItemLocation.LOCAL) + ':' + Mappings.mapId(mappingsSnapshot, item2, ItemLocation.SERVER) + - '-' + Mappings.mapId(mappingsSnapshot, item1, ItemLocation.LOCAL) + ':' + Mappings.mapId(mappingsSnapshot, item1, ItemLocation.SERVER) + static containsParent( + mappingsSnapshot: MappingSnapshot, + item1: TItem, + item2: TItem, + itemTree: Folder, + cache: Record + ): boolean { + const cacheKey = + 'contains:' + + Mappings.mapId(mappingsSnapshot, item2, ItemLocation.LOCAL) + + ':' + + Mappings.mapId(mappingsSnapshot, item2, ItemLocation.SERVER) + + '-' + + Mappings.mapId(mappingsSnapshot, item1, ItemLocation.LOCAL) + + ':' + + Mappings.mapId(mappingsSnapshot, item1, ItemLocation.SERVER) if (typeof cache[cacheKey] !== 'undefined') { return cache[cacheKey] } - const item1InTree = itemTree.findItem(item1.type, Mappings.mapId(mappingsSnapshot, item1, itemTree.location)) + const item1InTree = itemTree.findItem( + item1.type, + Mappings.mapId(mappingsSnapshot, item1, itemTree.location) + ) if ( - item1.findItem(ItemType.FOLDER, - Mappings.mapParentId(mappingsSnapshot, item2, item1.location)) || - (item1InTree && item1InTree.findItem(ItemType.FOLDER, Mappings.mapParentId(mappingsSnapshot, item2, itemTree.location))) || - String(Mappings.mapId(mappingsSnapshot, item1, item2.location)) === String(item2.parentId) || - String(Mappings.mapParentId(mappingsSnapshot, item2, item1.location)) === String(item1.id) + item1.findItem( + ItemType.FOLDER, + Mappings.mapParentId(mappingsSnapshot, item2, item1.location) + ) || + (item1InTree && + item1InTree.findItem( + ItemType.FOLDER, + Mappings.mapParentId(mappingsSnapshot, item2, itemTree.location) + )) || + String(Mappings.mapId(mappingsSnapshot, item1, item2.location)) === + String(item2.parentId) || + String(Mappings.mapParentId(mappingsSnapshot, item2, item1.location)) === + String(item1.id) ) { cache[cacheKey] = true return true @@ -140,24 +166,66 @@ export default class Diff = {}, chain: Action[] = [] ): boolean { - const currentItemLocalId = Mappings.mapId(mappingsSnapshot, currentItem, ItemLocation.LOCAL) - const currentItemServerId = Mappings.mapId(mappingsSnapshot, currentItem, ItemLocation.SERVER) - const targetPayloadLocalId = Mappings.mapId(mappingsSnapshot, targetAction.payload, ItemLocation.LOCAL) - const targetPayloadServerId = Mappings.mapId(mappingsSnapshot, targetAction.payload, ItemLocation.SERVER) + const currentItemLocalId = Mappings.mapId( + mappingsSnapshot, + currentItem, + ItemLocation.LOCAL + ) + const currentItemServerId = Mappings.mapId( + mappingsSnapshot, + currentItem, + ItemLocation.SERVER + ) + const targetPayloadLocalId = Mappings.mapId( + mappingsSnapshot, + targetAction.payload, + ItemLocation.LOCAL + ) + const targetPayloadServerId = Mappings.mapId( + mappingsSnapshot, + targetAction.payload, + ItemLocation.SERVER + ) const cacheKey = `hasChain:${currentItemLocalId}:${currentItemServerId}-${targetPayloadLocalId}:${targetPayloadServerId}` if (typeof cache[cacheKey] !== 'undefined') { return cache[cacheKey] } - if (Diff.containsParent(mappingsSnapshot, targetAction.payload, currentItem, itemTree, cache)) { + if ( + Diff.containsParent( + mappingsSnapshot, + targetAction.payload, + currentItem, + itemTree, + cache + ) + ) { cache[cacheKey] = true return true } - const newCurrentActions = actions.filter(newTargetAction => - !chain.includes(newTargetAction) && Diff.containsParent(mappingsSnapshot, newTargetAction.payload, currentItem, itemTree, cache) + const newCurrentActions = actions.filter( + (newTargetAction) => + !chain.includes(newTargetAction) && + Diff.containsParent( + mappingsSnapshot, + newTargetAction.payload, + currentItem, + itemTree, + cache + ) ) if (newCurrentActions.length) { for (const newCurrentAction of newCurrentActions) { - if (Diff.findChain(mappingsSnapshot, actions, itemTree, newCurrentAction.payload, targetAction, cache,[...chain, newCurrentAction])) { + if ( + Diff.findChain( + mappingsSnapshot, + actions, + itemTree, + newCurrentAction.payload, + targetAction, + cache, + [...chain, newCurrentAction] + ) + ) { return true } } @@ -166,27 +234,44 @@ export default class Diff(actions: MoveAction[], tree: Folder) :MoveAction[][] { - const bookmarks = actions.filter(a => a.payload.type === ItemType.BOOKMARK) - const folderMoves = actions.filter(a => a.payload.type === ItemType.FOLDER) - const DAG = folderMoves - .reduce((DAG, action1) => { - DAG[action1.payload.id] = folderMoves.filter(action2 => { - if (action1 === action2 || String(action1.payload.id) === String(action2.payload.id)) { + static sortMoves( + actions: MoveAction[], + tree: Folder + ): MoveAction[][] { + const bookmarks = actions.filter( + (a) => a.payload.type === ItemType.BOOKMARK + ) + const folderMoves = actions.filter( + (a) => a.payload.type === ItemType.FOLDER + ) + const DAG = folderMoves.reduce((DAG, action1) => { + DAG[action1.payload.id] = folderMoves + .filter((action2) => { + if ( + action1 === action2 || + String(action1.payload.id) === String(action2.payload.id) + ) { return false } return ( - (tree.findItem(action1.payload.type, action1.payload.id) && tree.findItem(action1.payload.type, action1.payload.id).findItem(action2.payload.type, action2.payload.id)) + tree.findItem(action1.payload.type, action1.payload.id) && + tree + .findItem(action1.payload.type, action1.payload.id) + .findItem(action2.payload.type, action2.payload.id) ) }) - .map(a => a.payload.id) - return DAG - }, {}) + .map((a) => a.payload.id) + return DAG + }, {}) let batches try { - batches = batchingToposort(DAG).map(batch => batch.map(id => folderMoves.find(a => String(a.payload.id) === String(id)))) + batches = batchingToposort(DAG).map((batch) => + batch.map((id) => + folderMoves.find((a) => String(a.payload.id) === String(id)) + ) + ) } catch (e) { - console.log({DAG, tree, actions}) + console.log({ DAG, tree, actions }) throw e } batches.push(bookmarks) @@ -202,13 +287,18 @@ export default class Diff(mappingsSnapshot:MappingSnapshot, targetLocation: L3, filter: (action: A)=>boolean = () => true, skipErroneousActions = false): Diff> { - const newDiff : Diff> = new Diff + map( + mappingsSnapshot: MappingSnapshot, + targetLocation: L3, + filter: (action: A) => boolean = () => true, + skipErroneousActions = false + ): Diff> { + const newDiff: Diff> = new Diff() // Map payloads this.getActions() - .map(a => a as A) - .forEach(action => { + .map((a) => a as A) + .forEach((action) => { let newAction if (!filter(action)) { @@ -227,7 +317,10 @@ export default class Diff { - return {...item, id: mappingsSnapshot[(targetLocation === ItemLocation.LOCAL ? ItemLocation.SERVER : ItemLocation.LOCAL) + 'To' + targetLocation][item.type][item.id]} + newAction.order = action.order.map((item) => { + return { + ...item, + id: mappingsSnapshot[ + (targetLocation === ItemLocation.LOCAL + ? ItemLocation.SERVER + : ItemLocation.LOCAL) + + 'To' + + targetLocation + ][item.type][item.id], + } }) } @@ -286,24 +412,48 @@ export default class Diff { - await yieldToEventLoop() - return { - ...action, - payload: await action.payload.clone(false).toJSONAsync(), - oldItem: await action.oldItem && action.oldItem.clone(false).toJSONAsync(), - } - }, 1) + return Parallel.map( + this.getActions(), + async(action: A) => { + await yieldToEventLoop() + return { + ...action, + payload: await action.payload.clone(false).toJSONAsync(), + oldItem: + (await action.oldItem) && action.oldItem.clone(false).toJSONAsync(), + } + }, + 1 + ) } - inspect(depth = 0):string { - return 'Diff\n' + this.getActions().map((action: A) => { - return `\nAction: ${action.type}\nPayload: #${action.payload.id}[${action.payload.title}]${'url' in action.payload ? `(${action.payload.url})` : ''} parentId: ${action.payload.parentId} ${'index' in action ? `Index: ${action.index}\n` : ''}${'order' in action ? `Order: ${JSON.stringify(action.order, null, '\t')}` : ''}` - }).join('\n') + inspect(depth = 0): string { + return ( + 'Diff\n' + + this.getActions() + .map((action: A) => { + return `\nAction: ${action.type}\nPayload: #${action.payload.id}[${ + action.payload.title + }]${ + 'url' in action.payload ? `(${action.payload.url})` : '' + } parentId: ${action.payload.parentId} ${ + 'index' in action ? `Index: ${action.index}\n` : '' + }${ + 'order' in action + ? `Order: ${JSON.stringify(action.order, null, '\t')}` + : '' + }` + }) + .join('\n') + ) } - static fromJSON>(json) { - const diff: Diff = new Diff + static fromJSON< + L1 extends TItemLocation, + L2 extends TItemLocation, + A2 extends Action + >(json) { + const diff: Diff = new Diff() json.forEach((action: A2): void => { action.payload = hydrate(action.payload) action.oldItem = action.oldItem && hydrate(action.oldItem) @@ -311,6 +461,21 @@ export default class Diff + >(json) { + const diff: Diff = new Diff() + await Parallel.map(json, async(action: A2): Promise => { + await yieldToEventLoop() + action.payload = hydrate(action.payload) + action.oldItem = action.oldItem && hydrate(action.oldItem) + diff.commit(action) + }, 1) + return diff + } } export interface PlanStage1 { diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index a35f1808d9..e30ed2e607 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -54,24 +54,24 @@ export default class SyncProcess { protected serverScanResult: ScanResult = null // Stage 1 - private localPlanStage1: PlanStage1 - private serverPlanStage1: PlanStage1 + private localPlanStage1: PlanStage1 = null + private serverPlanStage1: PlanStage1 = null // Stage 2 - private localPlanStage2: PlanStage2 + private localPlanStage2: PlanStage2 = null private serverPlanStage2: PlanStage2 // Stage 3 - private planStage3Local: PlanStage3 - private planStage3Server: PlanStage3 - private localDonePlan: PlanStage3 - private serverDonePlan: PlanStage3 - private prelimLocalReorders: Diff> - private prelimServerReorders: Diff> + private planStage3Local: PlanStage3 = null + private planStage3Server: PlanStage3 = null + private localDonePlan: PlanStage3 = null + private serverDonePlan: PlanStage3 = null + private prelimLocalReorders: Diff> = null + private prelimServerReorders: Diff> = null // Stage 4 - private localReorders: Diff> - private serverReorders: Diff> + private localReorders: Diff> = null + private serverReorders: Diff> = null protected actionsDone = 0 protected actionsPlanned = 0 @@ -194,20 +194,35 @@ export default class SyncProcess { Logger.log(`Executed ${this.actionsDone} actions from ${this.actionsPlanned} actions`) } - setProgress({actionsDone, actionsPlanned}: {actionsDone: number, actionsPlanned: number}) { + async setProgress(json: any) { + const {actionsDone, actionsPlanned} = json this.actionsDone = actionsDone this.actionsPlanned = actionsPlanned - this.throttledProgressCb( - Math.min( - 1, - 0.5 + (this.actionsDone / (this.actionsPlanned + 1)) * 0.5 - ), - this.actionsDone - ).catch((er) => { - if (er instanceof CanceledError) { - return + if (json.serverTreeRoot) { + this.serverTreeRoot = Folder.hydrate(json.serverTreeRoot) + } + if (json.localTreeRoot) { + this.localTreeRoot = Folder.hydrate(json.localTreeRoot) + } + if (json.cacheTreeRoot) { + this.cacheTreeRoot = Folder.hydrate(json.cacheTreeRoot) + } + Object.keys(json).forEach((member) => { + if (member in json) { + if (member.toLowerCase().includes('scanresult') || member.toLowerCase().includes('plan')) { + this[member] = { + CREATE: Diff.fromJSON(json[member].CREATE), + UPDATE: Diff.fromJSON(json[member].UPDATE), + MOVE: Diff.fromJSON(json[member].MOVE), + REMOVE: Diff.fromJSON(json[member].REMOVE), + REORDER: Diff.fromJSON(json[member].REORDER), + } + } else if (member.toLowerCase().includes('reorders')) { + this[member] = Diff.fromJSON(json[member]) + } else { + this[member] = json[member] + } } - throw er }) } @@ -1565,10 +1580,10 @@ export default class SyncProcess { Object.entries(this) .filter(([key]) => membersToPersist.includes(key)), async([key, value]) => { - if ('toJSONAsync' in value) { + if (value.toJSONAsync) { return [key, await value.toJSONAsync()] } - if ('toJSON' in value) { + if (value.toJSON) { await yieldToEventLoop() return [key, value.toJSON()] } @@ -1602,34 +1617,7 @@ export default class SyncProcess { default: throw new Error('Unknown strategy: ' + json.strategy) } - strategy.setProgress(json) - if (json.serverTreeRoot) { - strategy.serverTreeRoot = Folder.hydrate(json.serverTreeRoot) - } - if (json.localTreeRoot) { - strategy.localTreeRoot = Folder.hydrate(json.localTreeRoot) - } - if (json.cacheTreeRoot) { - strategy.cacheTreeRoot = Folder.hydrate(json.cacheTreeRoot) - } - strategy.getMembersToPersist().forEach((member) => { - if (member in json) { - if (member.toLowerCase().includes('scanresult') || member.toLowerCase().includes('plan')) { - strategy[member] = { - CREATE: Diff.fromJSON(json[member].CREATE), - UPDATE: Diff.fromJSON(json[member].UPDATE), - MOVE: Diff.fromJSON(json[member].MOVE), - REMOVE: Diff.fromJSON(json[member].REMOVE), - REORDER: Diff.fromJSON(json[member].REORDER), - } - } else if (member.toLowerCase().includes('reorders')) { - strategy[member] = Diff.fromJSON(json[member]) - } else { - strategy[member] = json[member] - } - } - }) - + await strategy.setProgress(json) return strategy } } diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 9ea59495b0..e16e66750e 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -16,16 +16,16 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { protected revertPlan: PlanStage1< TItemLocation, TOppositeLocation - > + > = null protected revertDonePlan: PlanRevert< TItemLocation, TOppositeLocation - > + > = null protected revertReorders: Diff< TItemLocation, TOppositeLocation, ReorderAction> - > + > = null setDirection(direction: TItemLocation): void { this.direction = direction @@ -52,6 +52,61 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { return members } + async setProgress(json: any) { + const {actionsDone, actionsPlanned} = json + this.actionsDone = actionsDone + this.actionsPlanned = actionsPlanned + if (json.serverTreeRoot) { + this.serverTreeRoot = Folder.hydrate(json.serverTreeRoot) + } + if (json.localTreeRoot) { + this.localTreeRoot = Folder.hydrate(json.localTreeRoot) + } + if (json.cacheTreeRoot) { + this.cacheTreeRoot = Folder.hydrate(json.cacheTreeRoot) + } + if (json.localScanResult) { + this.localScanResult = { + CREATE: await Diff.fromJSONAsync(json.localScanResult.CREATE), + UPDATE: await Diff.fromJSONAsync(json.localScanResult.UPDATE), + MOVE: await Diff.fromJSONAsync(json.localScanResult.MOVE), + REMOVE: await Diff.fromJSONAsync(json.localScanResult.REMOVE), + REORDER: await Diff.fromJSONAsync(json.localScanResult.REORDER), + } + } + if (json.serverScanResult) { + this.serverScanResult = { + CREATE: await Diff.fromJSONAsync(json.serverScanResult.CREATE), + UPDATE: await Diff.fromJSONAsync(json.serverScanResult.UPDATE), + MOVE: await Diff.fromJSONAsync(json.serverScanResult.MOVE), + REMOVE: await Diff.fromJSONAsync(json.serverScanResult.REMOVE), + REORDER: await Diff.fromJSONAsync(json.serverScanResult.REORDER), + } + } + if (json.revertPlan) { + this.revertPlan = { + CREATE: await Diff.fromJSONAsync(json.revertPlan.CREATE), + UPDATE: await Diff.fromJSONAsync(json.revertPlan.UPDATE), + MOVE: await Diff.fromJSONAsync(json.revertPlan.MOVE), + REMOVE: await Diff.fromJSONAsync(json.revertPlan.REMOVE), + REORDER: await Diff.fromJSONAsync(json.revertPlan.REORDER), + } + } + if (json.revertDonePlan) { + this.revertDonePlan = { + CREATE: await Diff.fromJSONAsync(json.revertDonePlan.CREATE), + UPDATE: await Diff.fromJSONAsync(json.revertDonePlan.UPDATE), + MOVE: await Diff.fromJSONAsync(json.revertDonePlan.MOVE), + REMOVE: await Diff.fromJSONAsync(json.revertDonePlan.REMOVE), + REORDER: await Diff.fromJSONAsync(json.revertDonePlan.REORDER), + } + } + if (json.revertReorders) { + this.revertReorders = await Diff.fromJSONAsync(json.revertReorders) + } + this.direction = json.direction + } + async getDiffs(): Promise<{ localScanResult: ScanResult serverScanResult: ScanResult @@ -164,7 +219,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { cacheTreeRoot: this.cacheTreeRoot, }) - if (!this.localScanResult && !this.serverScanResult && !this.revertPlan) { + if ((!this.localScanResult || !this.serverScanResult) && !this.revertPlan) { const { localScanResult, serverScanResult } = await this.getDiffs() Logger.log({ localScanResult, serverScanResult }) this.localScanResult = localScanResult @@ -529,7 +584,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { async toJSONAsync(): Promise { return { - ...(await DefaultSyncProcess.prototype.toJSON.apply(this)), + ...(await DefaultSyncProcess.prototype.toJSONAsync.apply(this)), strategy: 'unidirectional', } } From 4a5d598669aaf6c6045f0c78bb2aeffc1779b1e9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 14:14:44 +0100 Subject: [PATCH 05/53] fix(Tree): Fix updateIndex and removeFromIndex don't throw if index doesn't exist Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index ad25aaa98c..331e6373c9 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -578,7 +578,14 @@ export class Folder { return this.index } + /** + * Update the index with the given item (this method should be called on the root folder) + */ updateIndex(item: TItem) { + if (!this.index) { + this.createIndex() + return + } const itemIndex = item.index || item.createIndex() let currentItem = item while (currentItem) { @@ -588,8 +595,15 @@ export class Folder { } } + /** + * Update the index by removing the given item (this method should be called on the root folder) + */ removeFromIndex(item: TItem) { if (!item) return + if (!this.index) { + this.createIndex() + return + } if (item.parentId) { let parentFolder = this.index.folder[item.parentId] while (parentFolder) { From 119f3a96344078cc2051f93462ac884598fd7325 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 14:29:53 +0100 Subject: [PATCH 06/53] fix(Tree): Improve updateIndex Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 331e6373c9..27374b0773 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -587,7 +587,7 @@ export class Folder { return } const itemIndex = item.index || item.createIndex() - let currentItem = item + let currentItem = this.index.folder[item.parentId] while (currentItem) { Object.assign(currentItem.index.folder, itemIndex.folder) Object.assign(currentItem.index.bookmark, itemIndex.bookmark) From 6914962640f06786a8cc78ee81c2b67042bc2535 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 14:30:42 +0100 Subject: [PATCH 07/53] fix(CachingTreeWrapper): Make sure item index is up to date for updateIndex Signed-off-by: Marcel Klehr --- src/lib/CachingTreeWrapper.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/CachingTreeWrapper.ts b/src/lib/CachingTreeWrapper.ts index 4022b60fe4..6b0ed32f2c 100644 --- a/src/lib/CachingTreeWrapper.ts +++ b/src/lib/CachingTreeWrapper.ts @@ -29,6 +29,7 @@ export default class CachingTreeWrapper implements OrderFolderResource Date: Sun, 28 Dec 2025 15:08:59 +0100 Subject: [PATCH 08/53] fix(Tree): Fix incremental index updates Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 3 +++ src/lib/adapters/Caching.ts | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 27374b0773..3597eac355 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -582,6 +582,9 @@ export class Folder { * Update the index with the given item (this method should be called on the root folder) */ updateIndex(item: TItem) { + if (!item) { + return + } if (!this.index) { this.createIndex() return diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index 5a4ffde0af..fe8eca4849 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -109,9 +109,10 @@ export default class CachingAdapter implements Adapter, BulkImportResource Date: Sun, 28 Dec 2025 15:33:29 +0100 Subject: [PATCH 09/53] fix(CachingAdapter): Fix incremental index updates Signed-off-by: Marcel Klehr --- src/lib/adapters/Caching.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index fe8eca4849..e8fa47fd38 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -133,7 +133,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource): Promise { From b11a7b656b3f0377c136c400a03f57287cc4e8ef Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 16:29:14 +0100 Subject: [PATCH 10/53] fix(CachingAdapter): Fix incremental index updates Signed-off-by: Marcel Klehr --- src/lib/adapters/Caching.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index e8fa47fd38..6d60211bd4 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -113,7 +113,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource): Promise { From 545bddbae45366d609651d418b34e4bd7a57b666 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 17:20:14 +0100 Subject: [PATCH 11/53] fix(NextcloudBookmarks): Fix incremental index updates Signed-off-by: Marcel Klehr --- src/lib/adapters/NextcloudBookmarks.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/lib/adapters/NextcloudBookmarks.ts b/src/lib/adapters/NextcloudBookmarks.ts index 3cdaa0c0f4..b1183d79dc 100644 --- a/src/lib/adapters/NextcloudBookmarks.ts +++ b/src/lib/adapters/NextcloudBookmarks.ts @@ -749,20 +749,23 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes throw e } - if (oldParentId) { - const oldParentFolder = this.tree.findFolder(oldParentId) - const oldBm = oldParentFolder.findBookmark(newBm.id) - oldParentFolder.children = oldParentFolder.children.filter( - (item) => !(item.type === 'bookmark' && item.id === newBm.id) - ) - if (oldBm && this.tree) { - this.tree.removeFromIndex(oldBm) - } + const oldParentFolder = this.tree.findFolder(oldParentId) + if (!oldParentFolder) { + throw new UnknownFolderParentUpdateError() + } + const oldBm = oldParentFolder.findBookmark(newBm.id) + oldParentFolder.children = oldParentFolder.children.filter( + (item) => !(item.type === 'bookmark' && item.id === newBm.id) + ) + if (oldBm && this.tree) { + this.tree.removeFromIndex(oldBm) } + oldBm.title = newBm.title + oldBm.parentId = newBm.parentId if (!newFolder.children.find(item => String(item.id) === String(newBm.id) && item.type === 'bookmark')) { - newFolder.children.push(newBm) + newFolder.children.push(oldBm) } - newBm.id = upstreamId + ';' + newBm.parentId + oldBm.id = upstreamId + ';' + newBm.parentId this.tree.updateIndex(newBm) return newBm.id From b8b04364bf7351b57d5749e7360811ffbb38ae5b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 19:02:13 +0100 Subject: [PATCH 12/53] fix(NextcloudBookmarks): Fix incremental index updates Signed-off-by: Marcel Klehr --- src/lib/adapters/NextcloudBookmarks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/adapters/NextcloudBookmarks.ts b/src/lib/adapters/NextcloudBookmarks.ts index b1183d79dc..a0914cdee5 100644 --- a/src/lib/adapters/NextcloudBookmarks.ts +++ b/src/lib/adapters/NextcloudBookmarks.ts @@ -766,7 +766,7 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes newFolder.children.push(oldBm) } oldBm.id = upstreamId + ';' + newBm.parentId - this.tree.updateIndex(newBm) + this.tree.updateIndex(oldBm) return newBm.id }) From 9508bcd65cc48d8a269eaeb31c37d94d71642950 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 19:31:24 +0100 Subject: [PATCH 13/53] tests: Don't close about:addons page in tab tests Signed-off-by: Marcel Klehr --- src/lib/adapters/Caching.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index 6d60211bd4..a39d96dc20 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -53,13 +53,13 @@ export default class CachingAdapter implements Adapter, BulkImportResource Date: Sun, 28 Dec 2025 20:08:21 +0100 Subject: [PATCH 14/53] fix(Tree): Fix end condition in removeFromIndex Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 3597eac355..0ea6a90abd 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -609,7 +609,7 @@ export class Folder { } if (item.parentId) { let parentFolder = this.index.folder[item.parentId] - while (parentFolder) { + while (parentFolder && this.index.folder[parentFolder.parentId] !== parentFolder) { delete parentFolder.index[item.type][item.id] parentFolder = this.index.folder[parentFolder.parentId] } From e7bcdf480e1a815965c48a294d896a02fe331444 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 28 Dec 2025 20:29:15 +0100 Subject: [PATCH 15/53] fix(LocalTabs): Poperly distinguish between window IDs and group IDs (important for mappings and tree index) Signed-off-by: Marcel Klehr --- src/lib/LocalTabs.ts | 66 +++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/src/lib/LocalTabs.ts b/src/lib/LocalTabs.ts index dffb5c5252..860315da1f 100644 --- a/src/lib/LocalTabs.ts +++ b/src/lib/LocalTabs.ts @@ -15,6 +15,22 @@ export default class LocalTabs implements OrderFolderResource> { let tabs = await browser.tabs.query({ windowType: 'normal' // no devtools or panels or popups @@ -46,7 +62,7 @@ export default class LocalTabs implements OrderFolderResource - browser.tabGroups.get(bookmark.parentId) + browser.tabGroups.get(this.getTabGroupIdFromFolderId(bookmark.parentId)) ) if (tabGroup) { isTabGroup = true @@ -146,9 +162,9 @@ export default class LocalTabs implements OrderFolderResource browser.tabs.group({ tabIds: [node.id], - groupId: bookmark.parentId + groupId: this.getTabGroupIdFromFolderId(bookmark.parentId) }) ) } @@ -209,7 +225,7 @@ export default class LocalTabs implements OrderFolderResource - browser.tabGroups.get(bookmark.parentId) + browser.tabGroups.get(this.getTabGroupIdFromFolderId(bookmark.parentId)) ) isTabGroup = !!tabGroup } @@ -225,7 +241,7 @@ export default class LocalTabs implements OrderFolderResource browser.tabs.group({ tabIds: [bookmark.id], - groupId: bookmark.parentId + groupId: this.getTabGroupIdFromFolderId(bookmark.parentId) }) ) } else { @@ -240,7 +256,7 @@ export default class LocalTabs implements OrderFolderResource browser.tabs.move(bookmark.id, { - windowId: bookmark.parentId, + windowId: this.getWindowIdFromFolderId(bookmark.parentId), index: -1, // last }) ) @@ -268,7 +284,7 @@ export default class LocalTabs implements OrderFolderResource): Promise { + async createFolder(folder:Folder): Promise { Logger.log('(tabs)CREATEFOLDER', folder) // If parentId is 'tabs', create a window @@ -276,14 +292,14 @@ export default class LocalTabs implements OrderFolderResource browser.windows.create() ) - return node.id + return this.getFolderIdFromWindowId(node.id) } else { // Otherwise, create a tab group try { const groupId = await this.queue.add(async() => { // Create a dummy tab in the parent window to hold the group const dummyTab = await browser.tabs.create({ - windowId: folder.parentId, + windowId: this.getWindowIdFromFolderId(folder.parentId), url: 'about:blank', active: false }) @@ -292,8 +308,8 @@ export default class LocalTabs implements OrderFolderResource browser.tabs.query({groupId: id})) + const tabs = await this.queue.add(() => browser.tabs.query({groupId: this.getTabGroupIdFromFolderId(id)})) if (tabs.length) { isTabGroup = true // Get the tab group's current index @@ -382,14 +398,14 @@ export default class LocalTabs implements OrderFolderResource - browser.tabGroups.move(folder.id, { index: currentIndex }) + browser.tabGroups.move(this.getTabGroupIdFromFolderId(folder.id), { index: currentIndex }) ) // Get the size of the folder (number of tabs in the group) const folderTabs = await this.queue.add(() => browser.tabs.query({ windowType: 'normal', - groupId: folder.id + groupId: this.getTabGroupIdFromFolderId(folder.id) }) ) @@ -431,7 +447,7 @@ export default class LocalTabs implements OrderFolderResource - browser.tabGroups.update(folder.id, { + browser.tabGroups.update(this.getTabGroupIdFromFolderId(folder.id), { title: folder.title }) ) @@ -453,7 +469,7 @@ export default class LocalTabs implements OrderFolderResource browser.windows.remove(id)) + await this.queue.add(() => browser.windows.remove(this.getWindowIdFromFolderId(id))) } catch (e) { Logger.log('Failed to remove window', e) // Don't throw error if the window doesn't exist anymore @@ -467,7 +483,7 @@ export default class LocalTabs implements OrderFolderResource browser.tabs.query({ - groupId: id + groupId: this.getTabGroupIdFromFolderId(id) }) ) From 761955e411dc45a2c4a24f5f65c352b71a2c39b8 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 08:37:14 +0100 Subject: [PATCH 16/53] fix(CachingTreeWrapper): Make sure to copy the tree Signed-off-by: Marcel Klehr --- src/lib/Account.ts | 3 ++- src/lib/CachingTreeWrapper.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/Account.ts b/src/lib/Account.ts index 1c6675e241..bb2627a090 100644 --- a/src/lib/Account.ts +++ b/src/lib/Account.ts @@ -317,7 +317,8 @@ export default class Account { // set from the persisted continuation Logger.log('Fetching local bookmarks tree') this.syncProcess.setCacheTree(cacheTree) - await this.localCachingResource.setCacheTree(await this.localCachingResource.getBookmarksTree()) + // Allow Caching of the local tree + await this.localCachingResource.getBookmarksTree() } Logger.log('Starting sync process') diff --git a/src/lib/CachingTreeWrapper.ts b/src/lib/CachingTreeWrapper.ts index 6b0ed32f2c..08ef4ee668 100644 --- a/src/lib/CachingTreeWrapper.ts +++ b/src/lib/CachingTreeWrapper.ts @@ -14,12 +14,12 @@ export default class CachingTreeWrapper implements OrderFolderResource> { const tree = await this.innerTree.getBookmarksTree() - this.cacheTree.setTree(tree) + this.cacheTree.setTree(tree.copy()) return tree } async setCacheTree(tree: Folder) { - this.cacheTree.setTree(tree) + this.cacheTree.setTree(tree.copy()) } async createBookmark(bookmark:Bookmark): Promise { From c069b1f8f375202c04dc07fdf67c95cf031297a4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 09:47:59 +0100 Subject: [PATCH 17/53] fix(Caching#orderFolder): Fix orderFolder algorithm Signed-off-by: Marcel Klehr --- src/lib/adapters/Caching.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index a39d96dc20..4a49ee4497 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -182,15 +182,12 @@ export default class CachingAdapter implements Adapter, BulkImportResource { const child = folder.findItem(item.type, item.id) if (!child || String(child.parentId) !== String(folder.id)) { throw new UnknownFolderItemOrderError(id + ':' + JSON.stringify(item)) } - }) - let newChildren = [] - order.forEach(item => { - const child = folder.findItem(item.type, item.id) newChildren.push(child) }) const diff = difference(folder.children.map(i => i.type + ':' + i.id), order.map(i => i.type + ':' + i.id)) @@ -201,8 +198,9 @@ export default class CachingAdapter implements Adapter, BulkImportResource { const [type, id] = item.split(':') const child = folder.findItem(type, id) + if (!child) return const index = folder.children.indexOf(child) - newChildren = newChildren.slice(0, index - 1).concat([child], newChildren.slice(index - 1)) + newChildren = newChildren.slice(0, index).concat([child], newChildren.slice(index)) }) } folder.children = newChildren From 0fe68fe2f49239b9f62c4f090f753dd60a64756a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 09:49:00 +0100 Subject: [PATCH 18/53] fix(BrowserAccountStorage): Use setEntry instead of changeEntry to reduce memory pressure Signed-off-by: Marcel Klehr --- src/lib/browser/BrowserAccountStorage.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lib/browser/BrowserAccountStorage.js b/src/lib/browser/BrowserAccountStorage.js index 9862979ce6..d6ce75c560 100644 --- a/src/lib/browser/BrowserAccountStorage.js +++ b/src/lib/browser/BrowserAccountStorage.js @@ -128,9 +128,9 @@ export default class BrowserAccountStorage { } async initCache() { - await BrowserAccountStorage.changeEntry( + await BrowserAccountStorage.setEntry( `bookmarks[${this.accountId}].cache`, - () => ({}) + {} ) } @@ -144,9 +144,9 @@ export default class BrowserAccountStorage { } async setCache(data) { - await BrowserAccountStorage.changeEntry( + await BrowserAccountStorage.setEntry( `bookmarks[${this.accountId}].cache`, - () => data + data ) } @@ -157,9 +157,9 @@ export default class BrowserAccountStorage { } async initMappings() { - await BrowserAccountStorage.changeEntry( + await BrowserAccountStorage.setEntry( `bookmarks[${this.accountId}].mappings`, - () => ({}) + {} ) } @@ -185,9 +185,9 @@ export default class BrowserAccountStorage { } async setMappings(data) { - await BrowserAccountStorage.changeEntry( + await BrowserAccountStorage.setEntry( `bookmarks[${this.accountId}].mappings`, - () => data + data ) } From 9c64fda813d4121da4a9bf4af338ad4456c59dae Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 11:12:55 +0100 Subject: [PATCH 19/53] fix(Folder#toJSON): Make sure children are properly serialized Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 3 +++ src/lib/browser/BrowserAccountStorage.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 0ea6a90abd..6b91c1087f 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -516,6 +516,9 @@ export class Folder { Object.entries(obj).forEach(([key, value]) => { if (key === 'index') return if (!(key in result)) { + if (key === 'children') { + value = value.map((child) => child.toJSON()) + } result[key] = value } }) diff --git a/src/lib/browser/BrowserAccountStorage.js b/src/lib/browser/BrowserAccountStorage.js index d6ce75c560..03d2cefe76 100644 --- a/src/lib/browser/BrowserAccountStorage.js +++ b/src/lib/browser/BrowserAccountStorage.js @@ -146,7 +146,7 @@ export default class BrowserAccountStorage { async setCache(data) { await BrowserAccountStorage.setEntry( `bookmarks[${this.accountId}].cache`, - data + data.toJSON ? data.toJSON() : data ) } From 55c8a5656d5cd00b8934a59b57b8ef2f3e6970f4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 11:31:16 +0100 Subject: [PATCH 20/53] fix(NextcloudBookmarks): Fix updateBookmark() Signed-off-by: Marcel Klehr --- src/lib/adapters/NextcloudBookmarks.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lib/adapters/NextcloudBookmarks.ts b/src/lib/adapters/NextcloudBookmarks.ts index a0914cdee5..bf942c5ab5 100644 --- a/src/lib/adapters/NextcloudBookmarks.ts +++ b/src/lib/adapters/NextcloudBookmarks.ts @@ -760,13 +760,11 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes if (oldBm && this.tree) { this.tree.removeFromIndex(oldBm) } - oldBm.title = newBm.title - oldBm.parentId = newBm.parentId if (!newFolder.children.find(item => String(item.id) === String(newBm.id) && item.type === 'bookmark')) { - newFolder.children.push(oldBm) + newFolder.children.push(newBm) } - oldBm.id = upstreamId + ';' + newBm.parentId - this.tree.updateIndex(oldBm) + newBm.id = upstreamId + ';' + newBm.parentId + this.tree.updateIndex(newBm) return newBm.id }) From 7af309bb4fc32d9f2f146d0dc43964c7218514cd Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 11:51:43 +0100 Subject: [PATCH 21/53] tests: Don't pass anonymous object into Adapter#updateBookmark Signed-off-by: Marcel Klehr --- src/test/test.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/test.js b/src/test/test.js index abae5ce46e..dc25d6aea9 100644 --- a/src/test/test.js +++ b/src/test/test.js @@ -2674,7 +2674,7 @@ describe('Floccus', function() { await withSyncConnection(account, async() => { // move first separator - await account.server.updateBookmark({...tree.children[0].children[0].children[1], parentId: tree.children[0].id}) + await account.server.updateBookmark(new Bookmark({...tree.children[0].children[0].children[1], parentId: tree.children[0].id})) }) console.log('move done') @@ -5768,7 +5768,7 @@ describe('Floccus', function() { new Bookmark(serverMark2) ) - await adapter.updateBookmark({ ...serverMark, id: serverMarkId, url: TEST_URL + '#test2', title: TEST_URL_TITLE, parentId: tree.children[0].id }) + await adapter.updateBookmark(new Bookmark({ ...serverMark, id: serverMarkId, url: TEST_URL + '#test2', title: TEST_URL_TITLE, parentId: tree.children[0].id })) }) await account.setData({ strategy: 'slave'}) @@ -5855,7 +5855,7 @@ describe('Floccus', function() { new Bookmark(serverMark2) ) - await adapter.updateBookmark({ ...serverMark, id: serverMarkId, url: TEST_URL + '#test2', title: TEST_URL_TITLE, parentId: tree.children[0].id }) + await adapter.updateBookmark(new Bookmark({ ...serverMark, id: serverMarkId, url: TEST_URL + '#test2', title: TEST_URL_TITLE, parentId: tree.children[0].id })) }) await browser.tabs.create({url: TEST_URL + '#test4'}) @@ -6853,10 +6853,10 @@ describe('Floccus', function() { // Move the bookmarks out of the group for (const bookmark of tabGroupFolder.children) { - await account.server.updateBookmark({ + await account.server.updateBookmark(new Bookmark({ ...bookmark, parentId: windowFolder.id - }) + })) } // Remove the now-empty group folder From 110ed3ccd7b195c13ead042ff061ccf6c12db8c9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 14:59:02 +0100 Subject: [PATCH 22/53] fix(Diff): Do not call .toString on null, but use String() Signed-off-by: Marcel Klehr --- src/lib/Diff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Diff.ts b/src/lib/Diff.ts index db2ebf0914..15321847eb 100644 --- a/src/lib/Diff.ts +++ b/src/lib/Diff.ts @@ -375,7 +375,7 @@ export default class Diff< diff.commit(action) Logger.log('Failed to map parentId of action ' + diff.inspect()) Logger.log(JSON.stringify(mappingsSnapshot, null, '\t')) - throw new MappingFailureError(action.payload.parentId.toString()) + throw new MappingFailureError(String(action.payload.parentId)) } } } From 863aae6c989f35900bbb2dfe918db34000e872e1 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 15:31:24 +0100 Subject: [PATCH 23/53] fix: Fix typeof undefined checks to handle null Signed-off-by: Marcel Klehr --- src/lib/Diff.ts | 3 ++- src/lib/Mappings.ts | 2 +- src/lib/Scanner.ts | 2 +- src/lib/Tree.ts | 2 +- src/lib/browser/BrowserTree.ts | 2 +- src/lib/strategies/Default.ts | 12 ++++++++---- src/lib/strategies/Unidirectional.ts | 2 +- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/lib/Diff.ts b/src/lib/Diff.ts index 15321847eb..eaa80c7d2f 100644 --- a/src/lib/Diff.ts +++ b/src/lib/Diff.ts @@ -357,7 +357,8 @@ export default class Diff< ) if ( typeof newAction.payload.parentId === 'undefined' && - typeof action.payload.parentId !== 'undefined' + typeof action.payload.parentId !== 'undefined' && + action.payload.parentId !== null ) { if (skipErroneousActions) { // simply ignore this action as it appears to be no longer valid diff --git a/src/lib/Mappings.ts b/src/lib/Mappings.ts index fc288d6085..de9cabc14a 100644 --- a/src/lib/Mappings.ts +++ b/src/lib/Mappings.ts @@ -71,7 +71,7 @@ export default class Mappings { } private static add(mappings, { localId, remoteId }: { localId?:string|number, remoteId?:string|number }) { - if (typeof localId === 'undefined' || typeof remoteId === 'undefined') { + if (typeof localId === 'undefined' || typeof remoteId === 'undefined' || localId === null || remoteId === null) { throw new Error('Cannot add empty mapping') } mappings.LocalToServer[localId] = remoteId diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 7d8fdda4b2..754b059535 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -28,7 +28,7 @@ export default class Scanner this.newTree = newTree this.mergeable = mergeable this.hashSettings = hashSettings - this.checkHashes = typeof checkHashes === 'undefined' ? true : checkHashes + this.checkHashes = typeof checkHashes === 'undefined' || checkHashes === null ? true : checkHashes this.hasCache = hasCache this.result = { CREATE: new Diff(), diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 6b91c1087f..a92c491537 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -112,7 +112,7 @@ export class Bookmark { if (!this.hashValue) { this.hashValue = {} } - if (typeof this.hashValue[hashFn] === 'undefined') { + if (typeof this.hashValue[hashFn] === 'undefined' || this.hashValue[hashFn] === null) { const json = JSON.stringify({ title: this.title, url: this.url }) if (hashFn === 'sha256') { this.hashValue[hashFn] = await Crypto.sha256(json) diff --git a/src/lib/browser/BrowserTree.ts b/src/lib/browser/BrowserTree.ts index e9062be500..117e5e9c15 100644 --- a/src/lib/browser/BrowserTree.ts +++ b/src/lib/browser/BrowserTree.ts @@ -366,7 +366,7 @@ export default class BrowserTree implements IResource } static async getIdPathFromLocalId(localId:string|null, path:string[] = []):Promise { - if (typeof localId === 'undefined') { + if (typeof localId === 'undefined' || localId === null) { return path } path.unshift(localId) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index e30ed2e607..07cd8bbb47 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -175,7 +175,7 @@ export default class SyncProcess { } async updateProgress():Promise { - if (typeof this.actionsDone === 'undefined') { + if (typeof this.actionsDone === 'undefined' || this.actionsDone === null) { this.actionsDone = 0 } this.actionsDone++ @@ -516,7 +516,11 @@ export default class SyncProcess { // Failsafe kicks in if more than 20% is deleted or more than 1k bookmarks if ((countTotal > 5 && countDeleted / countTotal > 0.2) || countDeleted > 1000) { const failsafe = this.server.getData().failsafe - if (failsafe !== false || typeof failsafe === 'undefined') { + if ( + failsafe !== false || + typeof failsafe === 'undefined' || + failsafe === null + ) { const percentage = Math.ceil((countDeleted / countTotal) * 100) if (direction === ItemLocation.LOCAL) { throw new ClientsideDeletionFailsafeError(percentage) @@ -535,7 +539,7 @@ export default class SyncProcess { // Failsafe kicks in if more than 20% is added or more than 1k bookmarks if (countTotal > 5 && ((countAdded >= 20 && countAdded / countTotal > 0.2) || countAdded > 1000)) { const failsafe = this.server.getData().failsafe - if (failsafe !== false || typeof failsafe === 'undefined') { + if (failsafe !== false || typeof failsafe === 'undefined' || failsafe === null) { const percentage = Math.ceil((countAdded / countTotal) * 100) if (direction === ItemLocation.LOCAL) { throw new ClientsideAdditionFailsafeError(percentage) @@ -1041,7 +1045,7 @@ export default class SyncProcess { action.payload.visitCreate(resource), this.cancelPromise ]) - if (typeof id === 'undefined') { + if (typeof id === 'undefined' || id === null) { // undefined means we couldn't create the item. we're ignoring it await done() return diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index e16e66750e..9c529fab52 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -446,7 +446,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { const nonexistingItems = [] await newItem.traverse(async(child, parentFolder) => { child.id = Mappings.mapId(mappingsSnapshot, child, fakeLocation) - if (typeof child.id === 'undefined') { + if (typeof child.id === 'undefined' || child.id === null) { nonexistingItems.push(child) } child.parentId = parentFolder.id From 8886684c466bafaeb99501a0099dfeab909ea699 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 15:31:51 +0100 Subject: [PATCH 24/53] fix(Diff): Fix toJSONAsync null handling Signed-off-by: Marcel Klehr --- src/lib/Diff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Diff.ts b/src/lib/Diff.ts index eaa80c7d2f..40b7850ca3 100644 --- a/src/lib/Diff.ts +++ b/src/lib/Diff.ts @@ -421,7 +421,7 @@ export default class Diff< ...action, payload: await action.payload.clone(false).toJSONAsync(), oldItem: - (await action.oldItem) && action.oldItem.clone(false).toJSONAsync(), + action.oldItem && await action.oldItem.clone(false).toJSONAsync(), } }, 1 From 2e5afd1d5bb9e615f12d80f2b09110b2b6400aec Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 15:38:38 +0100 Subject: [PATCH 25/53] fix(SyncProcess): Simplify setProgress Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 36 ++++++++-------- src/lib/strategies/Unidirectional.ts | 62 +++++++++------------------- 2 files changed, 37 insertions(+), 61 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index 07cd8bbb47..099ced7e88 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -195,35 +195,33 @@ export default class SyncProcess { } async setProgress(json: any) { - const {actionsDone, actionsPlanned} = json - this.actionsDone = actionsDone - this.actionsPlanned = actionsPlanned if (json.serverTreeRoot) { this.serverTreeRoot = Folder.hydrate(json.serverTreeRoot) + delete json.serverTreeRoot } if (json.localTreeRoot) { this.localTreeRoot = Folder.hydrate(json.localTreeRoot) + delete json.localTreeRoot } if (json.cacheTreeRoot) { this.cacheTreeRoot = Folder.hydrate(json.cacheTreeRoot) - } - Object.keys(json).forEach((member) => { - if (member in json) { - if (member.toLowerCase().includes('scanresult') || member.toLowerCase().includes('plan')) { - this[member] = { - CREATE: Diff.fromJSON(json[member].CREATE), - UPDATE: Diff.fromJSON(json[member].UPDATE), - MOVE: Diff.fromJSON(json[member].MOVE), - REMOVE: Diff.fromJSON(json[member].REMOVE), - REORDER: Diff.fromJSON(json[member].REORDER), - } - } else if (member.toLowerCase().includes('reorders')) { - this[member] = Diff.fromJSON(json[member]) - } else { - this[member] = json[member] + delete json.cacheTreeRoot + } + for (const member of Object.keys(json)) { + if (member.toLowerCase().includes('scanresult') || member.toLowerCase().includes('plan')) { + this[member] = { + CREATE: await Diff.fromJSONAsync(json[member].CREATE), + UPDATE: await Diff.fromJSONAsync(json[member].UPDATE), + MOVE: await Diff.fromJSONAsync(json[member].MOVE), + REMOVE: await Diff.fromJSONAsync(json[member].REMOVE), + REORDER: await Diff.fromJSONAsync(json[member].REORDER), } + } else if (member.toLowerCase().includes('reorders')) { + this[member] = await Diff.fromJSONAsync(json[member]) + } else { + this[member] = json[member] } - }) + } } setDirection(direction:TItemLocation):void { diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 9c529fab52..3fa25f2b0c 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -53,58 +53,36 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { } async setProgress(json: any) { - const {actionsDone, actionsPlanned} = json - this.actionsDone = actionsDone - this.actionsPlanned = actionsPlanned if (json.serverTreeRoot) { this.serverTreeRoot = Folder.hydrate(json.serverTreeRoot) + delete json.serverTreeRoot } if (json.localTreeRoot) { this.localTreeRoot = Folder.hydrate(json.localTreeRoot) + delete json.localTreeRoot } if (json.cacheTreeRoot) { this.cacheTreeRoot = Folder.hydrate(json.cacheTreeRoot) - } - if (json.localScanResult) { - this.localScanResult = { - CREATE: await Diff.fromJSONAsync(json.localScanResult.CREATE), - UPDATE: await Diff.fromJSONAsync(json.localScanResult.UPDATE), - MOVE: await Diff.fromJSONAsync(json.localScanResult.MOVE), - REMOVE: await Diff.fromJSONAsync(json.localScanResult.REMOVE), - REORDER: await Diff.fromJSONAsync(json.localScanResult.REORDER), - } - } - if (json.serverScanResult) { - this.serverScanResult = { - CREATE: await Diff.fromJSONAsync(json.serverScanResult.CREATE), - UPDATE: await Diff.fromJSONAsync(json.serverScanResult.UPDATE), - MOVE: await Diff.fromJSONAsync(json.serverScanResult.MOVE), - REMOVE: await Diff.fromJSONAsync(json.serverScanResult.REMOVE), - REORDER: await Diff.fromJSONAsync(json.serverScanResult.REORDER), - } - } - if (json.revertPlan) { - this.revertPlan = { - CREATE: await Diff.fromJSONAsync(json.revertPlan.CREATE), - UPDATE: await Diff.fromJSONAsync(json.revertPlan.UPDATE), - MOVE: await Diff.fromJSONAsync(json.revertPlan.MOVE), - REMOVE: await Diff.fromJSONAsync(json.revertPlan.REMOVE), - REORDER: await Diff.fromJSONAsync(json.revertPlan.REORDER), - } - } - if (json.revertDonePlan) { - this.revertDonePlan = { - CREATE: await Diff.fromJSONAsync(json.revertDonePlan.CREATE), - UPDATE: await Diff.fromJSONAsync(json.revertDonePlan.UPDATE), - MOVE: await Diff.fromJSONAsync(json.revertDonePlan.MOVE), - REMOVE: await Diff.fromJSONAsync(json.revertDonePlan.REMOVE), - REORDER: await Diff.fromJSONAsync(json.revertDonePlan.REORDER), + delete json.cacheTreeRoot + } + for (const member of Object.keys(json)) { + if ( + member.toLowerCase().includes('scanresult') || + member.toLowerCase().includes('plan') + ) { + this[member] = { + CREATE: await Diff.fromJSONAsync(json[member].CREATE), + UPDATE: await Diff.fromJSONAsync(json[member].UPDATE), + MOVE: await Diff.fromJSONAsync(json[member].MOVE), + REMOVE: await Diff.fromJSONAsync(json[member].REMOVE), + REORDER: await Diff.fromJSONAsync(json[member].REORDER), + } + } else if (member.toLowerCase().includes('reorders')) { + this[member] = await Diff.fromJSONAsync(json[member]) + } else { + this[member] = json[member] } } - if (json.revertReorders) { - this.revertReorders = await Diff.fromJSONAsync(json.revertReorders) - } - this.direction = json.direction } async getDiffs(): Promise<{ From b770ca8e612b88f32623197d3081fc58aaab69c1 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 29 Dec 2025 16:55:31 +0100 Subject: [PATCH 26/53] fix(Tree): Fix removeFromIndex to remove the entire subtree instead of just the ID of the item that was passed Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index a92c491537..2ba18786b7 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -602,7 +602,7 @@ export class Folder { } /** - * Update the index by removing the given item (this method should be called on the root folder) + * Update the index by removing the given item and its children (this method should be called on the root folder) */ removeFromIndex(item: TItem) { if (!item) return @@ -613,7 +613,16 @@ export class Folder { if (item.parentId) { let parentFolder = this.index.folder[item.parentId] while (parentFolder && this.index.folder[parentFolder.parentId] !== parentFolder) { - delete parentFolder.index[item.type][item.id] + if (item instanceof Bookmark) { + delete parentFolder.index[item.type][item.id] + } else { + for (const folderId in item.index.folder) { + delete parentFolder.index.folder[folderId] + } + for (const bookmarkId in item.index.bookmark) { + delete parentFolder.index.bookmark[bookmarkId] + } + } parentFolder = this.index.folder[parentFolder.parentId] } } From 95fd68ddc4bb998b89d6c6a3701372fbd0cb829b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 10:37:01 +0100 Subject: [PATCH 27/53] =?UTF-8?q?fix(childrenSimilarity):=20Make=20O(n)=20?= =?UTF-8?q?instead=20of=20O(n=C2=B2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 2ba18786b7..4d57f17fd0 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -387,15 +387,10 @@ export class Folder { childrenSimilarity(otherItem: TItem): number { if (otherItem instanceof Folder) { - return ( - this.children.reduce( - (count, item) => - otherItem.children.find((i) => i.title === item.title) - ? count + 1 - : count, - 0 - ) / Math.max(this.children.length, otherItem.children.length) - ) + const myChildrenTitles = new Set(this.children.map((child) => child.title)) + const otherChildrenTitles = new Set(otherItem.children.map((child) => child.title)) + const overlappingTitles = new Set([...myChildrenTitles].filter((title) => otherChildrenTitles.has(title))) + return overlappingTitles.size / Math.max(myChildrenTitles.size, otherChildrenTitles.size) } return 0 } From 7ffb2697a57b59a62c11582f3ccce12761a0ebc0 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 10:37:17 +0100 Subject: [PATCH 28/53] fix(Tree#hash): Use yieldToEventLoop to allow browser to breathe Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 4d57f17fd0..5170229d24 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -416,6 +416,8 @@ export class Folder { throw new Error("Trying to calculate hash of a folder that isn't loaded") } + await yieldToEventLoop() + const children = this.children.slice() if (!preserveOrder) { // only re-sort unless we sync the order of the children as well From 2caffc582b42a8122d0073408e64f9a86fe13b5d Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 10:38:27 +0100 Subject: [PATCH 29/53] fix(Scanner): Reduce time complexity of findMoves to improve performance Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 209 +++++++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 113 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 754b059535..df670fa4ef 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -150,127 +150,110 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - let createActions - let removeActions - let reconciled = true - - // As soon as one match is found, action list is updated and search is started with the new list - // repeat until no rewrites happen anymore - while (reconciled) { - reconciled = false - let createAction: CreateAction, removeAction: RemoveAction - - // First find direct matches (avoids glitches when folders and their contents have been moved) - createActions = this.result.CREATE.getActions() - while (!reconciled && (createAction = createActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const createdItem = createAction.payload - removeActions = this.result.REMOVE.getActions() - while (!reconciled && (removeAction = removeActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const removedItem = removeAction.payload - - if (this.mergeable(removedItem, createdItem) && - (removedItem.type !== 'folder' || - (!this.hasCache && removedItem.childrenSimilarity(createdItem) > 0.8))) { - this.result.CREATE.retract(createAction) - this.result.REMOVE.retract(removeAction) - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem: removedItem, - index: createAction.index, - oldIndex: removeAction.index - }) - reconciled = true - // Don't use the items from the action, but the ones in the actual tree to avoid using tree parts mutated by this algorithm (see below) - await this.diffItem(removedItem, createdItem) - } - } + + const createActions = this.result.CREATE.getActions() + const removeActions = this.result.REMOVE.getActions() + + // 1. Index REMOVE actions for O(1) lookups + // Using a Map where key is ItemType + Title/URL (or other mergeable criteria) + const removalsMap = new Map[]>() + for (const action of removeActions) { + const item = action.payload + const key = `${item.type}_${item.title}_${item.type === 'bookmark' ? (item as any).url : ''}` + const list = removalsMap.get(key) || [] + list.push(action) + removalsMap.set(key, list) + } + + const pendingDiffs: [TItem, TItem][] = [] + + // 2. Single pass over CREATE actions to find direct matches + for (const createAction of createActions) { + await yieldToEventLoop() + const createdItem = createAction.payload + const key = `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as any).url : ''}` + + const potentialRemovals = removalsMap.get(key) + if (!potentialRemovals) continue + + const matchIndex = potentialRemovals.findIndex(removeAction => + this.mergeable(removeAction.payload, createdItem) && + (removeAction.payload.type !== 'folder' || (!this.hasCache && removeAction.payload.childrenSimilarity(createdItem) > 0.8)) + ) + + if (matchIndex !== -1) { + const removeAction = potentialRemovals.splice(matchIndex, 1)[0] + const removedItem = removeAction.payload + + this.result.CREATE.retract(createAction) + this.result.REMOVE.retract(removeAction) + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem: removedItem, + index: createAction.index, + oldIndex: removeAction.index + }) + + // Queue for diffing later to avoid breaking loop iterator/logic + pendingDiffs.push([removedItem, createdItem]) } + } + + // 3. Process descendant matches (kept as a second pass for logic clarity) + const remainingCreates = this.result.CREATE.getActions() + const remainingRemoves = this.result.REMOVE.getActions() + + for (const createAction of remainingCreates) { + await yieldToEventLoop() + const createdItem = createAction.payload + + for (const removeAction of remainingRemoves) { + const removedItem = removeAction.payload + + // Search within the removed subtree for the created item + const oldItem = removedItem.findItemFilter( + createdItem.type, + item => this.mergeable(item, createdItem), + item => item.childrenSimilarity(createdItem) + ) - // Then find descendant matches - createActions = this.result.CREATE.getActions() - while (!reconciled && (createAction = createActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const createdItem = createAction.payload - removeActions = this.result.REMOVE.getActions() - while (!reconciled && (removeAction = removeActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const removedItem = removeAction.payload - const oldItem = removedItem.findItemFilter( - createdItem.type, - item => this.mergeable(item, createdItem), - item => item.childrenSimilarity(createdItem) - ) - if (oldItem) { - let oldIndex - this.result.CREATE.retract(createAction) - if (oldItem === removedItem) { - this.result.REMOVE.retract(removeAction) - } else { - // We clone the item here, because we don't want to mutate all copies of this tree (item) - const removedItemClone = removedItem.copy(true) - const oldParentClone = removedItemClone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder - const oldItemClone = removedItemClone.findItem(oldItem.type, oldItem.id) - oldIndex = oldParentClone.children.indexOf(oldItemClone) - oldParentClone.children.splice(oldIndex, 1) - removeAction.payload = removedItemClone - removeAction.payload.createIndex() - } - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem, - index: createAction.index, - oldIndex: oldIndex || removeAction.index - }) - reconciled = true - if (oldItem.type === ItemType.FOLDER) { // TODO: Is this necessary? - await this.diffItem(oldItem, createdItem) - } + if (oldItem) { + let oldIndex + this.result.CREATE.retract(createAction) + if (oldItem === removedItem) { + this.result.REMOVE.retract(removeAction) } else { - const newItem = createdItem.findItemFilter( - removedItem.type, - item => this.mergeable(removedItem, item), - item => item.childrenSimilarity(removedItem) - ) - let index - if (newItem) { - this.result.REMOVE.retract(removeAction) - if (newItem === createdItem) { - this.result.CREATE.retract(createAction) - } else { - // We clone the item here, because we don't want to mutate all copies of this tree (item) - const createdItemClone = createdItem.copy(true) - const newParentClone = createdItemClone.findItem(ItemType.FOLDER, newItem.parentId) as Folder - const newClonedItem = createdItemClone.findItem(newItem.type, newItem.id) - index = newParentClone.children.indexOf(newClonedItem) - newParentClone.children.splice(index, 1) - createAction.payload = createdItemClone - createAction.payload.createIndex() - } - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: newItem, - oldItem: removedItem, - index: index || createAction.index, - oldIndex: removeAction.index - }) - reconciled = true - if (removedItem.type === ItemType.FOLDER) { - await this.diffItem(removedItem, newItem) - } - } + const removedItemClone = removedItem.copy(true) + const oldParentClone = removedItemClone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder + const oldItemClone = removedItemClone.findItem(oldItem.type, oldItem.id) + oldIndex = oldParentClone.children.indexOf(oldItemClone) + oldParentClone.children.splice(oldIndex, 1) + removeAction.payload = removedItemClone + removeAction.payload.createIndex() + } + + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem, + index: createAction.index, + oldIndex: oldIndex || removeAction.index + }) + + if (oldItem.type === ItemType.FOLDER) { + pendingDiffs.push([oldItem, createdItem]) } + break // Move to next CreateAction } } } + // 4. Execute deferred diffs + for (const [oldItem, newItem] of pendingDiffs) { + await this.diffItem(oldItem, newItem) + } + // Remove all UPDATEs that have already been handled by a MOVE const moves = this.result.MOVE.getActions() const updates = this.result.UPDATE.getActions() From 3925e017e575cd434174e3c7231b78561817f387 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 10:39:02 +0100 Subject: [PATCH 30/53] fix(Scanner): Reduce time complexity of diffFolder to improve performance Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 105 +++++++++++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 27 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index df670fa4ef..e1e2f5d3c4 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -70,16 +70,32 @@ export default class Scanner } } - if (oldFolder.title !== newFolder.title && typeof oldFolder.parentId !== 'undefined' && typeof newFolder.parentId !== 'undefined') { + if ( + oldFolder.title !== newFolder.title && + typeof oldFolder.parentId !== 'undefined' && + typeof newFolder.parentId !== 'undefined' + ) { // folder title changed and it's not the root folder - this.result.UPDATE.commit({type: ActionType.UPDATE, payload: newFolder, oldItem: oldFolder}) + this.result.UPDATE.commit({ + type: ActionType.UPDATE, + payload: newFolder, + oldItem: oldFolder, + }) } // Generate REORDERS before diffing anything to make sure REORDERS are from top to bottom (necessary for tab sync) if (newFolder.children.length > 1) { let needReorder = false - for (let i = 0; i < Math.max(newFolder.children.length, oldFolder.children.length); i++) { - if (!oldFolder.children[i] || !newFolder.children[i] || !this.mergeable(oldFolder.children[i], newFolder.children[i])) { + for ( + let i = 0; + i < Math.max(newFolder.children.length, oldFolder.children.length); + i++ + ) { + if ( + !oldFolder.children[i] || + !newFolder.children[i] || + !this.mergeable(oldFolder.children[i], newFolder.children[i]) + ) { needReorder = true break } @@ -88,40 +104,75 @@ export default class Scanner this.result.REORDER.commit({ type: ActionType.REORDER, payload: newFolder, - order: newFolder.children.map(i => ({ type: i.type, id: i.id })), + order: newFolder.children.map((i) => ({ type: i.type, id: i.id })), }) } } // Preserved Items and removed Items + // Optimization: Use a Map for O(1) lookups + const unmatchedMap = new Map[]>() + for (const child of newFolder.children) { + const key = `${child.type}_${child.title}` // Or a better unique key based on mergeable logic + const list = unmatchedMap.get(key) || [] + list.push(child) + unmatchedMap.set(key, list) + } + const stillUnmatched = new Set(newFolder.children) + // (using map here, because 'each' doesn't provide indices) - const unmatchedChildren = newFolder.children.slice(0) - await Parallel.map(oldFolder.children, async(old, index) => { - const newItem = unmatchedChildren.find((child) => old.type === child.type && this.mergeable(old, child)) - // we found an item in the new folder that matches the one in the old folder - if (newItem) { - await this.diffItem(old, newItem) - unmatchedChildren.splice(unmatchedChildren.indexOf(newItem), 1) - return - } + await Parallel.map( + oldFolder.children, + async(old, index) => { + const key = `${old.type}_${old.title}` + const potentialMatches = unmatchedMap.get(key) + let newItem = null + if (potentialMatches) { + const matchIndex = potentialMatches.findIndex((m) => + this.mergeable(old, m) + ) + if (matchIndex !== -1) { + newItem = potentialMatches.splice(matchIndex, 1)[0] + stillUnmatched.delete(newItem) + } + } + // we found an item in the new folder that matches the one in the old folder + if (newItem) { + await this.diffItem(old, newItem) + return + } - if (newFolder.isRoot && newFolder.location === ItemLocation.LOCAL) { - // We can't remove root folders locally - return - } + if (newFolder.isRoot && newFolder.location === ItemLocation.LOCAL) { + // We can't remove root folders locally + return + } - this.result.REMOVE.commit({type: ActionType.REMOVE, payload: old, index}) - }, 1) + this.result.REMOVE.commit({ + type: ActionType.REMOVE, + payload: old, + index, + }) + }, + 1 + ) // created Items // (using map here, because 'each' doesn't provide indices) - await Parallel.map(unmatchedChildren, async(newChild) => { - if (oldFolder.isRoot && oldFolder.location === ItemLocation.LOCAL) { - // We can't create root folders locally - return - } - this.result.CREATE.commit({type: ActionType.CREATE, payload: newChild, index: newFolder.children.findIndex(child => child === newChild)}) - }, 1) + await Parallel.map( + Array.from(stillUnmatched.values()), + async(newChild) => { + if (oldFolder.isRoot && oldFolder.location === ItemLocation.LOCAL) { + // We can't create root folders locally + return + } + this.result.CREATE.commit({ + type: ActionType.CREATE, + payload: newChild, + index: newFolder.children.findIndex((child) => child === newChild), + }) + }, + 1 + ) } async diffBookmark(oldBookmark:Bookmark, newBookmark:Bookmark):Promise { From 6eb984ca3c6b6894cd212d9cb092587e13d9d0ba Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 12:14:12 +0100 Subject: [PATCH 31/53] fix(Scanner): Reduce time complexity of findMoves Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 192 +++++++++++++++++++++++++++------------------ 1 file changed, 116 insertions(+), 76 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index e1e2f5d3c4..9ae2f08f60 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -202,86 +202,132 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - const createActions = this.result.CREATE.getActions() - const removeActions = this.result.REMOVE.getActions() - - // 1. Index REMOVE actions for O(1) lookups - // Using a Map where key is ItemType + Title/URL (or other mergeable criteria) - const removalsMap = new Map[]>() - for (const action of removeActions) { - const item = action.payload - const key = `${item.type}_${item.title}_${item.type === 'bookmark' ? (item as any).url : ''}` - const list = removalsMap.get(key) || [] - list.push(action) - removalsMap.set(key, list) - } + let hasNewActions = true + + while (hasNewActions) { + hasNewActions = false + + const createActions = this.result.CREATE.getActions() + const removeActions = this.result.REMOVE.getActions() + + // Index REMOVE actions for O(1) lookups + // Build a multi-value Fuzzy Index of all REMOVE subtrees + const removedFuzzyMap = new Map< + string, + { rootAction: RemoveAction; item: TItem }[] + >() + + const addToMap = (item: TItem, action: RemoveAction) => { + const key = `${item.type}_${item.title}_${ + item.type === 'bookmark' ? (item as Bookmark).url : '' + }` + const list = removedFuzzyMap.get(key) || [] + list.push({ rootAction: action, item }) + removedFuzzyMap.set(key, list) + } - const pendingDiffs: [TItem, TItem][] = [] + for (const action of removeActions) { + const rootItem = action.payload + addToMap(rootItem, action) - // 2. Single pass over CREATE actions to find direct matches - for (const createAction of createActions) { - await yieldToEventLoop() - const createdItem = createAction.payload - const key = `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as any).url : ''}` - - const potentialRemovals = removalsMap.get(key) - if (!potentialRemovals) continue - - const matchIndex = potentialRemovals.findIndex(removeAction => - this.mergeable(removeAction.payload, createdItem) && - (removeAction.payload.type !== 'folder' || (!this.hasCache && removeAction.payload.childrenSimilarity(createdItem) > 0.8)) - ) - - if (matchIndex !== -1) { - const removeAction = potentialRemovals.splice(matchIndex, 1)[0] - const removedItem = removeAction.payload - - this.result.CREATE.retract(createAction) - this.result.REMOVE.retract(removeAction) - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem: removedItem, - index: createAction.index, - oldIndex: removeAction.index - }) - - // Queue for diffing later to avoid breaking loop iterator/logic - pendingDiffs.push([removedItem, createdItem]) + if (rootItem instanceof Folder) { + await rootItem.traverse((child) => addToMap(child, action)) + } } - } - // 3. Process descendant matches (kept as a second pass for logic clarity) - const remainingCreates = this.result.CREATE.getActions() - const remainingRemoves = this.result.REMOVE.getActions() + // Single pass over CREATE actions to find direct matches + for (const createAction of createActions) { + await yieldToEventLoop() + const createdItem = createAction.payload + const key = `${createdItem.type}_${createdItem.title}_${ + createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : '' + }` + + const potentialRemovals = removedFuzzyMap.get(key) + if (!potentialRemovals) continue + + const matchIndex = potentialRemovals.findIndex( + ({rootAction, item }) => + item === rootAction.payload && + this.mergeable(item, createdItem) && + (rootAction.payload.type !== 'folder' || + (!this.hasCache && + item.childrenSimilarity(createdItem) > 0.8)) + ) - for (const createAction of remainingCreates) { - await yieldToEventLoop() - const createdItem = createAction.payload + if (matchIndex !== -1) { + const {rootAction: removeAction} = potentialRemovals.splice(matchIndex, 1)[0] + const removedItem = removeAction.payload - for (const removeAction of remainingRemoves) { - const removedItem = removeAction.payload + this.result.CREATE.retract(createAction) + this.result.REMOVE.retract(removeAction) + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem: removedItem, + index: createAction.index, + oldIndex: removeAction.index, + }) + + // Diff + await this.diffItem(removedItem, createdItem) + hasNewActions = true + break + } + } - // Search within the removed subtree for the created item - const oldItem = removedItem.findItemFilter( - createdItem.type, - item => this.mergeable(item, createdItem), - item => item.childrenSimilarity(createdItem) + if (hasNewActions) continue + + // Process descendant matches + for (const createAction of createActions) { + await yieldToEventLoop() + const createdItem = createAction.payload + const fuzzyKey = `${createdItem.type}_${createdItem.title}_${ + createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : '' + }` + + const potentialMatches = removedFuzzyMap.get(fuzzyKey) + if (!potentialMatches) continue + + // Find the first match that satisfies mergeable logic + const matchIndex = potentialMatches.findIndex( + (match) => + this.mergeable(match.item, createdItem) && + (match.item.type !== 'folder' || + (!this.hasCache && + match.item.childrenSimilarity(createdItem) > 0.8)) ) - if (oldItem) { + if (matchIndex !== -1) { + // Extract the specific match from the array + const { rootAction, item: oldItem } = potentialMatches.splice( + matchIndex, + 1 + )[0] + const removedItem = rootAction.payload let oldIndex + this.result.CREATE.retract(createAction) + if (oldItem === removedItem) { - this.result.REMOVE.retract(removeAction) + this.result.REMOVE.retract(rootAction) } else { const removedItemClone = removedItem.copy(true) - const oldParentClone = removedItemClone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder - const oldItemClone = removedItemClone.findItem(oldItem.type, oldItem.id) - oldIndex = oldParentClone.children.indexOf(oldItemClone) - oldParentClone.children.splice(oldIndex, 1) - removeAction.payload = removedItemClone - removeAction.payload.createIndex() + const oldParentClone = removedItemClone.findItem( + ItemType.FOLDER, + oldItem.parentId + ) as Folder + const oldItemClone = removedItemClone.findItem( + oldItem.type, + oldItem.id + ) + + if (oldParentClone && oldItemClone) { + oldIndex = oldParentClone.children.indexOf(oldItemClone) + oldParentClone.children.splice(oldIndex, 1) + rootAction.payload = removedItemClone + rootAction.payload.createIndex() + } } this.result.MOVE.commit({ @@ -289,22 +335,16 @@ export default class Scanner payload: createdItem, oldItem, index: createAction.index, - oldIndex: oldIndex || removeAction.index + oldIndex: oldIndex ?? rootAction.index, }) - if (oldItem.type === ItemType.FOLDER) { - pendingDiffs.push([oldItem, createdItem]) - } - break // Move to next CreateAction + await this.diffItem(oldItem, createdItem) + hasNewActions = true + break } } } - // 4. Execute deferred diffs - for (const [oldItem, newItem] of pendingDiffs) { - await this.diffItem(oldItem, newItem) - } - // Remove all UPDATEs that have already been handled by a MOVE const moves = this.result.MOVE.getActions() const updates = this.result.UPDATE.getActions() From da48e1dcbd2befa73d6df6c87b2ed85fb0daaa9a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 13:03:48 +0100 Subject: [PATCH 32/53] fix(Scanner): Reduce time complexity of findMoves Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 206 ++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 106 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 9ae2f08f60..9503544ab5 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -210,138 +210,132 @@ export default class Scanner const createActions = this.result.CREATE.getActions() const removeActions = this.result.REMOVE.getActions() - // Index REMOVE actions for O(1) lookups - // Build a multi-value Fuzzy Index of all REMOVE subtrees + if (createActions.length === 0 || removeActions.length === 0) break + + // 1. Build Fuzzy Indices for both REMOVE and CREATE subtrees (O(N)) const removedFuzzyMap = new Map< string, { rootAction: RemoveAction; item: TItem }[] >() + const allCreatedItems: { + rootAction: CreateAction + item: TItem + }[] = [] - const addToMap = (item: TItem, action: RemoveAction) => { - const key = `${item.type}_${item.title}_${ + const getFuzzyKey = (item: TItem) => + `${item.type}_${item.title}_${ item.type === 'bookmark' ? (item as Bookmark).url : '' }` - const list = removedFuzzyMap.get(key) || [] - list.push({ rootAction: action, item }) - removedFuzzyMap.set(key, list) - } for (const action of removeActions) { - const rootItem = action.payload - addToMap(rootItem, action) - - if (rootItem instanceof Folder) { - await rootItem.traverse((child) => addToMap(child, action)) + const indexSubtree = async (item: TItem) => { + const key = getFuzzyKey(item) + removedFuzzyMap.set( + key, + (removedFuzzyMap.get(key) || []).concat({ + rootAction: action, + item, + }) + ) + } + await indexSubtree(action.payload) + if (action.payload instanceof Folder) { + await action.payload.traverse(indexSubtree) } } - // Single pass over CREATE actions to find direct matches - for (const createAction of createActions) { - await yieldToEventLoop() - const createdItem = createAction.payload - const key = `${createdItem.type}_${createdItem.title}_${ - createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : '' - }` - - const potentialRemovals = removedFuzzyMap.get(key) - if (!potentialRemovals) continue - - const matchIndex = potentialRemovals.findIndex( - ({rootAction, item }) => - item === rootAction.payload && - this.mergeable(item, createdItem) && - (rootAction.payload.type !== 'folder' || - (!this.hasCache && - item.childrenSimilarity(createdItem) > 0.8)) - ) - - if (matchIndex !== -1) { - const {rootAction: removeAction} = potentialRemovals.splice(matchIndex, 1)[0] - const removedItem = removeAction.payload - - this.result.CREATE.retract(createAction) - this.result.REMOVE.retract(removeAction) - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem: removedItem, - index: createAction.index, - oldIndex: removeAction.index, + for (const action of createActions) { + allCreatedItems.push({ rootAction: action, item: action.payload }) + if (action.payload instanceof Folder) { + await action.payload.traverse((child) => { + allCreatedItems.push({ rootAction: action, item: child }) }) - - // Diff - await this.diffItem(removedItem, createdItem) - hasNewActions = true - break } } - if (hasNewActions) continue - - // Process descendant matches - for (const createAction of createActions) { + // 2. Match ALL created items (roots + descendants) against removed pool + for (const createdEntry of allCreatedItems) { await yieldToEventLoop() - const createdItem = createAction.payload - const fuzzyKey = `${createdItem.type}_${createdItem.title}_${ - createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : '' - }` + const { rootAction: createRootAction, item: createdItem } = createdEntry + const fuzzyKey = getFuzzyKey(createdItem) const potentialMatches = removedFuzzyMap.get(fuzzyKey) if (!potentialMatches) continue - // Find the first match that satisfies mergeable logic - const matchIndex = potentialMatches.findIndex( - (match) => - this.mergeable(match.item, createdItem) && - (match.item.type !== 'folder' || - (!this.hasCache && - match.item.childrenSimilarity(createdItem) > 0.8)) + const matches = potentialMatches.filter((m) => + this.mergeable(m.item, createdItem) ) + if (createdItem.type === 'folder' && !this.hasCache) { + matches.sort( + (a, b) => + b.item.childrenSimilarity(createdItem) - + a.item.childrenSimilarity(createdItem) + ) + } - if (matchIndex !== -1) { - // Extract the specific match from the array - const { rootAction, item: oldItem } = potentialMatches.splice( - matchIndex, - 1 - )[0] - const removedItem = rootAction.payload - let oldIndex - - this.result.CREATE.retract(createAction) - - if (oldItem === removedItem) { - this.result.REMOVE.retract(rootAction) - } else { - const removedItemClone = removedItem.copy(true) - const oldParentClone = removedItemClone.findItem( - ItemType.FOLDER, - oldItem.parentId - ) as Folder - const oldItemClone = removedItemClone.findItem( - oldItem.type, - oldItem.id - ) - - if (oldParentClone && oldItemClone) { - oldIndex = oldParentClone.children.indexOf(oldItemClone) - oldParentClone.children.splice(oldIndex, 1) - rootAction.payload = removedItemClone - rootAction.payload.createIndex() - } - } + if (matches.length === 0) { + continue + } - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem, - index: createAction.index, - oldIndex: oldIndex ?? rootAction.index, - }) + const { rootAction: removeRootAction, item: oldItem } = matches[0] + const removedRoot = removeRootAction.payload + const createdRoot = createRootAction.payload + + let oldIndex, newIndex + + // Retract or Mutate "Old" (Removed) side + if (oldItem === removedRoot) { + this.result.REMOVE.retract(removeRootAction) + } else { + const removedRootClone = removedRoot.copy(true) + const oldParentClone = removedRootClone.findItem( + ItemType.FOLDER, + oldItem.parentId + ) as Folder + const oldItemClone = removedRootClone.findItem( + oldItem.type, + oldItem.id + ) + if (oldParentClone && oldItemClone) { + oldIndex = oldParentClone.children.indexOf(oldItemClone) + oldParentClone.children.splice(oldIndex, 1) + removeRootAction.payload = removedRootClone + removeRootAction.payload.createIndex() + } + } - await this.diffItem(oldItem, createdItem) - hasNewActions = true - break + // Retract or Mutate "New" (Created) side + if (createdItem === createdRoot) { + this.result.CREATE.retract(createRootAction) + } else { + const createdRootClone = createdRoot.copy(true) + const newParentClone = createdRootClone.findItem( + ItemType.FOLDER, + createdItem.parentId + ) as Folder + const createdItemClone = createdRootClone.findItem( + createdItem.type, + createdItem.id + ) + if (newParentClone && createdItemClone) { + newIndex = newParentClone.children.indexOf(createdItemClone) + newParentClone.children.splice(newIndex, 1) + createRootAction.payload = createdRootClone + createRootAction.payload.createIndex() + } } + + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem, + index: newIndex ?? createRootAction.index, + oldIndex: oldIndex ?? removeRootAction.index, + }) + + await this.diffItem(oldItem, createdItem) + hasNewActions = true + break } } From 19ad39ba73ea6cbf6bd1ca3925c211b6bba705ca Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 13:35:14 +0100 Subject: [PATCH 33/53] fix(Scanner): Reduce time complexity of findMoves Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 9503544ab5..e65d74c5a8 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -228,7 +228,7 @@ export default class Scanner }` for (const action of removeActions) { - const indexSubtree = async (item: TItem) => { + const indexSubtree = async(item: TItem) => { const key = getFuzzyKey(item) removedFuzzyMap.set( key, @@ -253,6 +253,9 @@ export default class Scanner } } + allCreatedItems + .sort((a, b) => b.item.count() - a.item.count()) + // 2. Match ALL created items (roots + descendants) against removed pool for (const createdEntry of allCreatedItems) { await yieldToEventLoop() @@ -265,6 +268,12 @@ export default class Scanner const matches = potentialMatches.filter((m) => this.mergeable(m.item, createdItem) ) + + // Heuristic: Prefer matches that have more descendants + matches.sort((a, b) => { + return b.item.count() - a.item.count() + }) + if (createdItem.type === 'folder' && !this.hasCache) { matches.sort( (a, b) => From 01a4c7b77c45f57752c8755f582a3a3469834332 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 16:01:52 +0100 Subject: [PATCH 34/53] fix(Scanner): Reduce time complexity of findMoves Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 89 ++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index e65d74c5a8..a59c98da18 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -212,35 +212,32 @@ export default class Scanner if (createActions.length === 0 || removeActions.length === 0) break - // 1. Build Fuzzy Indices for both REMOVE and CREATE subtrees (O(N)) - const removedFuzzyMap = new Map< - string, - { rootAction: RemoveAction; item: TItem }[] - >() - const allCreatedItems: { - rootAction: CreateAction - item: TItem - }[] = [] - - const getFuzzyKey = (item: TItem) => - `${item.type}_${item.title}_${ - item.type === 'bookmark' ? (item as Bookmark).url : '' - }` + // Build Multi-Signal Fuzzy Index (O(N)) + const removedFuzzyMap = new Map; item: TItem }[]>() + const allCreatedItems: { rootAction: CreateAction; item: TItem }[] = [] - for (const action of removeActions) { - const indexSubtree = async(item: TItem) => { - const key = getFuzzyKey(item) - removedFuzzyMap.set( - key, - (removedFuzzyMap.get(key) || []).concat({ - rootAction: action, - item, - }) - ) + const addToFuzzyIndex = (item: TItem, action: RemoveAction) => { + const keys = new Set() + // Signal 1: Full signature (Type + Title + URL) + keys.add(`${item.type}_${item.title}_${item.type === 'bookmark' ? (item as Bookmark).url : ''}`) + + // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) + if (item.title) keys.add(`${item.type}_title_${item.title}`) + + // Signal 3: URL only (Handles Title changes for bookmarks) + if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) + + keys.add(item.type) + + for (const key of keys) { + removedFuzzyMap.set(key, (removedFuzzyMap.get(key) || []).concat({ rootAction: action, item })) } - await indexSubtree(action.payload) + } + + for (const action of removeActions) { + await addToFuzzyIndex(action.payload, action) if (action.payload instanceof Folder) { - await action.payload.traverse(indexSubtree) + await action.payload.traverse((child) => addToFuzzyIndex(child, action)) } } @@ -260,28 +257,44 @@ export default class Scanner for (const createdEntry of allCreatedItems) { await yieldToEventLoop() const { rootAction: createRootAction, item: createdItem } = createdEntry - const fuzzyKey = getFuzzyKey(createdItem) - const potentialMatches = removedFuzzyMap.get(fuzzyKey) - if (!potentialMatches) continue + // Gather potential matches from all signals + const searchKeys = [ + `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, + `${createdItem.type}_title_${createdItem.title}`, + ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), + createdItem.type + ] + + // Collect unique potential matches from all signals + const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() + for (const key of searchKeys) { + const list = removedFuzzyMap.get(key) + if (list && list.filter((m) => this.mergeable(m.item, createdItem)).length) { + list.forEach(m => potentialSet.add(m)) + break + } + } + + if (potentialSet.size === 0) { + continue + } - const matches = potentialMatches.filter((m) => + const matches = Array.from(potentialSet).filter((m) => this.mergeable(m.item, createdItem) ) // Heuristic: Prefer matches that have more descendants + // In case we have no cache: Calculate similarity and sore by it matches.sort((a, b) => { + if (createdItem.type === 'folder' && !this.hasCache) { + const simA = a.item.childrenSimilarity(createdItem) + const simB = b.item.childrenSimilarity(createdItem) + if (simA !== simB) return simB - simA + } return b.item.count() - a.item.count() }) - if (createdItem.type === 'folder' && !this.hasCache) { - matches.sort( - (a, b) => - b.item.childrenSimilarity(createdItem) - - a.item.childrenSimilarity(createdItem) - ) - } - if (matches.length === 0) { continue } From 4423a3c301f1468337ff71030b093585709700b1 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 16:58:12 +0100 Subject: [PATCH 35/53] fix(Caching): Make sure that created bookmarks have the correct location tag Signed-off-by: Marcel Klehr --- src/lib/adapters/Caching.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/adapters/Caching.ts b/src/lib/adapters/Caching.ts index 4a49ee4497..63be224c9a 100644 --- a/src/lib/adapters/Caching.ts +++ b/src/lib/adapters/Caching.ts @@ -73,7 +73,7 @@ export default class CachingAdapter implements Adapter, BulkImportResource):Promise { Logger.log('CREATE', bm) - bm = bm.copy() + bm = bm.copyWithLocation(true, this.location) bm.id = ++this.highestId const foundFolder = this.bookmarksCache.findFolder(bm.parentId) if (!foundFolder) { From bf977aebee79fe8fa6780a35ee2acbd04475cf76 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 17:09:23 +0100 Subject: [PATCH 36/53] fix(Unidirectional): Use source MOVEs for revertPlan instead of target MOVEs since we're no longer using the Cache in Unidirectional it doesn't matter and is cleaner Signed-off-by: Marcel Klehr --- src/lib/strategies/Unidirectional.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 3fa25f2b0c..75ebc86edc 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -392,15 +392,9 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) await Parallel.each( - targetScanResult.MOVE.getActions(), + sourceScanResult.MOVE.getActions(), async(action) => { - const payload = action.payload.cloneWithLocation( - false, - action.oldItem.location - ) - payload.id = action.oldItem.id - payload.parentId = action.oldItem.parentId - + const payload = action.payload.clone(false) slavePlan.MOVE.commit({ type: ActionType.MOVE, payload }) // no oldItem, because we want to map the id after having executed the CREATEs }, ACTION_CONCURRENCY From fb55c676ab8b2e023faf0e0caa469857afe818cb Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 18:16:34 +0100 Subject: [PATCH 37/53] fix(Unidirectional): Only do a single Scanner pass to improve performance Signed-off-by: Marcel Klehr --- src/lib/Diff.ts | 2 +- src/lib/strategies/Unidirectional.ts | 120 ++++++++++----------------- 2 files changed, 44 insertions(+), 78 deletions(-) diff --git a/src/lib/Diff.ts b/src/lib/Diff.ts index 40b7850ca3..4aea5816cd 100644 --- a/src/lib/Diff.ts +++ b/src/lib/Diff.ts @@ -508,5 +508,5 @@ export interface PlanRevert UPDATE: Diff> MOVE: Diff> REMOVE: Diff> - REORDER: Diff> + REORDER: Diff> } diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 75ebc86edc..875bd11341 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -13,6 +13,7 @@ const ACTION_CONCURRENCY = 12 export default class UnidirectionalSyncProcess extends DefaultStrategy { protected direction: TItemLocation + protected scanResult: ScanResult protected revertPlan: PlanStage1< TItemLocation, TOppositeLocation @@ -35,8 +36,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { const members = [] // Stage 0 if (!this.revertPlan && this.actionsPlanned === 0) { - members.push('localScanResult') - members.push('serverScanResult') + members.push('scanResult') } // Stage 1 @@ -85,18 +85,27 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { } } - async getDiffs(): Promise<{ - localScanResult: ScanResult - serverScanResult: ScanResult - }> { + async getDiff(): Promise> { const mappingsSnapshot = this.mappings.getSnapshot() const newMappings = [] - const localScanner = new Scanner( - this.serverTreeRoot, - this.localTreeRoot, + const slaveTree = + this.direction === ItemLocation.SERVER + ? this.serverTreeRoot + : this.localTreeRoot + const masterTree = + this.direction === ItemLocation.SERVER + ? this.localTreeRoot + : this.serverTreeRoot + const scanner = new Scanner( + slaveTree, + masterTree, // We can't rely on a cacheTree, thus we have to accept canMergeWith results as well - (serverItem, localItem) => { + (slaveItem, masterItem) => { + const localItem = + this.direction === ItemLocation.SERVER ? masterItem : slaveItem + const serverItem = + this.direction === ItemLocation.SERVER ? slaveItem : masterItem if (localItem.type !== serverItem.type) { return false } @@ -122,40 +131,10 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { false, false ) - const serverScanner = new Scanner( - this.localTreeRoot, - this.serverTreeRoot, - (localItem, serverItem) => { - if (serverItem.type !== localItem.type) { - return false - } - // If a bookmark's URL has changed we want to recreate it instead of updating it, because of Nextcloud Bookmarks' uniqueness constraints - if ( - serverItem.type === 'bookmark' && - localItem.type === 'bookmark' && - serverItem.url !== localItem.url - ) { - return false - } - if (serverItem.canMergeWith(localItem)) { - newMappings.push([localItem, serverItem]) - return true - } - if (Mappings.mappable(mappingsSnapshot, serverItem, localItem)) { - newMappings.push([localItem, serverItem]) - return true - } - return false - }, - this.hashSettings, - false, - false - ) Logger.log( 'Unidirectional: Calculating the diff between local and server trees' ) - const localScanResult = await localScanner.run() - const serverScanResult = await serverScanner.run() + const scanResult = await scanner.run() await Parallel.map( newMappings, ([localItem, serverItem]) => { @@ -164,7 +143,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { 1 ) - return { localScanResult, serverScanResult } + return scanResult } async loadChildren( @@ -197,11 +176,9 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { cacheTreeRoot: this.cacheTreeRoot, }) - if ((!this.localScanResult || !this.serverScanResult) && !this.revertPlan) { - const { localScanResult, serverScanResult } = await this.getDiffs() - Logger.log({ localScanResult, serverScanResult }) - this.localScanResult = localScanResult - this.serverScanResult = serverScanResult + if (!this.scanResult && !this.revertPlan) { + this.scanResult = await this.getDiff() + Logger.log({ scanResult: this.scanResult }) this.throttledProgressCb(0.45, 0) } @@ -209,27 +186,17 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { throw new CancelledSyncError() } - let sourceScanResult: ScanResult, - targetScanResult: ScanResult, - target: TResource + let target: TResource if (this.direction === ItemLocation.SERVER) { - sourceScanResult = this.localScanResult - targetScanResult = this.serverScanResult target = this.server } else { - sourceScanResult = this.serverScanResult - targetScanResult = this.localScanResult target = this.localTree } // First revert slave modifications if (!this.revertPlan) { - this.revertPlan = await this.revertDiff( - targetScanResult, - sourceScanResult, - this.direction - ) + this.revertPlan = await this.revertDiff(this.scanResult, this.direction) Logger.log({ revertPlan: this.revertPlan }) } @@ -292,7 +259,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { this.revertPlan, this.direction, this.revertDonePlan, - sourceScanResult.REORDER + this.scanResult.REORDER ) } @@ -309,7 +276,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { if ('orderFolder' in target && !this.revertReorders) { const mappingsSnapshot = this.mappings.getSnapshot() Logger.log('Mapping reorderings') - this.revertReorders = sourceScanResult.REORDER.map( + this.revertReorders = this.scanResult.REORDER.map( mappingsSnapshot, this.direction ) @@ -323,8 +290,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { } async revertDiff( - targetScanResult: ScanResult, - sourceScanResult: ScanResult, + scanResult: ScanResult, targetLocation: L1 ): Promise> { const mappingsSnapshot = this.mappings.getSnapshot() @@ -334,13 +300,13 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { UPDATE: new Diff(), MOVE: new Diff(), REMOVE: new Diff(), - REORDER: targetScanResult.REORDER.clone(), + REORDER: scanResult.REORDER.clone(), } - // Prepare slave plan for reversing slave changes + // Prepare slave plan for matching master state await Parallel.each( - sourceScanResult.CREATE.getActions(), + scanResult.CREATE.getActions(), async(action) => { // recreate it on slave resource otherwise const payload = await this.translateCompleteItem( @@ -363,7 +329,7 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) await Parallel.each( - targetScanResult.CREATE.getActions(), + scanResult.REMOVE.getActions(), async(action) => { slavePlan.REMOVE.commit({ ...action, type: ActionType.REMOVE }) }, @@ -371,28 +337,28 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) await Parallel.each( - targetScanResult.UPDATE.getActions(), + scanResult.UPDATE.getActions(), async(action) => { - const payload = action.oldItem.cloneWithLocation( + const payload = action.payload.cloneWithLocation( false, - action.payload.location + action.oldItem.location ) - payload.id = action.payload.id - payload.parentId = action.payload.parentId + payload.id = action.oldItem.id + payload.parentId = action.oldItem.parentId - const oldItem = action.payload.cloneWithLocation( + const oldItem = action.oldItem.cloneWithLocation( false, - action.oldItem.location + action.payload.location ) - oldItem.id = action.oldItem.id - oldItem.parentId = action.oldItem.parentId + oldItem.id = action.payload.id + oldItem.parentId = action.payload.parentId slavePlan.UPDATE.commit({ type: ActionType.UPDATE, payload, oldItem }) }, ACTION_CONCURRENCY ) await Parallel.each( - sourceScanResult.MOVE.getActions(), + scanResult.MOVE.getActions(), async(action) => { const payload = action.payload.clone(false) slavePlan.MOVE.commit({ type: ActionType.MOVE, payload }) // no oldItem, because we want to map the id after having executed the CREATEs From b9de31a9761e344c78f4369e6fcdcdcd01d6f61f Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 30 Dec 2025 19:22:03 +0100 Subject: [PATCH 38/53] fix(Scanner): yieldToEventLoop less often Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index a59c98da18..b07516ce25 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -253,9 +253,14 @@ export default class Scanner allCreatedItems .sort((a, b) => b.item.count() - a.item.count()) - // 2. Match ALL created items (roots + descendants) against removed pool + // Match ALL created items (roots + descendants) against removed pool + let i = 0 for (const createdEntry of allCreatedItems) { - await yieldToEventLoop() + if (i === 100) { + i = 0 + await yieldToEventLoop() + } + i++ const { rootAction: createRootAction, item: createdItem } = createdEntry // Gather potential matches from all signals From 18c5354dfd8bb29350db56003dff262689c718f6 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 06:50:42 +0100 Subject: [PATCH 39/53] fix(Scanner): Try to improve time complexity of Scanner Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 114 +++++++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index b07516ce25..8977bea9ab 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -1,4 +1,3 @@ -import * as Parallel from 'async-parallel' import Diff, { ActionType, CreateAction, MoveAction, RemoveAction, ReorderAction, UpdateAction } from './Diff' import { Bookmark, Folder, ItemLocation, ItemType, TItem, TItemLocation } from './Tree' import Logger from './Logger' @@ -121,58 +120,57 @@ export default class Scanner const stillUnmatched = new Set(newFolder.children) // (using map here, because 'each' doesn't provide indices) - await Parallel.map( - oldFolder.children, - async(old, index) => { - const key = `${old.type}_${old.title}` - const potentialMatches = unmatchedMap.get(key) - let newItem = null - if (potentialMatches) { - const matchIndex = potentialMatches.findIndex((m) => - this.mergeable(old, m) - ) - if (matchIndex !== -1) { - newItem = potentialMatches.splice(matchIndex, 1)[0] - stillUnmatched.delete(newItem) - } - } - // we found an item in the new folder that matches the one in the old folder - if (newItem) { - await this.diffItem(old, newItem) - return + let index = 0 + for (const old of oldFolder.children) { + const key = `${old.type}_${old.title}` + const potentialMatches = unmatchedMap.get(key) + let newItem = null + if (potentialMatches) { + const matchIndex = potentialMatches.findIndex((m) => + this.mergeable(old, m) + ) + if (matchIndex !== -1) { + newItem = potentialMatches.splice(matchIndex, 1)[0] + stillUnmatched.delete(newItem) } + } + // we found an item in the new folder that matches the one in the old folder + if (newItem) { + await this.diffItem(old, newItem) + index++ + continue + } - if (newFolder.isRoot && newFolder.location === ItemLocation.LOCAL) { - // We can't remove root folders locally - return - } + if (newFolder.isRoot && newFolder.location === ItemLocation.LOCAL) { + // We can't remove root folders locally + index++ + continue + } - this.result.REMOVE.commit({ - type: ActionType.REMOVE, - payload: old, - index, - }) - }, - 1 - ) + this.result.REMOVE.commit({ + type: ActionType.REMOVE, + payload: old, + index, + }) + + index++ + } // created Items - // (using map here, because 'each' doesn't provide indices) - await Parallel.map( - Array.from(stillUnmatched.values()), - async(newChild) => { - if (oldFolder.isRoot && oldFolder.location === ItemLocation.LOCAL) { - // We can't create root folders locally - return - } - this.result.CREATE.commit({ - type: ActionType.CREATE, - payload: newChild, - index: newFolder.children.findIndex((child) => child === newChild), - }) - }, - 1 - ) + const childToIndex = new Map, number>() + newFolder.children.forEach((child, i) => childToIndex.set(child, i)) + + for (const newChild of stillUnmatched) { + if (oldFolder.isRoot && oldFolder.location === ItemLocation.LOCAL) { + // We can't create root folders locally + continue + } + this.result.CREATE.commit({ + type: ActionType.CREATE, + payload: newChild, + index: childToIndex.get(newChild), + }) + } } async diffBookmark(oldBookmark:Bookmark, newBookmark:Bookmark):Promise { @@ -229,8 +227,14 @@ export default class Scanner keys.add(item.type) + const element = { rootAction: action, item } // outside the loop so we can later use Set#has for (const key of keys) { - removedFuzzyMap.set(key, (removedFuzzyMap.get(key) || []).concat({ rootAction: action, item })) + let list = removedFuzzyMap.get(key) + if (!list) { + list = [] + removedFuzzyMap.set(key, list) + } + list.push(element) } } @@ -275,8 +279,16 @@ export default class Scanner const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() for (const key of searchKeys) { const list = removedFuzzyMap.get(key) - if (list && list.filter((m) => this.mergeable(m.item, createdItem)).length) { - list.forEach(m => potentialSet.add(m)) + if (!list) continue + let hasMergeable = false + for (const m of list) { + if (this.mergeable(m.item, createdItem)) { + hasMergeable = true + break + } + } + if (hasMergeable) { + list.forEach((m) => potentialSet.add(m)) break } } From 55c68919694c7928800a334a49291d6779c2db03 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 07:32:03 +0100 Subject: [PATCH 40/53] fix(Scanner): Use clone instead of copy Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 8977bea9ab..b9de17bcb9 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -326,7 +326,7 @@ export default class Scanner if (oldItem === removedRoot) { this.result.REMOVE.retract(removeRootAction) } else { - const removedRootClone = removedRoot.copy(true) + const removedRootClone = removedRoot.clone(true) const oldParentClone = removedRootClone.findItem( ItemType.FOLDER, oldItem.parentId @@ -347,7 +347,7 @@ export default class Scanner if (createdItem === createdRoot) { this.result.CREATE.retract(createRootAction) } else { - const createdRootClone = createdRoot.copy(true) + const createdRootClone = createdRoot.clone(true) const newParentClone = createdRootClone.findItem( ItemType.FOLDER, createdItem.parentId @@ -373,6 +373,8 @@ export default class Scanner }) await this.diffItem(oldItem, createdItem) + // After diffing we need to start from scratch + // to make sure we match the newly created actions hasNewActions = true break } From e7bbfa9c5f355e06a5857f303c1895d27c42c7de Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 08:11:07 +0100 Subject: [PATCH 41/53] fix(Scanner): Try to speed up findMoves by using a single pass queue Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 266 ++++++++++++++++++++++----------------------- 1 file changed, 128 insertions(+), 138 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index b9de17bcb9..59053dc698 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -200,170 +200,158 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - let hasNewActions = true - - while (hasNewActions) { - hasNewActions = false - - const createActions = this.result.CREATE.getActions() - const removeActions = this.result.REMOVE.getActions() - - if (createActions.length === 0 || removeActions.length === 0) break - - // Build Multi-Signal Fuzzy Index (O(N)) - const removedFuzzyMap = new Map; item: TItem }[]>() - const allCreatedItems: { rootAction: CreateAction; item: TItem }[] = [] - - const addToFuzzyIndex = (item: TItem, action: RemoveAction) => { - const keys = new Set() - // Signal 1: Full signature (Type + Title + URL) - keys.add(`${item.type}_${item.title}_${item.type === 'bookmark' ? (item as Bookmark).url : ''}`) - - // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) - if (item.title) keys.add(`${item.type}_title_${item.title}`) - - // Signal 3: URL only (Handles Title changes for bookmarks) - if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) - - keys.add(item.type) - - const element = { rootAction: action, item } // outside the loop so we can later use Set#has - for (const key of keys) { - let list = removedFuzzyMap.get(key) - if (!list) { - list = [] - removedFuzzyMap.set(key, list) - } - list.push(element) + const handledRemovals = new Set>() + const handledCreations = new Set>() + const removedFuzzyMap = new Map; item: TItem }>>() + const creationQueue: { rootAction: CreateAction; item: TItem }[] = [] + + // Initial Indexing + + const addToRemovedFuzzyMap = ( + action: RemoveAction, + item: TItem + ) => { + const keys = new Set() + // Signal 1: Full signature (Type + Title + URL) + keys.add( + `${item.type}_${item.title}_${ + item.type === 'bookmark' ? (item as Bookmark).url : '' + }` + ) + + // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) + if (item.title) keys.add(`${item.type}_title_${item.title}`) + + // Signal 3: URL only (Handles Title changes for bookmarks) + if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) + + keys.add(item.type) + + const element = { rootAction: action, item } // outside the loop so we can later use Set#has + for (const key of keys) { + let list = removedFuzzyMap.get(key) + if (!list) { + list = new Set() + removedFuzzyMap.set(key, list) } + list.add(element) } + } - for (const action of removeActions) { - await addToFuzzyIndex(action.payload, action) - if (action.payload instanceof Folder) { - await action.payload.traverse((child) => addToFuzzyIndex(child, action)) - } + for (const action of this.result.REMOVE.getActions()) { + addToRemovedFuzzyMap(action, action.payload) + if (action.payload instanceof Folder) { + await action.payload.traverse(item => addToRemovedFuzzyMap(action, item)) } + } + + const enqueueNewCreations = async() => { + const currentActions = this.result.CREATE.getActions() + const newEntries: typeof creationQueue = [] + for (const action of currentActions) { + // Only enqueue items we haven't seen before + if (handledCreations.has(action.payload)) continue - for (const action of createActions) { - allCreatedItems.push({ rootAction: action, item: action.payload }) + // We use a property check or external tracking to avoid re-enqueuing the same Action object + // but for simplicity here we assume the queue only grows from new diffItem calls + newEntries.push({ rootAction: action, item: action.payload }) if (action.payload instanceof Folder) { - await action.payload.traverse((child) => { - allCreatedItems.push({ rootAction: action, item: child }) + await action.payload.traverse(child => { + newEntries.push({ rootAction: action, item: child }) }) } } + newEntries.sort((a, b) => b.item.count() - a.item.count()) + creationQueue.push(...newEntries) + } - allCreatedItems - .sort((a, b) => b.item.count() - a.item.count()) - - // Match ALL created items (roots + descendants) against removed pool - let i = 0 - for (const createdEntry of allCreatedItems) { - if (i === 100) { - i = 0 - await yieldToEventLoop() - } - i++ - const { rootAction: createRootAction, item: createdItem } = createdEntry - - // Gather potential matches from all signals - const searchKeys = [ - `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, - `${createdItem.type}_title_${createdItem.title}`, - ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), - createdItem.type - ] - - // Collect unique potential matches from all signals - const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() - for (const key of searchKeys) { - const list = removedFuzzyMap.get(key) - if (!list) continue - let hasMergeable = false - for (const m of list) { - if (this.mergeable(m.item, createdItem)) { - hasMergeable = true - break - } - } - if (hasMergeable) { - list.forEach((m) => potentialSet.add(m)) - break - } - } - - if (potentialSet.size === 0) { - continue - } - - const matches = Array.from(potentialSet).filter((m) => - this.mergeable(m.item, createdItem) + await enqueueNewCreations() + + // 2. Process queue in a single pass + let iterations = 0 + while (creationQueue.length > 0) { + const entry = creationQueue.shift() + const { rootAction: createRootAction, item: createdItem } = entry + + if (handledCreations.has(createdItem)) continue + if (++iterations % 1000 === 0) await yieldToEventLoop() + + const searchKeys = [ + `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, + `${createdItem.type}_title_${createdItem.title}`, + ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), + createdItem.type + ] + + let bestMatch = null + for (const key of searchKeys) { + const list = removedFuzzyMap.get(key) + if (!list) continue + const matches = Array.from(list).filter( + (m) => + !handledRemovals.has(m.item) && this.mergeable(m.item, createdItem) ) - - // Heuristic: Prefer matches that have more descendants - // In case we have no cache: Calculate similarity and sore by it - matches.sort((a, b) => { - if (createdItem.type === 'folder' && !this.hasCache) { - const simA = a.item.childrenSimilarity(createdItem) - const simB = b.item.childrenSimilarity(createdItem) - if (simA !== simB) return simB - simA - } - return b.item.count() - a.item.count() - }) - - if (matches.length === 0) { - continue + if (matches.length > 0) { + // Heuristic: Prefer matches that have more descendants + // In case we have no cache: Calculate similarity and sore by it + matches.sort((a, b) => { + if (createdItem.type === 'folder' && !this.hasCache) { + const simA = a.item.childrenSimilarity(createdItem) + const simB = b.item.childrenSimilarity(createdItem) + if (simA !== simB) return simB - simA + } + return b.item.count() - a.item.count() + }) + bestMatch = matches[0] + break } + } - const { rootAction: removeRootAction, item: oldItem } = matches[0] + if (bestMatch) { + const { rootAction: removeRootAction, item: oldItem } = bestMatch const removedRoot = removeRootAction.payload const createdRoot = createRootAction.payload let oldIndex, newIndex - // Retract or Mutate "Old" (Removed) side + // Handle the "Old" (Removed) side if (oldItem === removedRoot) { this.result.REMOVE.retract(removeRootAction) } else { - const removedRootClone = removedRoot.clone(true) - const oldParentClone = removedRootClone.findItem( - ItemType.FOLDER, - oldItem.parentId - ) as Folder - const oldItemClone = removedRootClone.findItem( - oldItem.type, - oldItem.id - ) - if (oldParentClone && oldItemClone) { - oldIndex = oldParentClone.children.indexOf(oldItemClone) - oldParentClone.children.splice(oldIndex, 1) - removeRootAction.payload = removedRootClone - removeRootAction.payload.createIndex() + const clone = (removedRoot as Folder).clone(true) + const parentClone = clone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder + const itemClone = clone.findItem(oldItem.type, oldItem.id) + if (parentClone && itemClone) { + oldIndex = parentClone.children.indexOf(itemClone) + parentClone.children.splice(oldIndex, 1) + clone.createIndex() + removeRootAction.payload = clone } } - // Retract or Mutate "New" (Created) side + // Handle the "New" (Created) side if (createdItem === createdRoot) { this.result.CREATE.retract(createRootAction) } else { - const createdRootClone = createdRoot.clone(true) - const newParentClone = createdRootClone.findItem( - ItemType.FOLDER, - createdItem.parentId - ) as Folder - const createdItemClone = createdRootClone.findItem( - createdItem.type, - createdItem.id - ) - if (newParentClone && createdItemClone) { - newIndex = newParentClone.children.indexOf(createdItemClone) - newParentClone.children.splice(newIndex, 1) - createRootAction.payload = createdRootClone - createRootAction.payload.createIndex() + const clone = (createdRoot as Folder).clone(true) + const parentClone = clone.findItem(ItemType.FOLDER, createdItem.parentId) as Folder + const itemClone = clone.findItem(createdItem.type, createdItem.id) + if (parentClone && itemClone) { + newIndex = parentClone.children.indexOf(itemClone) + parentClone.children.splice(newIndex, 1) + clone.createIndex() + createRootAction.payload = clone } } + // Mark matched branches as handled + const markHandled = async(item: TItem, set: Set>) => { + set.add(item) + if (item instanceof Folder) await item.traverse(child => set.add(child)) + } + await markHandled(oldItem, handledRemovals) + await markHandled(createdItem, handledCreations) + this.result.MOVE.commit({ type: ActionType.MOVE, payload: createdItem, @@ -372,11 +360,13 @@ export default class Scanner oldIndex: oldIndex ?? removeRootAction.index, }) + // Diff the matched items (which might discover more creates/removes) + const prevCreateCount = this.result.CREATE.getActions().length await this.diffItem(oldItem, createdItem) - // After diffing we need to start from scratch - // to make sure we match the newly created actions - hasNewActions = true - break + + if (this.result.CREATE.getActions().length > prevCreateCount) { + await enqueueNewCreations() + } } } From 12d319211bacac150f3fb635caded3db96b881f7 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 08:29:00 +0100 Subject: [PATCH 42/53] fix(persistence): Fix SyncProcess#toJSONAsync Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index 099ced7e88..ef5230028d 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -1582,10 +1582,10 @@ export default class SyncProcess { Object.entries(this) .filter(([key]) => membersToPersist.includes(key)), async([key, value]) => { - if (value.toJSONAsync) { + if (value && value.toJSONAsync) { return [key, await value.toJSONAsync()] } - if (value.toJSON) { + if (value && value.toJSON) { await yieldToEventLoop() return [key, value.toJSON()] } From b47446414bf8e3ee1dee97877f329f9d8991d93e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 08:30:08 +0100 Subject: [PATCH 43/53] fix(Scanner): Fix findMoves Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 59053dc698..d910ba3ed8 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -261,8 +261,8 @@ export default class Scanner }) } } - newEntries.sort((a, b) => b.item.count() - a.item.count()) creationQueue.push(...newEntries) + creationQueue.sort((a, b) => b.item.count() - a.item.count()) } await enqueueNewCreations() From 399890481f02f852c792a4dc2c8c110e33106631 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 08:51:35 +0100 Subject: [PATCH 44/53] fix(Scanner): Fix findMoves by adding folder count to sorting Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 4 ++-- src/lib/Tree.ts | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index d910ba3ed8..cbd5139e07 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -262,7 +262,7 @@ export default class Scanner } } creationQueue.push(...newEntries) - creationQueue.sort((a, b) => b.item.count() - a.item.count()) + creationQueue.sort((a, b) => (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count())) } await enqueueNewCreations() @@ -300,7 +300,7 @@ export default class Scanner const simB = b.item.childrenSimilarity(createdItem) if (simA !== simB) return simB - simA } - return b.item.count() - a.item.count() + return (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count()) }) bestMatch = matches[0] break diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 5170229d24..5ff6165655 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -224,6 +224,10 @@ export class Bookmark { return 1 } + countFolders(): number { + return 0 + } + inspect(depth = 0): string { return ( Array(depth < 0 ? 0 : depth) From 7537d0e34fb175100dc6f05d8a3ce61b141f6d65 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 09:10:35 +0100 Subject: [PATCH 45/53] Revert "fix(Scanner): Try to speed up findMoves by using a single pass queue" This reverts commit e7bbfa9c Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 220 ++++++++++++++++++++++----------------------- 1 file changed, 108 insertions(+), 112 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index cbd5139e07..91eab22dd0 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -200,121 +200,127 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - const handledRemovals = new Set>() - const handledCreations = new Set>() - const removedFuzzyMap = new Map; item: TItem }>>() - const creationQueue: { rootAction: CreateAction; item: TItem }[] = [] - - // Initial Indexing - - const addToRemovedFuzzyMap = ( - action: RemoveAction, - item: TItem - ) => { - const keys = new Set() - // Signal 1: Full signature (Type + Title + URL) - keys.add( - `${item.type}_${item.title}_${ - item.type === 'bookmark' ? (item as Bookmark).url : '' - }` - ) - - // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) - if (item.title) keys.add(`${item.type}_title_${item.title}`) - - // Signal 3: URL only (Handles Title changes for bookmarks) - if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) - - keys.add(item.type) - - const element = { rootAction: action, item } // outside the loop so we can later use Set#has - for (const key of keys) { - let list = removedFuzzyMap.get(key) - if (!list) { - list = new Set() - removedFuzzyMap.set(key, list) + let hasNewActions = true + + while (hasNewActions) { + hasNewActions = false + + const createActions = this.result.CREATE.getActions() + const removeActions = this.result.REMOVE.getActions() + + if (createActions.length === 0 || removeActions.length === 0) break + + // Build Multi-Signal Fuzzy Index (O(N)) + const removedFuzzyMap = new Map; item: TItem }[]>() + const allCreatedItems: { rootAction: CreateAction; item: TItem }[] = [] + + const addToFuzzyIndex = (item: TItem, action: RemoveAction) => { + const keys = new Set() + // Signal 1: Full signature (Type + Title + URL) + keys.add(`${item.type}_${item.title}_${item.type === 'bookmark' ? (item as Bookmark).url : ''}`) + + // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) + if (item.title) keys.add(`${item.type}_title_${item.title}`) + + // Signal 3: URL only (Handles Title changes for bookmarks) + if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) + + keys.add(item.type) + + const element = { rootAction: action, item } // outside the loop so we can later use Set#has + for (const key of keys) { + let list = removedFuzzyMap.get(key) + if (!list) { + list = [] + removedFuzzyMap.set(key, list) + } + list.push(element) } - list.add(element) } - } - for (const action of this.result.REMOVE.getActions()) { - addToRemovedFuzzyMap(action, action.payload) - if (action.payload instanceof Folder) { - await action.payload.traverse(item => addToRemovedFuzzyMap(action, item)) + for (const action of removeActions) { + await addToFuzzyIndex(action.payload, action) + if (action.payload instanceof Folder) { + await action.payload.traverse((child) => addToFuzzyIndex(child, action)) + } } - } - - const enqueueNewCreations = async() => { - const currentActions = this.result.CREATE.getActions() - const newEntries: typeof creationQueue = [] - for (const action of currentActions) { - // Only enqueue items we haven't seen before - if (handledCreations.has(action.payload)) continue - // We use a property check or external tracking to avoid re-enqueuing the same Action object - // but for simplicity here we assume the queue only grows from new diffItem calls - newEntries.push({ rootAction: action, item: action.payload }) + for (const action of createActions) { + allCreatedItems.push({ rootAction: action, item: action.payload }) if (action.payload instanceof Folder) { - await action.payload.traverse(child => { - newEntries.push({ rootAction: action, item: child }) + await action.payload.traverse((child) => { + allCreatedItems.push({ rootAction: action, item: child }) }) } } - creationQueue.push(...newEntries) - creationQueue.sort((a, b) => (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count())) - } - await enqueueNewCreations() - - // 2. Process queue in a single pass - let iterations = 0 - while (creationQueue.length > 0) { - const entry = creationQueue.shift() - const { rootAction: createRootAction, item: createdItem } = entry - - if (handledCreations.has(createdItem)) continue - if (++iterations % 1000 === 0) await yieldToEventLoop() - - const searchKeys = [ - `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, - `${createdItem.type}_title_${createdItem.title}`, - ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), - createdItem.type - ] - - let bestMatch = null - for (const key of searchKeys) { - const list = removedFuzzyMap.get(key) - if (!list) continue - const matches = Array.from(list).filter( - (m) => - !handledRemovals.has(m.item) && this.mergeable(m.item, createdItem) - ) - if (matches.length > 0) { - // Heuristic: Prefer matches that have more descendants - // In case we have no cache: Calculate similarity and sore by it - matches.sort((a, b) => { - if (createdItem.type === 'folder' && !this.hasCache) { - const simA = a.item.childrenSimilarity(createdItem) - const simB = b.item.childrenSimilarity(createdItem) - if (simA !== simB) return simB - simA + allCreatedItems + .sort((a, b) => b.item.count() - a.item.count()) + + // Match ALL created items (roots + descendants) against removed pool + let iterations = 0 + for (const createdEntry of allCreatedItems) { + if (++iterations % 1000 === 0) { + await yieldToEventLoop() + } + const { rootAction: createRootAction, item: createdItem } = createdEntry + + // Gather potential matches from all signals + const searchKeys = [ + `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, + `${createdItem.type}_title_${createdItem.title}`, + ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), + createdItem.type + ] + + // Collect unique potential matches from all signals + const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() + for (const key of searchKeys) { + const list = removedFuzzyMap.get(key) + if (!list) continue + let hasMergeable = false + for (const m of list) { + if (this.mergeable(m.item, createdItem)) { + hasMergeable = true + break } - return (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count()) - }) - bestMatch = matches[0] - break + } + if (hasMergeable) { + list.forEach((m) => potentialSet.add(m)) + break + } } - } - if (bestMatch) { - const { rootAction: removeRootAction, item: oldItem } = bestMatch + if (potentialSet.size === 0) { + continue + } + + const matches = Array.from(potentialSet).filter((m) => + this.mergeable(m.item, createdItem) + ) + + // Heuristic: Prefer matches that have more descendants + // In case we have no cache: Calculate similarity and sore by it + matches.sort((a, b) => { + if (createdItem.type === 'folder' && !this.hasCache) { + const simA = a.item.childrenSimilarity(createdItem) + const simB = b.item.childrenSimilarity(createdItem) + if (simA !== simB) return simB - simA + } + return (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count()) + }) + + if (matches.length === 0) { + continue + } + + const { rootAction: removeRootAction, item: oldItem } = matches[0] const removedRoot = removeRootAction.payload const createdRoot = createRootAction.payload let oldIndex, newIndex - // Handle the "Old" (Removed) side + // Retract or Mutate "Old" (Removed) side if (oldItem === removedRoot) { this.result.REMOVE.retract(removeRootAction) } else { @@ -329,7 +335,7 @@ export default class Scanner } } - // Handle the "New" (Created) side + // Retract or Mutate "New" (Created) side if (createdItem === createdRoot) { this.result.CREATE.retract(createRootAction) } else { @@ -344,14 +350,6 @@ export default class Scanner } } - // Mark matched branches as handled - const markHandled = async(item: TItem, set: Set>) => { - set.add(item) - if (item instanceof Folder) await item.traverse(child => set.add(child)) - } - await markHandled(oldItem, handledRemovals) - await markHandled(createdItem, handledCreations) - this.result.MOVE.commit({ type: ActionType.MOVE, payload: createdItem, @@ -360,13 +358,11 @@ export default class Scanner oldIndex: oldIndex ?? removeRootAction.index, }) - // Diff the matched items (which might discover more creates/removes) - const prevCreateCount = this.result.CREATE.getActions().length await this.diffItem(oldItem, createdItem) - - if (this.result.CREATE.getActions().length > prevCreateCount) { - await enqueueNewCreations() - } + // After diffing we need to start from scratch + // to make sure we match the newly created actions + hasNewActions = true + break } } From 204bf1182a465993f2d122765afdb1336a79923a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 13:42:48 +0100 Subject: [PATCH 46/53] Revert "fix(Scanner): Reduce time complexity of findMoves to improve performance" This reverts commit 2caffc58 Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 271 ++++++++++++++++++++------------------------- 1 file changed, 118 insertions(+), 153 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 91eab22dd0..615a959786 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -199,170 +199,135 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - - let hasNewActions = true - - while (hasNewActions) { - hasNewActions = false - - const createActions = this.result.CREATE.getActions() - const removeActions = this.result.REMOVE.getActions() - - if (createActions.length === 0 || removeActions.length === 0) break - - // Build Multi-Signal Fuzzy Index (O(N)) - const removedFuzzyMap = new Map; item: TItem }[]>() - const allCreatedItems: { rootAction: CreateAction; item: TItem }[] = [] - - const addToFuzzyIndex = (item: TItem, action: RemoveAction) => { - const keys = new Set() - // Signal 1: Full signature (Type + Title + URL) - keys.add(`${item.type}_${item.title}_${item.type === 'bookmark' ? (item as Bookmark).url : ''}`) - - // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) - if (item.title) keys.add(`${item.type}_title_${item.title}`) - - // Signal 3: URL only (Handles Title changes for bookmarks) - if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) - - keys.add(item.type) - - const element = { rootAction: action, item } // outside the loop so we can later use Set#has - for (const key of keys) { - let list = removedFuzzyMap.get(key) - if (!list) { - list = [] - removedFuzzyMap.set(key, list) - } - list.push(element) - } - } - - for (const action of removeActions) { - await addToFuzzyIndex(action.payload, action) - if (action.payload instanceof Folder) { - await action.payload.traverse((child) => addToFuzzyIndex(child, action)) - } - } - - for (const action of createActions) { - allCreatedItems.push({ rootAction: action, item: action.payload }) - if (action.payload instanceof Folder) { - await action.payload.traverse((child) => { - allCreatedItems.push({ rootAction: action, item: child }) - }) - } - } - - allCreatedItems - .sort((a, b) => b.item.count() - a.item.count()) - - // Match ALL created items (roots + descendants) against removed pool + let createActions + let removeActions + let reconciled = true + + // As soon as one match is found, action list is updated and search is started with the new list + // repeat until no rewrites happen anymore + while (reconciled) { + reconciled = false + let createAction: CreateAction, removeAction: RemoveAction + + // First find direct matches (avoids glitches when folders and their contents have been moved) + createActions = this.result.CREATE.getActions() let iterations = 0 - for (const createdEntry of allCreatedItems) { + while (!reconciled && (createAction = createActions.shift())) { + // give the browser time to breathe if (++iterations % 1000 === 0) { await yieldToEventLoop() } - const { rootAction: createRootAction, item: createdItem } = createdEntry - - // Gather potential matches from all signals - const searchKeys = [ - `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, - `${createdItem.type}_title_${createdItem.title}`, - ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), - createdItem.type - ] - - // Collect unique potential matches from all signals - const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() - for (const key of searchKeys) { - const list = removedFuzzyMap.get(key) - if (!list) continue - let hasMergeable = false - for (const m of list) { - if (this.mergeable(m.item, createdItem)) { - hasMergeable = true - break - } + const createdItem = createAction.payload + removeActions = this.result.REMOVE.getActions() + while (!reconciled && (removeAction = removeActions.shift())) { + // give the browser time to breathe + if (++iterations % 1000 === 0) { + await yieldToEventLoop() } - if (hasMergeable) { - list.forEach((m) => potentialSet.add(m)) - break + const removedItem = removeAction.payload + + if (this.mergeable(removedItem, createdItem) && + (removedItem.type !== 'folder' || + (!this.hasCache && removedItem.childrenSimilarity(createdItem) > 0.8))) { + this.result.CREATE.retract(createAction) + this.result.REMOVE.retract(removeAction) + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem: removedItem, + index: createAction.index, + oldIndex: removeAction.index + }) + reconciled = true + // Don't use the items from the action, but the ones in the actual tree to avoid using tree parts mutated by this algorithm (see below) + await this.diffItem(removedItem, createdItem) } } + } - if (potentialSet.size === 0) { - continue - } - - const matches = Array.from(potentialSet).filter((m) => - this.mergeable(m.item, createdItem) + // Then find descendant matches + createActions = this.result.CREATE.getActions() + createActions.sort((a, b) => + (b.payload.countFolders() * 1000 + b.payload.count()) - (a.payload.countFolders() * 1000 + a.payload.count()) + ) + while (!reconciled && (createAction = createActions.shift())) { + // give the browser time to breathe + await Promise.resolve() + const createdItem = createAction.payload + removeActions = this.result.REMOVE.getActions() + removeActions.sort((a, b) => + (b.payload.countFolders() * 1000 + b.payload.count()) - (a.payload.countFolders() * 1000 + a.payload.count()) ) - - // Heuristic: Prefer matches that have more descendants - // In case we have no cache: Calculate similarity and sore by it - matches.sort((a, b) => { - if (createdItem.type === 'folder' && !this.hasCache) { - const simA = a.item.childrenSimilarity(createdItem) - const simB = b.item.childrenSimilarity(createdItem) - if (simA !== simB) return simB - simA - } - return (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count()) - }) - - if (matches.length === 0) { - continue - } - - const { rootAction: removeRootAction, item: oldItem } = matches[0] - const removedRoot = removeRootAction.payload - const createdRoot = createRootAction.payload - - let oldIndex, newIndex - - // Retract or Mutate "Old" (Removed) side - if (oldItem === removedRoot) { - this.result.REMOVE.retract(removeRootAction) - } else { - const clone = (removedRoot as Folder).clone(true) - const parentClone = clone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder - const itemClone = clone.findItem(oldItem.type, oldItem.id) - if (parentClone && itemClone) { - oldIndex = parentClone.children.indexOf(itemClone) - parentClone.children.splice(oldIndex, 1) - clone.createIndex() - removeRootAction.payload = clone - } - } - - // Retract or Mutate "New" (Created) side - if (createdItem === createdRoot) { - this.result.CREATE.retract(createRootAction) - } else { - const clone = (createdRoot as Folder).clone(true) - const parentClone = clone.findItem(ItemType.FOLDER, createdItem.parentId) as Folder - const itemClone = clone.findItem(createdItem.type, createdItem.id) - if (parentClone && itemClone) { - newIndex = parentClone.children.indexOf(itemClone) - parentClone.children.splice(newIndex, 1) - clone.createIndex() - createRootAction.payload = clone + while (!reconciled && (removeAction = removeActions.shift())) { + // give the browser time to breathe + await Promise.resolve() + const removedItem = removeAction.payload + const oldItem = removedItem.findItemFilter( + createdItem.type, + item => this.mergeable(item, createdItem), + item => item.childrenSimilarity(createdItem) + ) + if (oldItem) { + let oldIndex + this.result.CREATE.retract(createAction) + if (oldItem === removedItem) { + this.result.REMOVE.retract(removeAction) + } else { + // We clone the item here, because we don't want to mutate all copies of this tree (item) + const removedItemClone = removedItem.copy(true) + const oldParentClone = removedItemClone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder + const oldItemClone = removedItemClone.findItem(oldItem.type, oldItem.id) + oldIndex = oldParentClone.children.indexOf(oldItemClone) + oldParentClone.children.splice(oldIndex, 1) + removeAction.payload = removedItemClone + removeAction.payload.createIndex() + } + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem, + index: createAction.index, + oldIndex: oldIndex || removeAction.index + }) + reconciled = true + if (oldItem.type === ItemType.FOLDER) { // TODO: Is this necessary? + await this.diffItem(oldItem, createdItem) + } + } else { + const newItem = createdItem.findItemFilter( + removedItem.type, + item => this.mergeable(removedItem, item), + item => item.childrenSimilarity(removedItem) + ) + let index + if (newItem) { + this.result.REMOVE.retract(removeAction) + if (newItem === createdItem) { + this.result.CREATE.retract(createAction) + } else { + // We clone the item here, because we don't want to mutate all copies of this tree (item) + const createdItemClone = createdItem.copy(true) + const newParentClone = createdItemClone.findItem(ItemType.FOLDER, newItem.parentId) as Folder + const newClonedItem = createdItemClone.findItem(newItem.type, newItem.id) + index = newParentClone.children.indexOf(newClonedItem) + newParentClone.children.splice(index, 1) + createAction.payload = createdItemClone + createAction.payload.createIndex() + } + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: newItem, + oldItem: removedItem, + index: index || createAction.index, + oldIndex: removeAction.index + }) + reconciled = true + if (removedItem.type === ItemType.FOLDER) { + await this.diffItem(removedItem, newItem) + } + } } } - - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem, - index: newIndex ?? createRootAction.index, - oldIndex: oldIndex ?? removeRootAction.index, - }) - - await this.diffItem(oldItem, createdItem) - // After diffing we need to start from scratch - // to make sure we match the newly created actions - hasNewActions = true - break } } From c0c640c6711c68dbc236308eab6f783a027a9f1a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 17:43:31 +0100 Subject: [PATCH 47/53] Fix: Reduce memory pressure by reducing ACTION_CONCURRENCY Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 2 +- src/lib/strategies/Merge.ts | 7 ++++--- src/lib/strategies/Unidirectional.ts | 4 +--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index ef5230028d..2dff7f0748 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -35,7 +35,7 @@ import NextcloudBookmarksAdapter from '../adapters/NextcloudBookmarks' import CachingAdapter from '../adapters/Caching' import { yieldToEventLoop } from '../yieldToEventLoop' -const ACTION_CONCURRENCY = 12 +export const ACTION_CONCURRENCY = 5 export default class SyncProcess { protected mappings: Mappings diff --git a/src/lib/strategies/Merge.ts b/src/lib/strategies/Merge.ts index 0726ba6956..620dd38bcf 100644 --- a/src/lib/strategies/Merge.ts +++ b/src/lib/strategies/Merge.ts @@ -2,12 +2,13 @@ import { Folder, ItemLocation, TItem, TItemLocation, TOppositeLocation } from '. import Diff, { CreateAction, MoveAction, PlanStage1 } from '../Diff' import Scanner, { ScanResult } from '../Scanner' import * as Parallel from 'async-parallel' -import DefaultSyncProcess, { ISerializedSyncProcess } from './Default' +import DefaultSyncProcess, { + ISerializedSyncProcess, + ACTION_CONCURRENCY, +} from './Default' import Mappings from '../Mappings' import Logger from '../Logger' -const ACTION_CONCURRENCY = 12 - export default class MergeSyncProcess extends DefaultSyncProcess { async getDiffs(): Promise<{ localScanResult: ScanResult diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 875bd11341..06cde0d198 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -7,9 +7,7 @@ import Logger from '../Logger' import { CancelledSyncError } from '../../errors/Error' import TResource from '../interfaces/Resource' import Scanner, { ScanResult } from '../Scanner' -import DefaultSyncProcess from './Default' - -const ACTION_CONCURRENCY = 12 +import DefaultSyncProcess, { ACTION_CONCURRENCY } from './Default' export default class UnidirectionalSyncProcess extends DefaultStrategy { protected direction: TItemLocation From 3aad9e99e15286624b75599579b065739f4b0b42 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 31 Dec 2025 18:30:47 +0100 Subject: [PATCH 48/53] Reapply "fix(Scanner): Reduce time complexity of findMoves to improve performance" This reverts commit 204bf1182a465993f2d122765afdb1336a79923a. --- src/lib/Scanner.ts | 271 +++++++++++++++++++++++++-------------------- 1 file changed, 153 insertions(+), 118 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 615a959786..91eab22dd0 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -199,135 +199,170 @@ export default class Scanner async findMoves():Promise { Logger.log('Scanner: Finding moves') - let createActions - let removeActions - let reconciled = true - - // As soon as one match is found, action list is updated and search is started with the new list - // repeat until no rewrites happen anymore - while (reconciled) { - reconciled = false - let createAction: CreateAction, removeAction: RemoveAction - - // First find direct matches (avoids glitches when folders and their contents have been moved) - createActions = this.result.CREATE.getActions() + + let hasNewActions = true + + while (hasNewActions) { + hasNewActions = false + + const createActions = this.result.CREATE.getActions() + const removeActions = this.result.REMOVE.getActions() + + if (createActions.length === 0 || removeActions.length === 0) break + + // Build Multi-Signal Fuzzy Index (O(N)) + const removedFuzzyMap = new Map; item: TItem }[]>() + const allCreatedItems: { rootAction: CreateAction; item: TItem }[] = [] + + const addToFuzzyIndex = (item: TItem, action: RemoveAction) => { + const keys = new Set() + // Signal 1: Full signature (Type + Title + URL) + keys.add(`${item.type}_${item.title}_${item.type === 'bookmark' ? (item as Bookmark).url : ''}`) + + // Signal 2: Title only (Handles URL changes for bookmarks or ID changes for folders) + if (item.title) keys.add(`${item.type}_title_${item.title}`) + + // Signal 3: URL only (Handles Title changes for bookmarks) + if (item instanceof Bookmark) keys.add(`bookmark_url_${item.url}`) + + keys.add(item.type) + + const element = { rootAction: action, item } // outside the loop so we can later use Set#has + for (const key of keys) { + let list = removedFuzzyMap.get(key) + if (!list) { + list = [] + removedFuzzyMap.set(key, list) + } + list.push(element) + } + } + + for (const action of removeActions) { + await addToFuzzyIndex(action.payload, action) + if (action.payload instanceof Folder) { + await action.payload.traverse((child) => addToFuzzyIndex(child, action)) + } + } + + for (const action of createActions) { + allCreatedItems.push({ rootAction: action, item: action.payload }) + if (action.payload instanceof Folder) { + await action.payload.traverse((child) => { + allCreatedItems.push({ rootAction: action, item: child }) + }) + } + } + + allCreatedItems + .sort((a, b) => b.item.count() - a.item.count()) + + // Match ALL created items (roots + descendants) against removed pool let iterations = 0 - while (!reconciled && (createAction = createActions.shift())) { - // give the browser time to breathe + for (const createdEntry of allCreatedItems) { if (++iterations % 1000 === 0) { await yieldToEventLoop() } - const createdItem = createAction.payload - removeActions = this.result.REMOVE.getActions() - while (!reconciled && (removeAction = removeActions.shift())) { - // give the browser time to breathe - if (++iterations % 1000 === 0) { - await yieldToEventLoop() + const { rootAction: createRootAction, item: createdItem } = createdEntry + + // Gather potential matches from all signals + const searchKeys = [ + `${createdItem.type}_${createdItem.title}_${createdItem.type === 'bookmark' ? (createdItem as Bookmark).url : ''}`, + `${createdItem.type}_title_${createdItem.title}`, + ...(createdItem instanceof Bookmark ? [`bookmark_url_${createdItem.url}`] : []), + createdItem.type + ] + + // Collect unique potential matches from all signals + const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() + for (const key of searchKeys) { + const list = removedFuzzyMap.get(key) + if (!list) continue + let hasMergeable = false + for (const m of list) { + if (this.mergeable(m.item, createdItem)) { + hasMergeable = true + break + } } - const removedItem = removeAction.payload - - if (this.mergeable(removedItem, createdItem) && - (removedItem.type !== 'folder' || - (!this.hasCache && removedItem.childrenSimilarity(createdItem) > 0.8))) { - this.result.CREATE.retract(createAction) - this.result.REMOVE.retract(removeAction) - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem: removedItem, - index: createAction.index, - oldIndex: removeAction.index - }) - reconciled = true - // Don't use the items from the action, but the ones in the actual tree to avoid using tree parts mutated by this algorithm (see below) - await this.diffItem(removedItem, createdItem) + if (hasMergeable) { + list.forEach((m) => potentialSet.add(m)) + break } } - } - // Then find descendant matches - createActions = this.result.CREATE.getActions() - createActions.sort((a, b) => - (b.payload.countFolders() * 1000 + b.payload.count()) - (a.payload.countFolders() * 1000 + a.payload.count()) - ) - while (!reconciled && (createAction = createActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const createdItem = createAction.payload - removeActions = this.result.REMOVE.getActions() - removeActions.sort((a, b) => - (b.payload.countFolders() * 1000 + b.payload.count()) - (a.payload.countFolders() * 1000 + a.payload.count()) + if (potentialSet.size === 0) { + continue + } + + const matches = Array.from(potentialSet).filter((m) => + this.mergeable(m.item, createdItem) ) - while (!reconciled && (removeAction = removeActions.shift())) { - // give the browser time to breathe - await Promise.resolve() - const removedItem = removeAction.payload - const oldItem = removedItem.findItemFilter( - createdItem.type, - item => this.mergeable(item, createdItem), - item => item.childrenSimilarity(createdItem) - ) - if (oldItem) { - let oldIndex - this.result.CREATE.retract(createAction) - if (oldItem === removedItem) { - this.result.REMOVE.retract(removeAction) - } else { - // We clone the item here, because we don't want to mutate all copies of this tree (item) - const removedItemClone = removedItem.copy(true) - const oldParentClone = removedItemClone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder - const oldItemClone = removedItemClone.findItem(oldItem.type, oldItem.id) - oldIndex = oldParentClone.children.indexOf(oldItemClone) - oldParentClone.children.splice(oldIndex, 1) - removeAction.payload = removedItemClone - removeAction.payload.createIndex() - } - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: createdItem, - oldItem, - index: createAction.index, - oldIndex: oldIndex || removeAction.index - }) - reconciled = true - if (oldItem.type === ItemType.FOLDER) { // TODO: Is this necessary? - await this.diffItem(oldItem, createdItem) - } - } else { - const newItem = createdItem.findItemFilter( - removedItem.type, - item => this.mergeable(removedItem, item), - item => item.childrenSimilarity(removedItem) - ) - let index - if (newItem) { - this.result.REMOVE.retract(removeAction) - if (newItem === createdItem) { - this.result.CREATE.retract(createAction) - } else { - // We clone the item here, because we don't want to mutate all copies of this tree (item) - const createdItemClone = createdItem.copy(true) - const newParentClone = createdItemClone.findItem(ItemType.FOLDER, newItem.parentId) as Folder - const newClonedItem = createdItemClone.findItem(newItem.type, newItem.id) - index = newParentClone.children.indexOf(newClonedItem) - newParentClone.children.splice(index, 1) - createAction.payload = createdItemClone - createAction.payload.createIndex() - } - this.result.MOVE.commit({ - type: ActionType.MOVE, - payload: newItem, - oldItem: removedItem, - index: index || createAction.index, - oldIndex: removeAction.index - }) - reconciled = true - if (removedItem.type === ItemType.FOLDER) { - await this.diffItem(removedItem, newItem) - } - } + + // Heuristic: Prefer matches that have more descendants + // In case we have no cache: Calculate similarity and sore by it + matches.sort((a, b) => { + if (createdItem.type === 'folder' && !this.hasCache) { + const simA = a.item.childrenSimilarity(createdItem) + const simB = b.item.childrenSimilarity(createdItem) + if (simA !== simB) return simB - simA + } + return (b.item.countFolders() * 1000 + b.item.count()) - (a.item.countFolders() * 1000 + a.item.count()) + }) + + if (matches.length === 0) { + continue + } + + const { rootAction: removeRootAction, item: oldItem } = matches[0] + const removedRoot = removeRootAction.payload + const createdRoot = createRootAction.payload + + let oldIndex, newIndex + + // Retract or Mutate "Old" (Removed) side + if (oldItem === removedRoot) { + this.result.REMOVE.retract(removeRootAction) + } else { + const clone = (removedRoot as Folder).clone(true) + const parentClone = clone.findItem(ItemType.FOLDER, oldItem.parentId) as Folder + const itemClone = clone.findItem(oldItem.type, oldItem.id) + if (parentClone && itemClone) { + oldIndex = parentClone.children.indexOf(itemClone) + parentClone.children.splice(oldIndex, 1) + clone.createIndex() + removeRootAction.payload = clone } } + + // Retract or Mutate "New" (Created) side + if (createdItem === createdRoot) { + this.result.CREATE.retract(createRootAction) + } else { + const clone = (createdRoot as Folder).clone(true) + const parentClone = clone.findItem(ItemType.FOLDER, createdItem.parentId) as Folder + const itemClone = clone.findItem(createdItem.type, createdItem.id) + if (parentClone && itemClone) { + newIndex = parentClone.children.indexOf(itemClone) + parentClone.children.splice(newIndex, 1) + clone.createIndex() + createRootAction.payload = clone + } + } + + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem, + index: newIndex ?? createRootAction.index, + oldIndex: oldIndex ?? removeRootAction.index, + }) + + await this.diffItem(oldItem, createdItem) + // After diffing we need to start from scratch + // to make sure we match the newly created actions + hasNewActions = true + break } } From 538d892766af3822d7f59f4555eb24af4549b17c Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 1 Jan 2026 10:32:32 +0100 Subject: [PATCH 49/53] Fix(Scanner#diffFolder): Fall back to O(n) strategy if fuzzy map doesn't yield results Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 91eab22dd0..610c716680 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -133,6 +133,9 @@ export default class Scanner newItem = potentialMatches.splice(matchIndex, 1)[0] stillUnmatched.delete(newItem) } + } else { + newItem = newFolder.children.find((child) => old.type === child.type && this.mergeable(old, child)) + if (newItem) stillUnmatched.delete(newItem) } // we found an item in the new folder that matches the one in the old folder if (newItem) { @@ -201,6 +204,7 @@ export default class Scanner Logger.log('Scanner: Finding moves') let hasNewActions = true + let iterations = 0 while (hasNewActions) { hasNewActions = false @@ -258,7 +262,6 @@ export default class Scanner .sort((a, b) => b.item.count() - a.item.count()) // Match ALL created items (roots + descendants) against removed pool - let iterations = 0 for (const createdEntry of allCreatedItems) { if (++iterations % 1000 === 0) { await yieldToEventLoop() From 6ef3f3a2a79e09b5f1004c8a05d65ba2afa64afa Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 1 Jan 2026 17:02:27 +0100 Subject: [PATCH 50/53] Fix(Scanner): Go back to two-tiered algorithm with direct matches first, but keep fuzzy index Signed-off-by: Marcel Klehr --- src/lib/Scanner.ts | 138 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 126 insertions(+), 12 deletions(-) diff --git a/src/lib/Scanner.ts b/src/lib/Scanner.ts index 610c716680..15a1a969ca 100644 --- a/src/lib/Scanner.ts +++ b/src/lib/Scanner.ts @@ -249,6 +249,123 @@ export default class Scanner } } + // First pass: Try to find direct matches + + // Match directly created items against removed pool + for (const createAction of createActions) { + if (++iterations % 1000 === 0) { + await yieldToEventLoop() + } + + const createdItem = createAction.payload + + // Gather potential matches from all signals + const searchKeys = [ + `${createdItem.type}_${createdItem.title}_${ + createdItem.type === 'bookmark' + ? (createdItem as Bookmark).url + : '' + }`, + `${createdItem.type}_title_${createdItem.title}`, + ...(createdItem instanceof Bookmark + ? [`bookmark_url_${createdItem.url}`] + : []), + createdItem.type, + ] + + // Collect unique potential matches from all signals + const matchesSet = new Set<{ + rootAction: RemoveAction + item: TItem + }>() + for (const key of searchKeys) { + const list = removedFuzzyMap.get(key) + if (!list) continue + for (const m of list) { + if (this.mergeable(m.item, createdItem)) { + matchesSet.add(m) + } + } + if (matchesSet.size > 0) { + break + } + } + + if (matchesSet.size === 0) { + continue + } + + const matches = Array.from(matchesSet) + + // Heuristic: Prefer matches that have more descendants + // In case we have no cache: Calculate similarity and sore by it + matches.sort((a, b) => { + if (createdItem.type === 'folder' && !this.hasCache) { + const simA = a.item.childrenSimilarity(createdItem) + const simB = b.item.childrenSimilarity(createdItem) + if (simA !== simB) return simB - simA + } + return ( + b.item.countFolders() * 1000 + + b.item.count() - + (a.item.countFolders() * 1000 + a.item.count()) + ) + }) + + if (matches.length === 0) { + continue + } + + const { rootAction: removeRootAction, item: oldItem } = matches[0] + const removedRoot = removeRootAction.payload + + let oldIndex, newIndex + + // Retract or Mutate "Old" (Removed) side + if (oldItem === removedRoot) { + this.result.REMOVE.retract(removeRootAction) + } else { + const clone = (removedRoot as Folder).clone(true) + const parentClone = clone.findItem( + ItemType.FOLDER, + oldItem.parentId + ) as Folder + const itemClone = clone.findItem(oldItem.type, oldItem.id) + if (parentClone && itemClone) { + oldIndex = parentClone.children.indexOf(itemClone) + parentClone.children.splice(oldIndex, 1) + clone.createIndex() + removeRootAction.payload = clone + } + } + + // Retract Created side + this.result.CREATE.retract(createAction) + + this.result.MOVE.commit({ + type: ActionType.MOVE, + payload: createdItem, + oldItem, + index: newIndex ?? createAction.index, + oldIndex: oldIndex ?? removeRootAction.index, + }) + + await this.diffItem(oldItem, createdItem) + // After diffing we need to start from scratch + // to make sure we match the newly created actions + if (oldItem instanceof Folder) { + hasNewActions = true + break + } + } + + if (hasNewActions) { + continue + } + + // Then find subtree matches + + // We enumerate all created items for (const action of createActions) { allCreatedItems.push({ rootAction: action, item: action.payload }) if (action.payload instanceof Folder) { @@ -277,30 +394,25 @@ export default class Scanner ] // Collect unique potential matches from all signals - const potentialSet = new Set<{ rootAction: RemoveAction; item: TItem }>() + const matchesSet = new Set<{ rootAction: RemoveAction; item: TItem }>() for (const key of searchKeys) { const list = removedFuzzyMap.get(key) if (!list) continue - let hasMergeable = false for (const m of list) { if (this.mergeable(m.item, createdItem)) { - hasMergeable = true - break + matchesSet.add(m) } } - if (hasMergeable) { - list.forEach((m) => potentialSet.add(m)) + if (matchesSet.size > 0) { break } } - if (potentialSet.size === 0) { + if (matchesSet.size === 0) { continue } - const matches = Array.from(potentialSet).filter((m) => - this.mergeable(m.item, createdItem) - ) + const matches = Array.from(matchesSet) // Heuristic: Prefer matches that have more descendants // In case we have no cache: Calculate similarity and sore by it @@ -364,8 +476,10 @@ export default class Scanner await this.diffItem(oldItem, createdItem) // After diffing we need to start from scratch // to make sure we match the newly created actions - hasNewActions = true - break + if (oldItem instanceof Folder) { + hasNewActions = true + break + } } } From e5b4f5b34b219a39408a15043c41e18ff8a98812 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 2 Jan 2026 11:38:56 +0100 Subject: [PATCH 51/53] fix(executeReorderings): Try to prevent crash Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index 2dff7f0748..2138fd6254 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -1381,6 +1381,7 @@ export default class SyncProcess { const isUsingTabs = await this.localTree.isUsingBrowserTabs?.() await Parallel.each(reorderings.getActions(), async(action) => { + await yieldToEventLoop() Logger.log('Executing reorder action', `${action.type} Payload: #${action.payload.id}[${action.payload.title}]${'url' in action.payload ? `(${action.payload.url})` : ''} parentId: ${action.payload.parentId}`) const item = action.payload From 95ba6baaa51ef56ab3caf36edc18e7f596666d0b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 2 Jan 2026 11:52:00 +0100 Subject: [PATCH 52/53] fix(SyncProcess): Fix serialization for continuation persistence Signed-off-by: Marcel Klehr --- src/lib/strategies/Default.ts | 62 +++++++++++++--------------- src/lib/strategies/Merge.ts | 9 +--- src/lib/strategies/Unidirectional.ts | 53 +++++++++++++++++++----- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/src/lib/strategies/Default.ts b/src/lib/strategies/Default.ts index 2138fd6254..223a244db1 100644 --- a/src/lib/strategies/Default.ts +++ b/src/lib/strategies/Default.ts @@ -1546,25 +1546,6 @@ export default class SyncProcess { parentReorder.order = parentReorder.order.filter(item => !(item.type === oldItem.type && String(Mappings.mapId(mappingsSnapshot, oldItem, parentReorder.payload.location)) === String(item.id))) } - toJSON(): ISerializedSyncProcess { - if (!this.staticContinuation) { - this.staticContinuation = { - // Do not store these as the continuation size can get huge otherwise - localTreeRoot: null, - cacheTreeRoot: null, - serverTreeRoot: null, - } - } - const membersToPersist = this.getMembersToPersist() - return { - strategy: 'default', - ...this.staticContinuation, - ...(Object.fromEntries(Object.entries(this) - .filter(([key]) => membersToPersist.includes(key))) - ), - } - } - async toJSONAsync(): Promise { if (!this.staticContinuation) { this.staticContinuation = { @@ -1576,23 +1557,36 @@ export default class SyncProcess { } const membersToPersist = this.getMembersToPersist() return { - strategy: 'default', + strategy: 'unidirectional', ...this.staticContinuation, ...(Object.fromEntries( - await Parallel.map( - Object.entries(this) - .filter(([key]) => membersToPersist.includes(key)), - async([key, value]) => { - if (value && value.toJSONAsync) { - return [key, await value.toJSONAsync()] - } - if (value && value.toJSON) { - await yieldToEventLoop() - return [key, value.toJSON()] - } - return [key, value] - }, 1) - ) + await Parallel.map( + Object.entries(this) + .filter(([key]) => membersToPersist.includes(key)), + async([key, value]) => { + if (value && value.CREATE && value.REMOVE && value.UPDATE && value.MOVE && value.REORDER) { + // property holds a Plan + return [key, Object.fromEntries(await Parallel.map(Object.entries(value), async([key, diff]: [string, Diff>]) => { + if (diff && diff.toJSONAsync) { + return [key, await diff.toJSONAsync()] + } + if (diff && diff.toJSON) { + await yieldToEventLoop() + return [key, diff.toJSON()] + } + return [key, diff] + }))] + } + if (value && value.toJSONAsync) { + return [key, await value.toJSONAsync()] + } + if (value && value.toJSON) { + await yieldToEventLoop() + return [key, value.toJSON()] + } + return [key, value] + }, 1) + ) ), } } diff --git a/src/lib/strategies/Merge.ts b/src/lib/strategies/Merge.ts index 620dd38bcf..63db482d9b 100644 --- a/src/lib/strategies/Merge.ts +++ b/src/lib/strategies/Merge.ts @@ -363,16 +363,9 @@ export default class MergeSyncProcess extends DefaultSyncProcess { ).children } - toJSON(): ISerializedSyncProcess { - return { - ...DefaultSyncProcess.prototype.toJSON.apply(this), - strategy: 'merge', - } - } - async toJSONAsync(): Promise { return { - ...(await DefaultSyncProcess.prototype.toJSON.apply(this)), + ...(await DefaultSyncProcess.prototype.toJSONAsync.apply(this)), strategy: 'merge', } } diff --git a/src/lib/strategies/Unidirectional.ts b/src/lib/strategies/Unidirectional.ts index 06cde0d198..d54d408c4b 100644 --- a/src/lib/strategies/Unidirectional.ts +++ b/src/lib/strategies/Unidirectional.ts @@ -1,5 +1,5 @@ -import DefaultStrategy, { ISerializedSyncProcess } from './Default' -import Diff, { ActionType, PlanRevert, PlanStage1, PlanStage3, ReorderAction } from '../Diff' +import DefaultStrategy, { ISerializedSyncProcess , ACTION_CONCURRENCY } from './Default' +import Diff, { Action, ActionType, PlanRevert, PlanStage1, PlanStage3, ReorderAction } from '../Diff' import * as Parallel from 'async-parallel' import Mappings, { MappingSnapshot } from '../Mappings' import { Folder, ItemLocation, TItem, TItemLocation, TOppositeLocation } from '../Tree' @@ -7,7 +7,7 @@ import Logger from '../Logger' import { CancelledSyncError } from '../../errors/Error' import TResource from '../interfaces/Resource' import Scanner, { ScanResult } from '../Scanner' -import DefaultSyncProcess, { ACTION_CONCURRENCY } from './Default' +import { yieldToEventLoop } from '../yieldToEventLoop' export default class UnidirectionalSyncProcess extends DefaultStrategy { protected direction: TItemLocation @@ -511,17 +511,48 @@ export default class UnidirectionalSyncProcess extends DefaultStrategy { ) } - toJSON(): ISerializedSyncProcess { - return { - ...DefaultSyncProcess.prototype.toJSON.apply(this), - strategy: 'unidirectional', - } - } - async toJSONAsync(): Promise { + if (!this.staticContinuation) { + this.staticContinuation = { + // Do not store these as the continuation size can get huge otherwise + localTreeRoot: null, + cacheTreeRoot: null, + serverTreeRoot: null, + } + } + const membersToPersist = this.getMembersToPersist() return { - ...(await DefaultSyncProcess.prototype.toJSONAsync.apply(this)), strategy: 'unidirectional', + ...this.staticContinuation, + ...(Object.fromEntries( + await Parallel.map( + Object.entries(this) + .filter(([key]) => membersToPersist.includes(key)), + async([key, value]) => { + if (value && value.CREATE && value.REMOVE && value.UPDATE && value.MOVE && value.REORDER) { + // property holds a Plan + return [key, Object.fromEntries(await Parallel.map(Object.entries(value), async([key, diff]: [string, Diff>]) => { + if (diff && diff.toJSONAsync) { + return [key, await diff.toJSONAsync()] + } + if (diff && diff.toJSON) { + await yieldToEventLoop() + return [key, diff.toJSON()] + } + return [key, diff] + }))] + } + if (value && value.toJSONAsync) { + return [key, await value.toJSONAsync()] + } + if (value && value.toJSON) { + await yieldToEventLoop() + return [key, value.toJSON()] + } + return [key, value] + }, 1) + ) + ), } } } From f860bf6bc529c0d3be7323eeece1bc1d02e324b9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 4 Jan 2026 10:25:01 +0100 Subject: [PATCH 53/53] fix(Tree): Harden against undefined errors Signed-off-by: Marcel Klehr --- src/lib/Tree.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/Tree.ts b/src/lib/Tree.ts index 5ff6165655..468d3e0467 100644 --- a/src/lib/Tree.ts +++ b/src/lib/Tree.ts @@ -596,8 +596,12 @@ export class Folder { const itemIndex = item.index || item.createIndex() let currentItem = this.index.folder[item.parentId] while (currentItem) { - Object.assign(currentItem.index.folder, itemIndex.folder) - Object.assign(currentItem.index.bookmark, itemIndex.bookmark) + if (currentItem.index) { + Object.assign(currentItem.index.folder, itemIndex.folder) + Object.assign(currentItem.index.bookmark, itemIndex.bookmark) + } else { + currentItem.createIndex() + } currentItem = this.index.folder[currentItem.parentId] } } @@ -611,11 +615,14 @@ export class Folder { this.createIndex() return } + if (!item.index) { + item.createIndex() + } if (item.parentId) { let parentFolder = this.index.folder[item.parentId] while (parentFolder && this.index.folder[parentFolder.parentId] !== parentFolder) { if (item instanceof Bookmark) { - delete parentFolder.index[item.type][item.id] + delete parentFolder.index.bookmark[item.id] } else { for (const folderId in item.index.folder) { delete parentFolder.index.folder[folderId]