From 63209cb828ab75db0cacff1dccb59af86d2d5e94 Mon Sep 17 00:00:00 2001 From: Youwes09 Date: Tue, 28 Apr 2026 22:45:19 -0500 Subject: [PATCH] Fix: Futile Attempt to Implement Image-Dedupe (#55) --- .../series/lib => core/cover}/autoLink.ts | 0 .../lib => core/cover}/autoLinkWorker.ts | 2 +- src/core/cover/coverHash.ts | 54 +++++++++++ .../lib => core/cover}/coverResolver.ts | 25 ++--- src/core/cover/index.ts | 4 + .../library/components/LibraryGrid.svelte | 2 +- .../series/components/SeriesDetail.svelte | 2 +- .../series/components/SeriesHeader.svelte | 2 +- src/features/series/lib/coverHash.ts | 89 ------------------ .../series/panels/CoverPickerPanel.svelte | 91 ++++++++++--------- .../series/panels/SeriesLinkPanel.svelte | 2 +- src/shared/manga/MangaPreview.svelte | 4 +- 12 files changed, 124 insertions(+), 153 deletions(-) rename src/{features/series/lib => core/cover}/autoLink.ts (100%) rename src/{features/series/lib => core/cover}/autoLinkWorker.ts (91%) create mode 100644 src/core/cover/coverHash.ts rename src/{features/series/lib => core/cover}/coverResolver.ts (79%) create mode 100644 src/core/cover/index.ts delete mode 100644 src/features/series/lib/coverHash.ts diff --git a/src/features/series/lib/autoLink.ts b/src/core/cover/autoLink.ts similarity index 100% rename from src/features/series/lib/autoLink.ts rename to src/core/cover/autoLink.ts diff --git a/src/features/series/lib/autoLinkWorker.ts b/src/core/cover/autoLinkWorker.ts similarity index 91% rename from src/features/series/lib/autoLinkWorker.ts rename to src/core/cover/autoLinkWorker.ts index 132fd6f..b5bae69 100644 --- a/src/features/series/lib/autoLinkWorker.ts +++ b/src/core/cover/autoLinkWorker.ts @@ -22,7 +22,7 @@ self.onmessage = (e: MessageEvent) => { for (const m of allManga) { if (m.id === focalId) continue; if (linkedIds.includes(m.id)) continue; - if (titleSimilarity(focalTitle, m.title) >= 0.4) matches.push(m.id); + if (titleSimilarity(focalTitle, m.title) >= 0.65) matches.push(m.id); } self.postMessage(matches); diff --git a/src/core/cover/coverHash.ts b/src/core/cover/coverHash.ts new file mode 100644 index 0000000..5dbf7c2 --- /dev/null +++ b/src/core/cover/coverHash.ts @@ -0,0 +1,54 @@ +const THUMB_SIZE = 16; +const DUPE_THRESH = 0.12; + +const hashCache = new Map(); + +function toGray(data: Uint8ClampedArray, pixels: number): Uint8ClampedArray { + const gray = new Uint8ClampedArray(pixels); + for (let i = 0; i < pixels; i++) { + const o = i * 4; + gray[i] = (data[o] * 299 + data[o + 1] * 587 + data[o + 2] * 114) / 1000; + } + return gray; +} + +function loadThumb(url: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.crossOrigin = "anonymous"; + img.onload = () => { + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = THUMB_SIZE; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(img, 0, 0, THUMB_SIZE, THUMB_SIZE); + resolve(toGray(ctx.getImageData(0, 0, THUMB_SIZE, THUMB_SIZE).data, THUMB_SIZE * THUMB_SIZE)); + }; + img.onerror = reject; + img.src = url; + }); +} + +function similarity(a: Uint8ClampedArray, b: Uint8ClampedArray): number { + let diff = 0; + for (let i = 0; i < a.length; i++) diff += Math.abs(a[i] - b[i]); + return diff / (a.length * 255); +} + +export async function getHash(url: string): Promise { + if (hashCache.has(url)) return hashCache.get(url)!; + try { + const thumb = await loadThumb(url); + hashCache.set(url, thumb); + return thumb; + } catch { + return null; + } +} + +export function areDuplicates(a: Uint8ClampedArray, b: Uint8ClampedArray): boolean { + return similarity(a, b) <= DUPE_THRESH; +} + +export function clearHashCache(): void { + hashCache.clear(); +} \ No newline at end of file diff --git a/src/features/series/lib/coverResolver.ts b/src/core/cover/coverResolver.ts similarity index 79% rename from src/features/series/lib/coverResolver.ts rename to src/core/cover/coverResolver.ts index fc9e375..9dd3912 100644 --- a/src/features/series/lib/coverResolver.ts +++ b/src/core/cover/coverResolver.ts @@ -1,6 +1,6 @@ import { store } from "@store/state.svelte"; import { searchWithScore } from "@core/algorithms/search"; -import { getHash, areDuplicates } from "@features/series/lib/coverHash"; +import { getHash, areDuplicates } from "@core/cover/coverHash"; type CoverManga = { id: number; thumbnailUrl: string; source?: { displayName: string } | null }; @@ -11,7 +11,7 @@ export type CoverCandidate = { isActive: boolean; }; -const FUZZY_SCORE_THRESHOLD = 0.5; +const FUZZY_SCORE_THRESHOLD = 0.65; function normalizeUrl(url: string): string { try { @@ -76,19 +76,20 @@ export function coverCandidatesSync( export async function dedupeByImage(candidates: CoverCandidate[]): Promise { const hashes = await Promise.all(candidates.map(c => getHash(c.url))); - const keptIndices: number[] = []; + const groups: number[][] = []; for (let i = 0; i < candidates.length; i++) { const hi = hashes[i]; - if (!hi) { keptIndices.push(i); continue; } - - const isDupe = keptIndices.some(j => { - const hj = hashes[j]; - return hj ? areDuplicates(hi, hj) : false; - }); - - if (!isDupe) keptIndices.push(i); + const existing = hi + ? groups.find(g => { const hj = hashes[g[0]]; return hj ? areDuplicates(hi, hj) : false; }) + : undefined; + if (existing) existing.push(i); + else groups.push([i]); } - return keptIndices.map(i => candidates[i]); + return groups.map(group => { + const active = group.find(i => candidates[i].isActive) ?? group[0]; + const labels = [...new Set(group.map(i => candidates[i].label))]; + return { ...candidates[active], label: labels.join(" · ") }; + }); } \ No newline at end of file diff --git a/src/core/cover/index.ts b/src/core/cover/index.ts new file mode 100644 index 0000000..130da9f --- /dev/null +++ b/src/core/cover/index.ts @@ -0,0 +1,4 @@ +export { getHash, areDuplicates, clearHashCache } from "./coverHash"; +export { resolvedCover, coverCandidatesSync, dedupeByImage } from "./coverResolver"; +export type { CoverCandidate } from "./coverResolver"; +export { autoLinkLibrary } from "./autoLink"; diff --git a/src/features/library/components/LibraryGrid.svelte b/src/features/library/components/LibraryGrid.svelte index bea2a3e..7aeac42 100644 --- a/src/features/library/components/LibraryGrid.svelte +++ b/src/features/library/components/LibraryGrid.svelte @@ -1,7 +1,7 @@