Fix: Futile Attempt to Implement Image-Dedupe (#55)

This commit is contained in:
Youwes09
2026-04-28 22:45:19 -05:00
parent 2c1391c378
commit 63209cb828
12 changed files with 124 additions and 153 deletions
@@ -1,7 +1,7 @@
<script lang="ts">
import { Folder, Trash, CheckSquare, Robot } from "phosphor-svelte";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import { resolvedCover } from "@features/series/lib/coverResolver";
import { resolvedCover } from "@core/cover/coverResolver";
import type { Manga, Category } from "@types";
interface Props {
@@ -35,7 +35,7 @@
import ChapterList from "./ChapterList.svelte";
import { buildChapterList, chaptersAscending } from "../lib/chapterList";
import { getPref, setPref } from "../lib/mangaPrefs";
import { autoLinkLibrary } from "@features/series/lib/autoLink";
import { autoLinkLibrary } from "@core/cover/autoLink";
const CHAPTERS_PER_PAGE = 25;
const MANGA_TTL_MS = 5 * 60 * 1000;
@@ -5,7 +5,7 @@
MapPin, Gear, Trash, Image,
} from "phosphor-svelte";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import { resolvedCover } from "@features/series/lib/coverResolver";
import { resolvedCover } from "@core/cover/coverResolver";
import type { Manga, Chapter, Category } from "@types";
import type { MangaPrefs } from "@store/state.svelte";
import { store, setActiveManga, setGenreFilter, setNavPage, setPreviewManga, linkManga, unlinkManga } from "@store/state.svelte";
-27
View File
@@ -1,27 +0,0 @@
import { store, linkManga } from "@store/state.svelte";
import type { Manga } from "@types";
export function autoLinkLibrary(focal: Manga, allManga: Manga[]): Promise<number> {
return new Promise((resolve) => {
const worker = new Worker(
new URL("./autoLinkWorker.ts", import.meta.url),
{ type: "module" },
);
worker.onmessage = (e: MessageEvent<number[]>) => {
const matches = e.data;
for (const id of matches) linkManga(focal.id, id);
worker.terminate();
resolve(matches.length);
};
worker.onerror = () => { worker.terminate(); resolve(0); };
worker.postMessage({
focalTitle: focal.title,
focalId: focal.id,
allManga: allManga.map(m => ({ id: m.id, title: m.title })),
linkedIds: store.settings.mangaLinks?.[focal.id] ?? [],
});
});
}
-29
View File
@@ -1,29 +0,0 @@
interface WorkerMsg {
focalTitle: string;
focalId: number;
allManga: { id: number; title: string }[];
linkedIds: number[];
}
function titleSimilarity(a: string, b: string): number {
const norm = (s: string) =>
s.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter(Boolean);
const wa = new Set(norm(a));
const wb = new Set(norm(b));
if (!wa.size || !wb.size) return 0;
const intersection = [...wa].filter(w => wb.has(w)).length;
return intersection / new Set([...wa, ...wb]).size;
}
self.onmessage = (e: MessageEvent<WorkerMsg>) => {
const { focalTitle, focalId, allManga, linkedIds } = e.data;
const matches: number[] = [];
for (const m of allManga) {
if (m.id === focalId) continue;
if (linkedIds.includes(m.id)) continue;
if (titleSimilarity(focalTitle, m.title) >= 0.4) matches.push(m.id);
}
self.postMessage(matches);
};
-89
View File
@@ -1,89 +0,0 @@
import { getBlobUrl } from "@core/cache/imageCache";
const HASH_SIZE = 8;
const HASH_PIXELS = HASH_SIZE * HASH_SIZE;
const CANVAS_SIZE = 32;
const DUPE_THRESH = 10;
const hashCache = new Map<string, Uint8Array>();
async function loadGrayscale(blobUrl: string): Promise<Uint8ClampedArray> {
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => {
const canvas = document.createElement("canvas");
canvas.width = canvas.height = CANVAS_SIZE;
const ctx = canvas.getContext("2d")!;
ctx.drawImage(img, 0, 0, CANVAS_SIZE, CANVAS_SIZE);
const { data } = ctx.getImageData(0, 0, CANVAS_SIZE, CANVAS_SIZE);
const gray = new Uint8ClampedArray(CANVAS_SIZE * CANVAS_SIZE);
for (let i = 0; i < gray.length; i++) {
const o = i * 4;
gray[i] = (data[o] * 299 + data[o + 1] * 587 + data[o + 2] * 114) / 1000;
}
resolve(gray);
};
img.onerror = reject;
img.src = blobUrl;
});
}
function dct8x8(gray: Uint8ClampedArray): number[] {
const N = CANVAS_SIZE;
const step = N / HASH_SIZE;
const block: number[] = [];
for (let by = 0; by < HASH_SIZE; by++) {
for (let bx = 0; bx < HASH_SIZE; bx++) {
let sum = 0, count = 0;
for (let dy = 0; dy < step; dy++) {
for (let dx = 0; dx < step; dx++) {
sum += gray[(by * step + dy) * N + (bx * step + dx)];
count++;
}
}
block.push(sum / count);
}
}
return block;
}
function pHash(block: number[]): Uint8Array {
const mean = block.reduce((a, b) => a + b, 0) / HASH_PIXELS;
const bits = new Uint8Array(Math.ceil(HASH_PIXELS / 8));
for (let i = 0; i < HASH_PIXELS; i++) {
if (block[i] >= mean) bits[i >> 3] |= 1 << (i & 7);
}
return bits;
}
function hammingDistance(a: Uint8Array, b: Uint8Array): number {
let dist = 0;
for (let i = 0; i < a.length; i++) {
let x = a[i] ^ b[i];
while (x) { dist += x & 1; x >>= 1; }
}
return dist;
}
export async function getHash(url: string, priority = -1): Promise<Uint8Array | null> {
if (hashCache.has(url)) return hashCache.get(url)!;
try {
const blob = await getBlobUrl(url, priority);
const gray = await loadGrayscale(blob);
const block = dct8x8(gray);
const hash = pHash(block);
hashCache.set(url, hash);
return hash;
} catch {
return null;
}
}
export function areDuplicates(a: Uint8Array, b: Uint8Array): boolean {
return hammingDistance(a, b) <= DUPE_THRESH;
}
export function clearHashCache(): void {
hashCache.clear();
}
-94
View File
@@ -1,94 +0,0 @@
import { store } from "@store/state.svelte";
import { searchWithScore } from "@core/algorithms/search";
import { getHash, areDuplicates } from "@features/series/lib/coverHash";
type CoverManga = { id: number; thumbnailUrl: string; source?: { displayName: string } | null };
export type CoverCandidate = {
mangaId: number;
url: string;
label: string;
isActive: boolean;
};
const FUZZY_SCORE_THRESHOLD = 0.5;
function normalizeUrl(url: string): string {
try {
const u = new URL(url);
u.search = "";
return u.href.toLowerCase();
} catch {
return url.toLowerCase();
}
}
export function resolvedCover(mangaId: number, ownUrl: string): string {
return store.settings.mangaPrefs?.[mangaId]?.coverUrl ?? ownUrl;
}
function fuzzyMatchIds(
mangaId: number,
title: string,
mangaById: Map<number, CoverManga & { title: string }>,
): number[] {
const results = searchWithScore(
[...mangaById.values()].filter(m => m.id !== mangaId),
title,
m => m.title,
);
return results
.filter(r => r.score >= FUZZY_SCORE_THRESHOLD)
.map(r => r.item.id);
}
export function coverCandidatesSync(
mangaId: number,
title: string,
ownUrl: string,
mangaById: Map<number, CoverManga & { title: string }>,
): CoverCandidate[] {
const linkedIds = store.getLinkedMangaIds(mangaId);
const fuzzyIds = fuzzyMatchIds(mangaId, title, mangaById);
const current = store.settings.mangaPrefs?.[mangaId]?.coverUrl ?? ownUrl;
const allIds = Array.from(new Set([...linkedIds, ...fuzzyIds]));
const raw: { mangaId: number; url: string; label: string }[] = [
{ mangaId, url: ownUrl, label: "This source" },
...allIds.flatMap(id => {
const m = mangaById.get(id);
return m ? [{ mangaId: m.id, url: m.thumbnailUrl, label: m.source?.displayName ?? `ID ${m.id}` }] : [];
}),
];
const seen = new Set<string>();
return raw
.filter(c => {
const key = normalizeUrl(c.url);
if (seen.has(key)) return false;
seen.add(key);
return true;
})
.map(c => ({ ...c, isActive: normalizeUrl(c.url) === normalizeUrl(current) }));
}
export async function dedupeByImage(candidates: CoverCandidate[]): Promise<CoverCandidate[]> {
const hashes = await Promise.all(candidates.map(c => getHash(c.url)));
const keptIndices: number[] = [];
for (let i = 0; i < candidates.length; i++) {
const hi = hashes[i];
if (!hi) { keptIndices.push(i); continue; }
const isDupe = keptIndices.some(j => {
const hj = hashes[j];
return hj ? areDuplicates(hi, hj) : false;
});
if (!isDupe) keptIndices.push(i);
}
return keptIndices.map(i => candidates[i]);
}
@@ -1,7 +1,7 @@
<script lang="ts">
import { X, CaretLeft, CaretRight } from "phosphor-svelte";
import { X, CaretLeft, CaretRight, CircleNotch } from "phosphor-svelte";
import { setPref } from "@features/series/lib/mangaPrefs";
import { coverCandidatesSync, dedupeByImage } from "@features/series/lib/coverResolver";
import { coverCandidatesSync, dedupeByImage } from "@core/cover/coverResolver";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import type { Manga } from "@types";
@@ -21,22 +21,19 @@
coverCandidatesSync(manga.id, manga.title, manga.thumbnailUrl, mangaById)
);
let candidates = $state(syncCandidates);
let candidates = $state<typeof syncCandidates>([]);
let hashingDone = $state(false);
let index = $state(0);
$effect(() => {
const snap = syncCandidates;
candidates = snap;
candidates = [];
hashingDone = false;
index = Math.max(0, snap.findIndex(c => c.isActive));
index = 0;
dedupeByImage(snap).then(deduped => {
const activeInDeduped = deduped.some(c => c.isActive);
candidates = activeInDeduped
? deduped
: (() => { const a = snap.find(c => c.isActive); return a ? [a, ...deduped.filter(c => !c.isActive)] : deduped; })();
index = Math.max(0, candidates.findIndex(c => c.isActive));
dedupeByImage(snap).then(merged => {
candidates = merged;
index = Math.max(0, merged.findIndex(c => c.isActive));
hashingDone = true;
});
});
@@ -70,41 +67,44 @@
<div class="modal" role="dialog" aria-label="Choose cover image" onkeydown={onKeydown}>
<div class="header">
<span class="title">Cover Image</span>
{#if !hashingDone}
<span class="comparing">Comparing…</span>
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
</div>
{#if !hashingDone}
<div class="loading">
<CircleNotch size={24} weight="light" class="anim-spin" style="color:var(--text-faint)" />
</div>
{:else}
<div class="stage">
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
<CaretLeft size={18} weight="bold" />
</button>
<div class="cover-wrap">
{#if current}
<Thumbnail src={current.url} alt="" class="cover-img" />
{/if}
</div>
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
<CaretRight size={18} weight="bold" />
</button>
</div>
{#if candidates.length > 1}
<div class="filmstrip">
{#each candidates as c, i (c.url)}
<button
class="film-thumb"
class:film-active={i === index}
onclick={() => index = i}
aria-label="Cover {i + 1}"
>
<Thumbnail src={c.url} alt="" class="film-img" />
</button>
{/each}
</div>
{/if}
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
</div>
<div class="stage">
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
<CaretLeft size={18} weight="bold" />
</button>
<div class="cover-wrap">
{#if current}
<Thumbnail src={current.url} alt="" class="cover-img" />
{/if}
</div>
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
<CaretRight size={18} weight="bold" />
</button>
</div>
{#if candidates.length > 1}
<div class="filmstrip">
{#each candidates as c, i (c.url)}
<button
class="film-thumb"
class:film-active={i === index}
onclick={() => index = i}
aria-label="Cover {i + 1}"
>
<Thumbnail src={c.url} alt="" class="film-img" />
</button>
{/each}
</div>
{/if}
<div class="footer">
@@ -209,6 +209,7 @@
transition: opacity var(--t-base);
}
.confirm-btn:hover { opacity: 0.88; }
.loading { display: flex; align-items: center; justify-content: center; padding: var(--sp-10) 0; }
@keyframes fadeIn { from { opacity: 0 } to { opacity: 1 } }
@keyframes scaleIn { from { opacity: 0; transform: scale(0.97) } to { opacity: 1; transform: scale(1) } }
@keyframes pulse { 0%, 100% { opacity: 0.4 } 50% { opacity: 1 } }
@@ -33,7 +33,7 @@
return others
.filter(m => !linkedIds.includes(m.id))
.map(m => ({ manga: m, score: titleSimilarity(manga.title, m.title) }))
.filter(r => r.score >= 0.4)
.filter(r => r.score >= 0.65)
.sort((a, b) => b.score - a.score)
.slice(0, 8);
});