mirror of
https://github.com/moku-project/Moku.git
synced 2026-06-13 09:19:56 -05:00
Fix: Futile Attempt to Implement Image-Dedupe (#55)
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
<script lang="ts">
|
||||
import { Folder, Trash, CheckSquare, Robot } from "phosphor-svelte";
|
||||
import Thumbnail from "@shared/manga/Thumbnail.svelte";
|
||||
import { resolvedCover } from "@features/series/lib/coverResolver";
|
||||
import { resolvedCover } from "@core/cover/coverResolver";
|
||||
import type { Manga, Category } from "@types";
|
||||
|
||||
interface Props {
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
import ChapterList from "./ChapterList.svelte";
|
||||
import { buildChapterList, chaptersAscending } from "../lib/chapterList";
|
||||
import { getPref, setPref } from "../lib/mangaPrefs";
|
||||
import { autoLinkLibrary } from "@features/series/lib/autoLink";
|
||||
import { autoLinkLibrary } from "@core/cover/autoLink";
|
||||
|
||||
const CHAPTERS_PER_PAGE = 25;
|
||||
const MANGA_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
MapPin, Gear, Trash, Image,
|
||||
} from "phosphor-svelte";
|
||||
import Thumbnail from "@shared/manga/Thumbnail.svelte";
|
||||
import { resolvedCover } from "@features/series/lib/coverResolver";
|
||||
import { resolvedCover } from "@core/cover/coverResolver";
|
||||
import type { Manga, Chapter, Category } from "@types";
|
||||
import type { MangaPrefs } from "@store/state.svelte";
|
||||
import { store, setActiveManga, setGenreFilter, setNavPage, setPreviewManga, linkManga, unlinkManga } from "@store/state.svelte";
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
import { store, linkManga } from "@store/state.svelte";
|
||||
import type { Manga } from "@types";
|
||||
|
||||
export function autoLinkLibrary(focal: Manga, allManga: Manga[]): Promise<number> {
|
||||
return new Promise((resolve) => {
|
||||
const worker = new Worker(
|
||||
new URL("./autoLinkWorker.ts", import.meta.url),
|
||||
{ type: "module" },
|
||||
);
|
||||
|
||||
worker.onmessage = (e: MessageEvent<number[]>) => {
|
||||
const matches = e.data;
|
||||
for (const id of matches) linkManga(focal.id, id);
|
||||
worker.terminate();
|
||||
resolve(matches.length);
|
||||
};
|
||||
|
||||
worker.onerror = () => { worker.terminate(); resolve(0); };
|
||||
|
||||
worker.postMessage({
|
||||
focalTitle: focal.title,
|
||||
focalId: focal.id,
|
||||
allManga: allManga.map(m => ({ id: m.id, title: m.title })),
|
||||
linkedIds: store.settings.mangaLinks?.[focal.id] ?? [],
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
interface WorkerMsg {
|
||||
focalTitle: string;
|
||||
focalId: number;
|
||||
allManga: { id: number; title: string }[];
|
||||
linkedIds: number[];
|
||||
}
|
||||
|
||||
function titleSimilarity(a: string, b: string): number {
|
||||
const norm = (s: string) =>
|
||||
s.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter(Boolean);
|
||||
const wa = new Set(norm(a));
|
||||
const wb = new Set(norm(b));
|
||||
if (!wa.size || !wb.size) return 0;
|
||||
const intersection = [...wa].filter(w => wb.has(w)).length;
|
||||
return intersection / new Set([...wa, ...wb]).size;
|
||||
}
|
||||
|
||||
self.onmessage = (e: MessageEvent<WorkerMsg>) => {
|
||||
const { focalTitle, focalId, allManga, linkedIds } = e.data;
|
||||
const matches: number[] = [];
|
||||
|
||||
for (const m of allManga) {
|
||||
if (m.id === focalId) continue;
|
||||
if (linkedIds.includes(m.id)) continue;
|
||||
if (titleSimilarity(focalTitle, m.title) >= 0.4) matches.push(m.id);
|
||||
}
|
||||
|
||||
self.postMessage(matches);
|
||||
};
|
||||
@@ -1,89 +0,0 @@
|
||||
import { getBlobUrl } from "@core/cache/imageCache";
|
||||
|
||||
const HASH_SIZE = 8;
|
||||
const HASH_PIXELS = HASH_SIZE * HASH_SIZE;
|
||||
const CANVAS_SIZE = 32;
|
||||
const DUPE_THRESH = 10;
|
||||
|
||||
const hashCache = new Map<string, Uint8Array>();
|
||||
|
||||
async function loadGrayscale(blobUrl: string): Promise<Uint8ClampedArray> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = CANVAS_SIZE;
|
||||
const ctx = canvas.getContext("2d")!;
|
||||
ctx.drawImage(img, 0, 0, CANVAS_SIZE, CANVAS_SIZE);
|
||||
const { data } = ctx.getImageData(0, 0, CANVAS_SIZE, CANVAS_SIZE);
|
||||
const gray = new Uint8ClampedArray(CANVAS_SIZE * CANVAS_SIZE);
|
||||
for (let i = 0; i < gray.length; i++) {
|
||||
const o = i * 4;
|
||||
gray[i] = (data[o] * 299 + data[o + 1] * 587 + data[o + 2] * 114) / 1000;
|
||||
}
|
||||
resolve(gray);
|
||||
};
|
||||
img.onerror = reject;
|
||||
img.src = blobUrl;
|
||||
});
|
||||
}
|
||||
|
||||
function dct8x8(gray: Uint8ClampedArray): number[] {
|
||||
const N = CANVAS_SIZE;
|
||||
const step = N / HASH_SIZE;
|
||||
const block: number[] = [];
|
||||
|
||||
for (let by = 0; by < HASH_SIZE; by++) {
|
||||
for (let bx = 0; bx < HASH_SIZE; bx++) {
|
||||
let sum = 0, count = 0;
|
||||
for (let dy = 0; dy < step; dy++) {
|
||||
for (let dx = 0; dx < step; dx++) {
|
||||
sum += gray[(by * step + dy) * N + (bx * step + dx)];
|
||||
count++;
|
||||
}
|
||||
}
|
||||
block.push(sum / count);
|
||||
}
|
||||
}
|
||||
return block;
|
||||
}
|
||||
|
||||
function pHash(block: number[]): Uint8Array {
|
||||
const mean = block.reduce((a, b) => a + b, 0) / HASH_PIXELS;
|
||||
const bits = new Uint8Array(Math.ceil(HASH_PIXELS / 8));
|
||||
for (let i = 0; i < HASH_PIXELS; i++) {
|
||||
if (block[i] >= mean) bits[i >> 3] |= 1 << (i & 7);
|
||||
}
|
||||
return bits;
|
||||
}
|
||||
|
||||
function hammingDistance(a: Uint8Array, b: Uint8Array): number {
|
||||
let dist = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
let x = a[i] ^ b[i];
|
||||
while (x) { dist += x & 1; x >>= 1; }
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
export async function getHash(url: string, priority = -1): Promise<Uint8Array | null> {
|
||||
if (hashCache.has(url)) return hashCache.get(url)!;
|
||||
try {
|
||||
const blob = await getBlobUrl(url, priority);
|
||||
const gray = await loadGrayscale(blob);
|
||||
const block = dct8x8(gray);
|
||||
const hash = pHash(block);
|
||||
hashCache.set(url, hash);
|
||||
return hash;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function areDuplicates(a: Uint8Array, b: Uint8Array): boolean {
|
||||
return hammingDistance(a, b) <= DUPE_THRESH;
|
||||
}
|
||||
|
||||
export function clearHashCache(): void {
|
||||
hashCache.clear();
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
import { store } from "@store/state.svelte";
|
||||
import { searchWithScore } from "@core/algorithms/search";
|
||||
import { getHash, areDuplicates } from "@features/series/lib/coverHash";
|
||||
|
||||
type CoverManga = { id: number; thumbnailUrl: string; source?: { displayName: string } | null };
|
||||
|
||||
export type CoverCandidate = {
|
||||
mangaId: number;
|
||||
url: string;
|
||||
label: string;
|
||||
isActive: boolean;
|
||||
};
|
||||
|
||||
const FUZZY_SCORE_THRESHOLD = 0.5;
|
||||
|
||||
function normalizeUrl(url: string): string {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
u.search = "";
|
||||
return u.href.toLowerCase();
|
||||
} catch {
|
||||
return url.toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
export function resolvedCover(mangaId: number, ownUrl: string): string {
|
||||
return store.settings.mangaPrefs?.[mangaId]?.coverUrl ?? ownUrl;
|
||||
}
|
||||
|
||||
function fuzzyMatchIds(
|
||||
mangaId: number,
|
||||
title: string,
|
||||
mangaById: Map<number, CoverManga & { title: string }>,
|
||||
): number[] {
|
||||
const results = searchWithScore(
|
||||
[...mangaById.values()].filter(m => m.id !== mangaId),
|
||||
title,
|
||||
m => m.title,
|
||||
);
|
||||
return results
|
||||
.filter(r => r.score >= FUZZY_SCORE_THRESHOLD)
|
||||
.map(r => r.item.id);
|
||||
}
|
||||
|
||||
export function coverCandidatesSync(
|
||||
mangaId: number,
|
||||
title: string,
|
||||
ownUrl: string,
|
||||
mangaById: Map<number, CoverManga & { title: string }>,
|
||||
): CoverCandidate[] {
|
||||
const linkedIds = store.getLinkedMangaIds(mangaId);
|
||||
const fuzzyIds = fuzzyMatchIds(mangaId, title, mangaById);
|
||||
const current = store.settings.mangaPrefs?.[mangaId]?.coverUrl ?? ownUrl;
|
||||
|
||||
const allIds = Array.from(new Set([...linkedIds, ...fuzzyIds]));
|
||||
|
||||
const raw: { mangaId: number; url: string; label: string }[] = [
|
||||
{ mangaId, url: ownUrl, label: "This source" },
|
||||
...allIds.flatMap(id => {
|
||||
const m = mangaById.get(id);
|
||||
return m ? [{ mangaId: m.id, url: m.thumbnailUrl, label: m.source?.displayName ?? `ID ${m.id}` }] : [];
|
||||
}),
|
||||
];
|
||||
|
||||
const seen = new Set<string>();
|
||||
return raw
|
||||
.filter(c => {
|
||||
const key = normalizeUrl(c.url);
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
})
|
||||
.map(c => ({ ...c, isActive: normalizeUrl(c.url) === normalizeUrl(current) }));
|
||||
}
|
||||
|
||||
export async function dedupeByImage(candidates: CoverCandidate[]): Promise<CoverCandidate[]> {
|
||||
const hashes = await Promise.all(candidates.map(c => getHash(c.url)));
|
||||
|
||||
const keptIndices: number[] = [];
|
||||
|
||||
for (let i = 0; i < candidates.length; i++) {
|
||||
const hi = hashes[i];
|
||||
if (!hi) { keptIndices.push(i); continue; }
|
||||
|
||||
const isDupe = keptIndices.some(j => {
|
||||
const hj = hashes[j];
|
||||
return hj ? areDuplicates(hi, hj) : false;
|
||||
});
|
||||
|
||||
if (!isDupe) keptIndices.push(i);
|
||||
}
|
||||
|
||||
return keptIndices.map(i => candidates[i]);
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
<script lang="ts">
|
||||
import { X, CaretLeft, CaretRight } from "phosphor-svelte";
|
||||
import { X, CaretLeft, CaretRight, CircleNotch } from "phosphor-svelte";
|
||||
import { setPref } from "@features/series/lib/mangaPrefs";
|
||||
import { coverCandidatesSync, dedupeByImage } from "@features/series/lib/coverResolver";
|
||||
import { coverCandidatesSync, dedupeByImage } from "@core/cover/coverResolver";
|
||||
import Thumbnail from "@shared/manga/Thumbnail.svelte";
|
||||
import type { Manga } from "@types";
|
||||
|
||||
@@ -21,22 +21,19 @@
|
||||
coverCandidatesSync(manga.id, manga.title, manga.thumbnailUrl, mangaById)
|
||||
);
|
||||
|
||||
let candidates = $state(syncCandidates);
|
||||
let candidates = $state<typeof syncCandidates>([]);
|
||||
let hashingDone = $state(false);
|
||||
let index = $state(0);
|
||||
|
||||
$effect(() => {
|
||||
const snap = syncCandidates;
|
||||
candidates = snap;
|
||||
candidates = [];
|
||||
hashingDone = false;
|
||||
index = Math.max(0, snap.findIndex(c => c.isActive));
|
||||
index = 0;
|
||||
|
||||
dedupeByImage(snap).then(deduped => {
|
||||
const activeInDeduped = deduped.some(c => c.isActive);
|
||||
candidates = activeInDeduped
|
||||
? deduped
|
||||
: (() => { const a = snap.find(c => c.isActive); return a ? [a, ...deduped.filter(c => !c.isActive)] : deduped; })();
|
||||
index = Math.max(0, candidates.findIndex(c => c.isActive));
|
||||
dedupeByImage(snap).then(merged => {
|
||||
candidates = merged;
|
||||
index = Math.max(0, merged.findIndex(c => c.isActive));
|
||||
hashingDone = true;
|
||||
});
|
||||
});
|
||||
@@ -70,41 +67,44 @@
|
||||
<div class="modal" role="dialog" aria-label="Choose cover image" onkeydown={onKeydown}>
|
||||
<div class="header">
|
||||
<span class="title">Cover Image</span>
|
||||
{#if !hashingDone}
|
||||
<span class="comparing">Comparing…</span>
|
||||
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
|
||||
</div>
|
||||
|
||||
{#if !hashingDone}
|
||||
<div class="loading">
|
||||
<CircleNotch size={24} weight="light" class="anim-spin" style="color:var(--text-faint)" />
|
||||
</div>
|
||||
{:else}
|
||||
<div class="stage">
|
||||
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
|
||||
<CaretLeft size={18} weight="bold" />
|
||||
</button>
|
||||
|
||||
<div class="cover-wrap">
|
||||
{#if current}
|
||||
<Thumbnail src={current.url} alt="" class="cover-img" />
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
|
||||
<CaretRight size={18} weight="bold" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if candidates.length > 1}
|
||||
<div class="filmstrip">
|
||||
{#each candidates as c, i (c.url)}
|
||||
<button
|
||||
class="film-thumb"
|
||||
class:film-active={i === index}
|
||||
onclick={() => index = i}
|
||||
aria-label="Cover {i + 1}"
|
||||
>
|
||||
<Thumbnail src={c.url} alt="" class="film-img" />
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
|
||||
</div>
|
||||
|
||||
<div class="stage">
|
||||
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
|
||||
<CaretLeft size={18} weight="bold" />
|
||||
</button>
|
||||
|
||||
<div class="cover-wrap">
|
||||
{#if current}
|
||||
<Thumbnail src={current.url} alt="" class="cover-img" />
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
|
||||
<CaretRight size={18} weight="bold" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if candidates.length > 1}
|
||||
<div class="filmstrip">
|
||||
{#each candidates as c, i (c.url)}
|
||||
<button
|
||||
class="film-thumb"
|
||||
class:film-active={i === index}
|
||||
onclick={() => index = i}
|
||||
aria-label="Cover {i + 1}"
|
||||
>
|
||||
<Thumbnail src={c.url} alt="" class="film-img" />
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="footer">
|
||||
@@ -209,6 +209,7 @@
|
||||
transition: opacity var(--t-base);
|
||||
}
|
||||
.confirm-btn:hover { opacity: 0.88; }
|
||||
.loading { display: flex; align-items: center; justify-content: center; padding: var(--sp-10) 0; }
|
||||
@keyframes fadeIn { from { opacity: 0 } to { opacity: 1 } }
|
||||
@keyframes scaleIn { from { opacity: 0; transform: scale(0.97) } to { opacity: 1; transform: scale(1) } }
|
||||
@keyframes pulse { 0%, 100% { opacity: 0.4 } 50% { opacity: 1 } }
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
return others
|
||||
.filter(m => !linkedIds.includes(m.id))
|
||||
.map(m => ({ manga: m, score: titleSimilarity(manga.title, m.title) }))
|
||||
.filter(r => r.score >= 0.4)
|
||||
.filter(r => r.score >= 0.65)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, 8);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user