gamezonline/Moku
1
0
Fork 0
mirror of https://github.com/moku-project/Moku.git synced 2026-06-13 01:09:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Futile Attempt to Implement Image-Dedupe (#55)

Browse Source
This commit is contained in:
Youwes09
2026-04-28 22:45:19 -05:00
parent 2c1391c378
commit 63209cb828
12 changed files with 124 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/features/series/lib/autoLink.ts → src/core/cover/autoLink.ts
View File
src/features/series/lib/autoLinkWorker.ts → src/core/cover/autoLinkWorker.ts
+1 -1
View File
@@ -22,7 +22,7 @@ self.onmessage = (e: MessageEvent<WorkerMsg>) => {
for (const m of allManga) {
if (m.id === focalId) continue;
if (linkedIds.includes(m.id)) continue;
if (titleSimilarity(focalTitle, m.title) >= 0.4) matches.push(m.id);
if (titleSimilarity(focalTitle, m.title) >= 0.65) matches.push(m.id);
}
self.postMessage(matches);
src/core/cover/coverHash.ts
+54
View File
@@ -0,0 +1,54 @@
const THUMB_SIZE = 16;
const DUPE_THRESH = 0.12;
const hashCache = new Map<string, Uint8ClampedArray>();
function toGray(data: Uint8ClampedArray, pixels: number): Uint8ClampedArray {
const gray = new Uint8ClampedArray(pixels);
for (let i = 0; i < pixels; i++) {
const o = i * 4;
gray[i] = (data[o] * 299 + data[o + 1] * 587 + data[o + 2] * 114) / 1000;
}
return gray;
}
function loadThumb(url: string): Promise<Uint8ClampedArray> {
return new Promise((resolve, reject) => {
const img = new Image();
img.crossOrigin = "anonymous";
img.onload = () => {
const canvas = document.createElement("canvas");
canvas.width = canvas.height = THUMB_SIZE;
const ctx = canvas.getContext("2d")!;
ctx.drawImage(img, 0, 0, THUMB_SIZE, THUMB_SIZE);
resolve(toGray(ctx.getImageData(0, 0, THUMB_SIZE, THUMB_SIZE).data, THUMB_SIZE * THUMB_SIZE));
};
img.onerror = reject;
img.src = url;
});
}
function similarity(a: Uint8ClampedArray, b: Uint8ClampedArray): number {
let diff = 0;
for (let i = 0; i < a.length; i++) diff += Math.abs(a[i] - b[i]);
return diff / (a.length * 255);
}
export async function getHash(url: string): Promise<Uint8ClampedArray | null> {
if (hashCache.has(url)) return hashCache.get(url)!;
try {
const thumb = await loadThumb(url);
hashCache.set(url, thumb);
return thumb;
} catch {
return null;
}
}
export function areDuplicates(a: Uint8ClampedArray, b: Uint8ClampedArray): boolean {
return similarity(a, b) <= DUPE_THRESH;
}
export function clearHashCache(): void {
hashCache.clear();
}
src/features/series/lib/coverResolver.ts → src/core/cover/coverResolver.ts
+13 -12
View File
@@ -1,6 +1,6 @@
import { store } from "@store/state.svelte";
import { searchWithScore } from "@core/algorithms/search";
import { getHash, areDuplicates } from "@features/series/lib/coverHash";
import { getHash, areDuplicates } from "@core/cover/coverHash";
type CoverManga = { id: number; thumbnailUrl: string; source?: { displayName: string } | null };
@@ -11,7 +11,7 @@ export type CoverCandidate = {
isActive: boolean;
};
const FUZZY_SCORE_THRESHOLD = 0.5;
const FUZZY_SCORE_THRESHOLD = 0.65;
function normalizeUrl(url: string): string {
try {
@@ -76,19 +76,20 @@ export function coverCandidatesSync(
export async function dedupeByImage(candidates: CoverCandidate[]): Promise<CoverCandidate[]> {
const hashes = await Promise.all(candidates.map(c => getHash(c.url)));
const keptIndices: number[] = [];
const groups: number[][] = [];
for (let i = 0; i < candidates.length; i++) {
const hi = hashes[i];
if (!hi) { keptIndices.push(i); continue; }
const isDupe = keptIndices.some(j => {
const hj = hashes[j];
return hj ? areDuplicates(hi, hj) : false;
});
if (!isDupe) keptIndices.push(i);
const existing = hi
? groups.find(g => { const hj = hashes[g[0]]; return hj ? areDuplicates(hi, hj) : false; })
: undefined;
if (existing) existing.push(i);
else groups.push([i]);
}
return keptIndices.map(i => candidates[i]);
return groups.map(group => {
const active = group.find(i => candidates[i].isActive) ?? group[0];
const labels = [...new Set(group.map(i => candidates[i].label))];
return { ...candidates[active], label: labels.join(" · ") };
});
}
src/core/cover/index.ts
+4
View File
@@ -0,0 +1,4 @@
export { getHash, areDuplicates, clearHashCache } from "./coverHash";
export { resolvedCover, coverCandidatesSync, dedupeByImage } from "./coverResolver";
export type { CoverCandidate } from "./coverResolver";
export { autoLinkLibrary } from "./autoLink";
src/features/library/components/LibraryGrid.svelte
+1 -1
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import { Folder, Trash, CheckSquare, Robot } from "phosphor-svelte";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import { resolvedCover } from "@features/series/lib/coverResolver";
import { resolvedCover } from "@core/cover/coverResolver";
import type { Manga, Category } from "@types";
interface Props {
src/features/series/components/SeriesDetail.svelte
+1 -1
View File
@@ -35,7 +35,7 @@
import ChapterList from "./ChapterList.svelte";
import { buildChapterList, chaptersAscending } from "../lib/chapterList";
import { getPref, setPref } from "../lib/mangaPrefs";
import { autoLinkLibrary } from "@features/series/lib/autoLink";
import { autoLinkLibrary } from "@core/cover/autoLink";
const CHAPTERS_PER_PAGE = 25;
const MANGA_TTL_MS = 5 * 60 * 1000;
src/features/series/components/SeriesHeader.svelte
+1 -1
View File
@@ -5,7 +5,7 @@
MapPin, Gear, Trash, Image,
} from "phosphor-svelte";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import { resolvedCover } from "@features/series/lib/coverResolver";
import { resolvedCover } from "@core/cover/coverResolver";
import type { Manga, Chapter, Category } from "@types";
import type { MangaPrefs } from "@store/state.svelte";
import { store, setActiveManga, setGenreFilter, setNavPage, setPreviewManga, linkManga, unlinkManga } from "@store/state.svelte";
src/features/series/lib/coverHash.ts
-89
View File
@@ -1,89 +0,0 @@
import { getBlobUrl } from "@core/cache/imageCache";
const HASH_SIZE = 8;
const HASH_PIXELS = HASH_SIZE * HASH_SIZE;
const CANVAS_SIZE = 32;
const DUPE_THRESH = 10;
const hashCache = new Map<string, Uint8Array>();
async function loadGrayscale(blobUrl: string): Promise<Uint8ClampedArray> {
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => {
const canvas = document.createElement("canvas");
canvas.width = canvas.height = CANVAS_SIZE;
const ctx = canvas.getContext("2d")!;
ctx.drawImage(img, 0, 0, CANVAS_SIZE, CANVAS_SIZE);
const { data } = ctx.getImageData(0, 0, CANVAS_SIZE, CANVAS_SIZE);
const gray = new Uint8ClampedArray(CANVAS_SIZE * CANVAS_SIZE);
for (let i = 0; i < gray.length; i++) {
const o = i * 4;
gray[i] = (data[o] * 299 + data[o + 1] * 587 + data[o + 2] * 114) / 1000;
}
resolve(gray);
};
img.onerror = reject;
img.src = blobUrl;
});
}
function dct8x8(gray: Uint8ClampedArray): number[] {
const N = CANVAS_SIZE;
const step = N / HASH_SIZE;
const block: number[] = [];
for (let by = 0; by < HASH_SIZE; by++) {
for (let bx = 0; bx < HASH_SIZE; bx++) {
let sum = 0, count = 0;
for (let dy = 0; dy < step; dy++) {
for (let dx = 0; dx < step; dx++) {
sum += gray[(by * step + dy) * N + (bx * step + dx)];
count++;
}
}
block.push(sum / count);
}
}
return block;
}
function pHash(block: number[]): Uint8Array {
const mean = block.reduce((a, b) => a + b, 0) / HASH_PIXELS;
const bits = new Uint8Array(Math.ceil(HASH_PIXELS / 8));
for (let i = 0; i < HASH_PIXELS; i++) {
if (block[i] >= mean) bits[i >> 3] |= 1 << (i & 7);
}
return bits;
}
function hammingDistance(a: Uint8Array, b: Uint8Array): number {
let dist = 0;
for (let i = 0; i < a.length; i++) {
let x = a[i] ^ b[i];
while (x) { dist += x & 1; x >>= 1; }
}
return dist;
}
export async function getHash(url: string, priority = -1): Promise<Uint8Array | null> {
if (hashCache.has(url)) return hashCache.get(url)!;
try {
const blob = await getBlobUrl(url, priority);
const gray = await loadGrayscale(blob);
const block = dct8x8(gray);
const hash = pHash(block);
hashCache.set(url, hash);
return hash;
} catch {
return null;
}
}
export function areDuplicates(a: Uint8Array, b: Uint8Array): boolean {
return hammingDistance(a, b) <= DUPE_THRESH;
}
export function clearHashCache(): void {
hashCache.clear();
}
src/features/series/panels/CoverPickerPanel.svelte
+46 -45
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import { X, CaretLeft, CaretRight } from "phosphor-svelte";
import { X, CaretLeft, CaretRight, CircleNotch } from "phosphor-svelte";
import { setPref } from "@features/series/lib/mangaPrefs";
import { coverCandidatesSync, dedupeByImage } from "@features/series/lib/coverResolver";
import { coverCandidatesSync, dedupeByImage } from "@core/cover/coverResolver";
import Thumbnail from "@shared/manga/Thumbnail.svelte";
import type { Manga } from "@types";
@@ -21,22 +21,19 @@
coverCandidatesSync(manga.id, manga.title, manga.thumbnailUrl, mangaById)
);
let candidates = $state(syncCandidates);
let candidates = $state<typeof syncCandidates>([]);
let hashingDone = $state(false);
let index = $state(0);
$effect(() => {
const snap = syncCandidates;
candidates = snap;
candidates = [];
hashingDone = false;
index = Math.max(0, snap.findIndex(c => c.isActive));
index = 0;
dedupeByImage(snap).then(deduped => {
const activeInDeduped = deduped.some(c => c.isActive);
candidates = activeInDeduped
? deduped
: (() => { const a = snap.find(c => c.isActive); return a ? [a, ...deduped.filter(c => !c.isActive)] : deduped; })();
index = Math.max(0, candidates.findIndex(c => c.isActive));
dedupeByImage(snap).then(merged => {
candidates = merged;
index = Math.max(0, merged.findIndex(c => c.isActive));
hashingDone = true;
});
});
@@ -70,41 +67,44 @@
<div class="modal" role="dialog" aria-label="Choose cover image" onkeydown={onKeydown}>
<div class="header">
<span class="title">Cover Image</span>
{#if !hashingDone}
<span class="comparing">Comparing…</span>
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
</div>
{#if !hashingDone}
<div class="loading">
<CircleNotch size={24} weight="light" class="anim-spin" style="color:var(--text-faint)" />
</div>
{:else}
<div class="stage">
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
<CaretLeft size={18} weight="bold" />
</button>
<div class="cover-wrap">
{#if current}
<Thumbnail src={current.url} alt="" class="cover-img" />
{/if}
</div>
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
<CaretRight size={18} weight="bold" />
</button>
</div>
{#if candidates.length > 1}
<div class="filmstrip">
{#each candidates as c, i (c.url)}
<button
class="film-thumb"
class:film-active={i === index}
onclick={() => index = i}
aria-label="Cover {i + 1}"
>
<Thumbnail src={c.url} alt="" class="film-img" />
</button>
{/each}
</div>
{/if}
<button class="close-btn" onclick={onClose}><X size={14} weight="light" /></button>
</div>
<div class="stage">
<button class="arrow" onclick={prev} disabled={candidates.length <= 1} aria-label="Previous">
<CaretLeft size={18} weight="bold" />
</button>
<div class="cover-wrap">
{#if current}
<Thumbnail src={current.url} alt="" class="cover-img" />
{/if}
</div>
<button class="arrow" onclick={next} disabled={candidates.length <= 1} aria-label="Next">
<CaretRight size={18} weight="bold" />
</button>
</div>
{#if candidates.length > 1}
<div class="filmstrip">
{#each candidates as c, i (c.url)}
<button
class="film-thumb"
class:film-active={i === index}
onclick={() => index = i}
aria-label="Cover {i + 1}"
>
<Thumbnail src={c.url} alt="" class="film-img" />
</button>
{/each}
</div>
{/if}
<div class="footer">
@@ -209,6 +209,7 @@
transition: opacity var(--t-base);
}
.confirm-btn:hover { opacity: 0.88; }
.loading { display: flex; align-items: center; justify-content: center; padding: var(--sp-10) 0; }
@keyframes fadeIn { from { opacity: 0 } to { opacity: 1 } }
@keyframes scaleIn { from { opacity: 0; transform: scale(0.97) } to { opacity: 1; transform: scale(1) } }
@keyframes pulse { 0%, 100% { opacity: 0.4 } 50% { opacity: 1 } }
src/features/series/panels/SeriesLinkPanel.svelte
+1 -1
View File
@@ -33,7 +33,7 @@
return others
.filter(m => !linkedIds.includes(m.id))
.map(m => ({ manga: m, score: titleSimilarity(manga.title, m.title) }))
.filter(r => r.score >= 0.4)
.filter(r => r.score >= 0.65)
.sort((a, b) => b.score - a.score)
.slice(0, 8);
});
src/shared/manga/MangaPreview.svelte
+2 -2
View File
@@ -14,10 +14,10 @@
setPreviewManga, setActiveManga, setNavPage, setGenreFilter,
checkAndMarkCompleted as storeCheckAndMarkCompleted, addBookmark,
} from "@store/state.svelte";
import { resolvedCover } from "@features/series/lib/coverResolver";
import { resolvedCover } from "@core/cover/coverResolver";
import CoverPickerPanel from "@features/series/panels/CoverPickerPanel.svelte";
import SeriesLinkPanel from "@features/series/panels/SeriesLinkPanel.svelte";
import { autoLinkLibrary } from "@features/series/lib/autoLink";
import { autoLinkLibrary } from "@core/cover/autoLink";
import type { Manga, Chapter, Category } from "@types/index";