perf: update emojis, cache in IndexedDb

Closes #9959
May fix #9724
This commit is contained in:
ThatOneCalculator 2023-06-22 19:34:52 -07:00
parent 5e0a0df262
commit d296fee10a
No known key found for this signature in database
GPG Key ID: 8703CACD01000000
8 changed files with 167 additions and 1817 deletions

View File

@ -6,7 +6,7 @@
"type": "git",
"url": "https://codeberg.org/calckey/calckey.git"
},
"packageManager": "pnpm@8.6.2",
"packageManager": "pnpm@8.6.3",
"private": true,
"scripts": {
"rebuild": "pnpm run clean && pnpm -r run build && pnpm run gulp",

View File

@ -44,6 +44,7 @@
"cross-env": "7.0.3",
"cypress": "10.11.0",
"date-fns": "2.30.0",
"emojilib": "^3.0.10",
"escape-regexp": "0.0.1",
"eventemitter3": "4.0.7",
"focus-trap": "^7.4.3",
@ -79,6 +80,7 @@
"tsconfig-paths": "4.2.0",
"twemoji-parser": "14.0.0",
"typescript": "5.1.3",
"unicode-emoji-json": "^0.4.0",
"uuid": "9.0.0",
"vanilla-tilt": "1.8.0",
"vite": "4.3.9",

View File

@ -99,7 +99,7 @@ import { acct } from "@/filters/user";
import * as os from "@/os";
import { MFM_TAGS } from "@/scripts/mfm-tags";
import { defaultStore } from "@/store";
import { emojilist } from "@/scripts/emojilist";
import { getEmojiData } from "@/scripts/emojilist";
import { instance } from "@/instance";
import { i18n } from "@/i18n";
@ -111,22 +111,23 @@ type EmojiDef = {
isCustomEmoji?: boolean;
};
const emojilist = await getEmojiData();
const lib = emojilist.filter((x) => x.category !== "flags");
const emjdb: EmojiDef[] = lib.map((x) => ({
emoji: x.char,
name: x.name,
url: char2filePath(x.char),
emoji: x.emoji,
name: x.slug,
url: char2filePath(x.emoji),
}));
for (const x of lib) {
if (x.keywords) {
for (const k of x.keywords) {
emjdb.push({
emoji: x.char,
emoji: x.emoji,
name: k,
aliasOf: x.name,
url: char2filePath(x.char),
aliasOf: x.slug,
url: char2filePath(x.emoji),
});
}
}

View File

@ -42,13 +42,13 @@
<div v-if="searchResultUnicode.length > 0" class="body">
<button
v-for="emoji in searchResultUnicode"
:key="emoji.name"
:key="emoji.slug"
class="_button item"
:title="emoji.name"
:title="emoji.slug"
tabindex="0"
@click="chosen(emoji, $event)"
>
<MkEmoji class="emoji" :emoji="emoji.char" />
<MkEmoji class="emoji" :emoji="emoji.emoji" />
</button>
</div>
</section>
@ -116,10 +116,12 @@
:emojis="
emojilist
.filter((e) => e.category === category)
.map((e) => e.char)
.map((e) => e.emoji)
"
@chosen="chosen"
>{{ category }}</XSection
>{{
getNicelyLabeledCategory(category) || category
}}</XSection
>
</div>
</div>
@ -162,9 +164,10 @@ import { ref, computed, watch, onMounted } from "vue";
import * as Misskey from "calckey-js";
import XSection from "@/components/MkEmojiPicker.section.vue";
import {
emojilist,
getEmojiData,
UnicodeEmojiDef,
unicodeEmojiCategories as categories,
getNicelyLabeledCategory,
} from "@/scripts/emojilist";
import { getStaticImageUrl } from "@/scripts/get-static-image-url";
import Ripple from "@/components/MkRipple.vue";
@ -216,6 +219,7 @@ const height = computed(() =>
const customEmojiCategories = emojiCategories;
const customEmojis = instance.emojis;
const q = ref<string | null>(null);
const emojilist = await getEmojiData();
const searchResultCustom = ref<Misskey.entities.CustomEmoji[]>([]);
const searchResultUnicode = ref<UnicodeEmojiDef[]>([]);
const tab = ref<"index" | "custom" | "unicode" | "tags">("index");
@ -232,7 +236,7 @@ watch(q, () => {
const newQ = q.value.replace(/:/g, "").toLowerCase();
const searchCustom = () => {
const max = 8;
const max = 16;
const emojis = customEmojis;
const matches = new Set<Misskey.entities.CustomEmoji>();
@ -304,11 +308,11 @@ watch(q, () => {
};
const searchUnicode = () => {
const max = 8;
const max = 32;
const emojis = emojilist;
const matches = new Set<UnicodeEmojiDef>();
const exactMatch = emojis.find((emoji) => emoji.name === newQ);
const exactMatch = emojis.find((emoji) => emoji.slug === newQ);
if (exactMatch) matches.add(exactMatch);
if (newQ.includes(" ")) {
@ -329,8 +333,8 @@ watch(q, () => {
if (
keywords.every(
(keyword) =>
emoji.name.includes(keyword) ||
emoji.keywords.some((alias) =>
emoji.slug.includes(keyword) ||
emoji.keywords?.some((alias) =>
alias.includes(keyword)
)
)
@ -341,7 +345,7 @@ watch(q, () => {
}
} else {
for (const emoji of emojis) {
if (emoji.name.startsWith(newQ)) {
if (emoji.slug.startsWith(newQ)) {
matches.add(emoji);
if (matches.size >= max) break;
}
@ -350,7 +354,7 @@ watch(q, () => {
for (const emoji of emojis) {
if (
emoji.keywords.some((keyword) => keyword.startsWith(newQ))
emoji.keywords?.some((keyword) => keyword.startsWith(newQ))
) {
matches.add(emoji);
if (matches.size >= max) break;
@ -359,7 +363,7 @@ watch(q, () => {
if (matches.size >= max) return matches;
for (const emoji of emojis) {
if (emoji.name.includes(newQ)) {
if (emoji.slug.includes(newQ)) {
matches.add(emoji);
if (matches.size >= max) break;
}
@ -367,7 +371,7 @@ watch(q, () => {
if (matches.size >= max) return matches;
for (const emoji of emojis) {
if (emoji.keywords.some((keyword) => keyword.includes(newQ))) {
if (emoji.keywords?.some((keyword) => keyword.includes(newQ))) {
matches.add(emoji);
if (matches.size >= max) break;
}
@ -441,7 +445,7 @@ function done(query?: any): boolean | void {
return true;
}
const exactMatchUnicode = emojilist.find(
(emoji) => emoji.char === q2 || emoji.name === q2
(emoji) => emoji.emoji === q2 || emoji.slug === q2
);
if (exactMatchUnicode) {
chosen(exactMatchUnicode);

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,9 @@
import data from "unicode-emoji-json/data-by-group.json";
import components from "unicode-emoji-json/data-emoji-components.json";
import keywordSet from "emojilib";
export const unicodeEmojiCategories = [
"face",
"emotion",
"people",
"animals_and_nature",
"food_and_drink",
@ -10,14 +14,125 @@ export const unicodeEmojiCategories = [
"flags",
] as const;
export const categoryMapping = {
"Smileys & Emotion": "emotion",
"People & Body": "people",
"Animals & Nature": "animals_and_nature",
"Food & Drink": "food_and_drink",
"Activities": "activity",
"Travel & Places": "travel_and_places",
"Objects": "objects",
"Symbols": "symbols",
"Flags": "flags",
} as const;
const skinToneModifiers = [
"light_skin_tone",
"medium_light_skin_tone",
"medium_skin_tone",
"medium_dark_skin_tone",
"dark_skin_tone",
];
const newData = {};
Object.keys(data).forEach((originalCategory) => {
const newCategory = categoryMapping[originalCategory];
if (newCategory) {
newData[newCategory] = newData[newCategory] || [];
Object.keys(data[originalCategory]).forEach((emojiIndex) => {
const emojiObj = { ...data[originalCategory][emojiIndex] };
emojiObj.keywords = keywordSet[emojiObj.emoji];
newData[newCategory].push(emojiObj);
if (emojiObj.skin_tone_support) {
skinToneModifiers.forEach((modifier) => {
const modifiedEmojiObj = { ...emojiObj };
modifiedEmojiObj.emoji += components[modifier];
modifiedEmojiObj.skin_tone = modifier;
newData[newCategory].push(modifiedEmojiObj);
});
}
});
}
});
export type UnicodeEmojiDef = {
name: string;
keywords: string[];
char: string;
emoji: string;
category: typeof unicodeEmojiCategories[number];
skin_tone_support: boolean;
name: string;
slug: string;
emoji_version: string;
skin_tone?: string;
keywords?: string[];
};
// initial converted from https://github.com/muan/emojilib/commit/242fe68be86ed6536843b83f7e32f376468b38fb
import _emojilist from "../emojilist.json";
export const emojilist = newData as UnicodeEmojiDef[];
export const emojilist = _emojilist as UnicodeEmojiDef[];
const storeName = "emojiList";
function openDatabase() {
return new Promise<IDBDatabase>((resolve, reject) => {
const openRequest = indexedDB.open("emojiDatabase", 1);
openRequest.onupgradeneeded = () => {
const db = openRequest.result;
if (!db.objectStoreNames.contains(storeName)) {
db.createObjectStore(storeName);
}
};
openRequest.onsuccess = () => {
resolve(openRequest.result);
};
openRequest.onerror = () => {
reject(openRequest.error);
};
});
}
function storeData(db: IDBDatabase, data) {
return new Promise((resolve, reject) => {
const transaction = db.transaction(storeName, "readwrite");
const store = transaction.objectStore(storeName);
store.put(data, "emojiListKey");
transaction.oncomplete = resolve;
transaction.onerror = reject;
});
}
function getData(db: IDBDatabase): Promise<UnicodeEmojiDef[]> {
return new Promise((resolve, reject) => {
const transaction = db.transaction(storeName, "readonly");
const store = transaction.objectStore(storeName);
const getRequest = store.get("emojiListKey");
getRequest.onsuccess = () => resolve(getRequest.result);
getRequest.onerror = reject;
});
}
export async function getEmojiData(): Promise<UnicodeEmojiDef[]> {
try {
const db = await openDatabase();
const cachedData = await getData(db);
if (cachedData) {
return cachedData;
} else {
await storeData(db, emojilist);
console.log("Emoji data stored in IndexedDB");
return emojilist;
}
} catch (err) {
console.error("Error accessing IndexedDB:", err);
return emojilist;
}
}
export function getNicelyLabeledCategory(internalName) {
return Object.keys(categoryMapping).find(
(key) => categoryMapping[key] === internalName
) || internalName;
}

View File

@ -777,6 +777,9 @@ importers:
date-fns:
specifier: 2.30.0
version: 2.30.0
emojilib:
specifier: ^3.0.10
version: 3.0.10
escape-regexp:
specifier: 0.0.1
version: 0.0.1
@ -882,6 +885,9 @@ importers:
typescript:
specifier: 5.1.3
version: 5.1.3
unicode-emoji-json:
specifier: ^0.4.0
version: 0.4.0
uuid:
specifier: 9.0.0
version: 9.0.0
@ -6777,6 +6783,10 @@ packages:
/emoji-regex@9.2.2:
resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
/emojilib@3.0.10:
resolution: {integrity: sha512-VQtCRroFykPTJaoEBEGFg5tI+rEluabjuaVDDbSftDtiRJ5GuqRG/LGV1mmDzkJP4bh5rzuEBOafMN68/YXQcQ==}
dev: true
/emojis-list@3.0.0:
resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==}
engines: {node: '>= 4'}
@ -14866,6 +14876,10 @@ packages:
busboy: 1.6.0
dev: false
/unicode-emoji-json@0.4.0:
resolution: {integrity: sha512-lVNOwh2AnmbwqtSrEVjAWKQoVzWgyWmXVqPuPkPfKb0tnA0+uYN/4ILCTdy9IRj/+3drAVhmjwjNJQr2dhCwnA==}
dev: true
/union-value@1.0.1:
resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==}
engines: {node: '>=0.10.0'}

View File

@ -14,7 +14,6 @@
"packages/backend/src/server/web/manifest.ts",
"packages/backend/built/",
"*/model.json",
"packages/client/src/emojilist.json",
"*.md",
"**/tsconfig.json",
"*/.yml"